From 6a2ba4baca326272c45930208b70cfedf8cb1638 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Tue, 1 May 2018 16:32:15 -0700 Subject: Introduced basic multipath support --- include/ZeroTierOne.h | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'include/ZeroTierOne.h') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 6da53a73..15e15bd1 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -422,6 +422,42 @@ enum ZT_ResultCode */ #define ZT_ResultCode_isFatal(x) ((((int)(x)) >= 100)&&(((int)(x)) < 1000)) +/** + * The multipath algorithm in use by this node. + */ +enum ZT_MultipathMode +{ + /** + * No active multipath. + * + * Traffic is merely sent over the strongest path. That being + * said, this mode will automatically failover in the event that a link goes down. + */ + ZT_MULTIPATH_NONE = 0, + + /** + * Traffic is randomly distributed among all active paths. + * + * Will cease sending traffic over links that appear to be stale. + */ + ZT_MULTIPATH_RANDOM = 1, + + /** + * Traffic is allocated across all active paths in proportion to their strength and + * reliability. + * + * Will cease sending traffic over links that appear to be stale. + */ + ZT_MULTIPATH_PROPORTIONALLY_BALANCED = 2, + + /** + * Traffic is allocated across a user-defined interface/allocation + * + * Will cease sending traffic over links that appear to be stale. + */ + ZT_MULTIPATH_MANUALLY_BALANCED = 3 +}; + /** * Status codes sent to status update callback when things happen */ -- cgit v1.2.3 From 9681fedbb44a25ffa1108b88a4795e017746ca6c Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 7 Jun 2018 17:25:27 -0700 Subject: Spellcheck sweep across codebase --- RELEASE-NOTES.md | 8 ++++---- include/ZeroTierOne.h | 8 ++++---- node/Capability.hpp | 6 +++--- node/CertificateOfMembership.hpp | 2 +- node/Constants.hpp | 4 ++-- node/Identity.cpp | 4 ++-- node/IncomingPacket.cpp | 2 +- node/InetAddress.cpp | 26 +++++++++++++------------- node/MulticastGroup.hpp | 2 +- node/Network.hpp | 2 +- node/NetworkConfig.hpp | 2 +- node/Packet.hpp | 18 +++++++++--------- node/Path.hpp | 6 +++--- node/Peer.cpp | 2 +- node/Peer.hpp | 2 +- node/Salsa20.cpp | 2 +- node/SelfAwareness.cpp | 2 +- node/Tag.hpp | 2 +- node/Topology.cpp | 2 +- osdep/OSUtils.cpp | 2 +- osdep/WindowsEthernetTap.hpp | 2 +- service/OneService.cpp | 20 ++++---------------- 22 files changed, 57 insertions(+), 69 deletions(-) (limited to 'include/ZeroTierOne.h') diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 54dd1375..bec144f0 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -36,7 +36,7 @@ ZeroTier Release Notes * Fixed two very rare multithreading issues that were only observed on certain systems * Platform-Specific Changes * MacOS - * Installer now loads the kernel extension right away so that High Sierra users will see the prompt to authorize it. This is done in the "Security & Privacy" preference pane and must be done driectly on the console (not via remote desktop). On High Sierra and newer kexts must be authorized at the console via security settings system preferences pane. + * Installer now loads the kernel extension right away so that High Sierra users will see the prompt to authorize it. This is done in the "Security & Privacy" preference pane and must be done directly on the console (not via remote desktop). On High Sierra and newer kexts must be authorized at the console via security settings system preferences pane. * Windows * The Windows installer should now install the driver without requiring a special prompt in most cases. This should make it easier for our packages to be accepted into and updated in the Chocolatey repository and should make it easier to perform remote installs across groups of machines using IT management and provisioning tools. * The Windows official packages are now signed with an EV certificate (with hardware key). @@ -78,7 +78,7 @@ The largest new feature in 1.2.0, and the product of many months of work, is our Rules allow you to filter packets on your network and vector traffic to security observers. Security observation can be performed in-band using REDIRECT or out of band using TEE. -Tags and capabilites provide advanced methods for implementing fine grained permission structures and micro-segmentation schemes without bloating the size and complexity of your rules table. +Tags and capabilities provide advanced methods for implementing fine grained permission structures and micro-segmentation schemes without bloating the size and complexity of your rules table. See the [rules engine announcement blog post](https://www.zerotier.com/blog/?p=927) for an in-depth discussion of theory and implementation. The [manual](https://www.zerotier.com/manual.shtml) contains detailed information on rule, tag, and capability use, and the `rule-compiler/` subfolder of the ZeroTier source tree contains a JavaScript function to compile rules in our human-readable rule definition language into rules suitable for import into a network controller. (ZeroTier Central uses this same script to compile rules on [my.zerotier.com](https://my.zerotier.com/).) @@ -161,7 +161,7 @@ A special kind of public network called an ad-hoc network may be accessed by joi | Start of port range (hex) Reserved ZeroTier address prefix indicating a controller-less network -Ad-hoc networks are public (no access control) networks that have no network controller. Instead their configuration and other credentials are generated locally. Ad-hoc networks permit only IPv6 UDP and TCP unicast traffic (no multicast or broadcast) using 6plane format NDP-emulated IPv6 addresses. In addition an ad-hoc network ID encodes an IP port range. UDP packets and TCP SYN (connection open) packets are only allowed to desintation ports within the encoded range. +Ad-hoc networks are public (no access control) networks that have no network controller. Instead their configuration and other credentials are generated locally. Ad-hoc networks permit only IPv6 UDP and TCP unicast traffic (no multicast or broadcast) using 6plane format NDP-emulated IPv6 addresses. In addition an ad-hoc network ID encodes an IP port range. UDP packets and TCP SYN (connection open) packets are only allowed to destination ports within the encoded range. For example `ff00160016000000` is an ad-hoc network allowing only SSH, while `ff0000ffff000000` is an ad-hoc network allowing any UDP or TCP port. @@ -176,7 +176,7 @@ If you have data in an old SQLite3 controller we've included a NodeJS script in ## Major Bug Fixes in 1.2.0 * **The Windows HyperV 100% CPU bug is FINALLY DEAD**: This long-running problem turns out to have been an issue with Windows itself, but one we were triggering by placing invalid data into the Windows registry. Microsoft is aware of the issue but we've also fixed the triggering problem on our side. ZeroTier should now co-exist quite well with HyperV and should now be able to be bridged with a HyperV virtual switch. - * **Segmenation faults on musl-libc based Linux systems**: Alpine Linux and some embedded Linux systems that use musl libc (a minimal libc) experienced segmentation faults. These were due to a smaller default stack size. A work-around that sets the stack size for new threads has been added. + * **Segmentation faults on musl-libc based Linux systems**: Alpine Linux and some embedded Linux systems that use musl libc (a minimal libc) experienced segmentation faults. These were due to a smaller default stack size. A work-around that sets the stack size for new threads has been added. * **Windows firewall blocks local JSON API**: On some Windows systems the firewall likes to block 127.0.0.1:9993 for mysterious reasons. This is now fixed in the installer via the addition of another firewall exemption rule. * **UI crash on embedded Windows due to missing fonts**: The MSI installer now ships fonts and will install them if they are not present, so this should be fixed. diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 15e15bd1..b3927c2e 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -411,7 +411,7 @@ enum ZT_ResultCode ZT_RESULT_ERROR_UNSUPPORTED_OPERATION = 1001, /** - * The requestion operation was given a bad parameter or was called in an invalid state + * The requested operation was given a bad parameter or was called in an invalid state */ ZT_RESULT_ERROR_BAD_PARAMETER = 1002 }; @@ -1498,7 +1498,7 @@ typedef int (*ZT_WirePacketSendFunction)( /** * Function to check whether a path should be used for ZeroTier traffic * - * Paramters: + * Parameters: * (1) Node * (2) User pointer * (3) ZeroTier address or 0 for none/any @@ -1531,7 +1531,7 @@ typedef int (*ZT_PathCheckFunction)( * (1) Node * (2) User pointer * (3) ZeroTier address (least significant 40 bits) - * (4) Desried address family or -1 for any + * (4) Desired address family or -1 for any * (5) Buffer to fill with result * * If provided this function will be occasionally called to get physical @@ -1696,7 +1696,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void * Join a network * * This may generate calls to the port config callback before it returns, - * or these may be deffered if a netconf is not available yet. + * or these may be differed if a netconf is not available yet. * * If we are already a member of the network, nothing is done and OK is * returned. diff --git a/node/Capability.hpp b/node/Capability.hpp index 91a46566..775532d9 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -52,7 +52,7 @@ class RuntimeEnvironment; * (1) Evaluates its capabilities in ascending order of ID to determine * which capability allows it to transmit this packet. * (2) If it has not done so lately, it then sends this capability to the - * receving peer ("presents" it). + * receiving peer ("presents" it). * (3) The sender then sends the packet. * * On the receiving side the receiver evaluates the capabilities presented @@ -64,7 +64,7 @@ class RuntimeEnvironment; * * Capabilities support a chain of custody. This is currently unused but * in the future would allow the publication of capabilities that can be - * handed off between nodes. Limited transferrability of capabilities is + * handed off between nodes. Limited transferability of capabilities is * a feature of true capability based security. */ class Capability : public Credential @@ -81,7 +81,7 @@ public: * @param id Capability ID * @param nwid Network ID * @param ts Timestamp (at controller) - * @param mccl Maximum custody chain length (1 to create non-transferrable capability) + * @param mccl Maximum custody chain length (1 to create non-transferable capability) * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index b5a90007..7fd38ad7 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -243,7 +243,7 @@ public: * Compare two certificates for parameter agreement * * This compares this certificate with the other and returns true if all - * paramters in this cert are present in the other and if they agree to + * parameters in this cert are present in the other and if they agree to * within this cert's max delta value for each given parameter. * * Tuples present in other but not in this cert are ignored, but any diff --git a/node/Constants.hpp b/node/Constants.hpp index b85c1b93..d86775c5 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -275,13 +275,13 @@ #define ZT_MULTIPATH_BINDER_REFRESH_PERIOD 5000 /** - * Time horizon for VERB_QOS_MEASUREMENT and VERB_ACK packet processesing cutoff + * Time horizon for VERB_QOS_MEASUREMENT and VERB_ACK packet processing cutoff */ #define ZT_PATH_QOS_ACK_CUTOFF_TIME 30000 /** * Maximum number of VERB_QOS_MEASUREMENT and VERB_ACK packets allowed to be - * processesed within cutoff time. Separate totals are kept for each type but + * processed within cutoff time. Separate totals are kept for each type but * the limit is the same for both. * * This limits how often this peer will compute statistical estimates diff --git a/node/Identity.cpp b/node/Identity.cpp index 03f27083..1687746b 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -50,8 +50,8 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub SHA512::hash(digest,publicKey,publicKeyBytes); // Initialize genmem[] using Salsa20 in a CBC-like configuration since - // ordinary Salsa20 is randomly seekable. This is good for a cipher - // but is not what we want for sequential memory-harndess. + // ordinary Salsa20 is randomly seek-able. This is good for a cipher + // but is not what we want for sequential memory-hardness. memset(genmem,0,ZT_IDENTITY_GEN_MEMORY); Salsa20 s20(digest,(char *)digest + 32); s20.crypt20((char *)genmem,(char *)genmem,64); diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 70dcff5d..6bb9734c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -237,7 +237,7 @@ bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment *RR,void *tPtr, char *ptr = begin; int count = 0; int len = payloadLength(); - // Read packet IDs and latency compensation intervals for each packet tracked by thie QoS packet + // Read packet IDs and latency compensation intervals for each packet tracked by this QoS packet while (ptr < (begin + len) && (count < ZT_PATH_QOS_TABLE_SIZE)) { memcpy((void*)&rx_id[count], ptr, sizeof(uint64_t)); ptr+=sizeof(uint64_t); diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 36b4e434..5d6ad07f 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -5,7 +5,7 @@ * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or - * (at your oion) any later version. + * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -62,23 +62,23 @@ InetAddress::IpScope InetAddress::ipScope() const case 0x37: return IP_SCOPE_PSEUDOPRIVATE; // 55.0.0.0/8 (US DoD) case 0x38: return IP_SCOPE_PSEUDOPRIVATE; // 56.0.0.0/8 (US Postal Service) case 0x64: - if ((ip & 0xffc00000) == 0x64400000) return IP_SCOPE_PRIVATE; // 100.64.0.0/10 + if ((ip & 0xffc00000) == 0x64400000) return IP_SCOPE_PRIVATE; // 100.64.0.0/10 break; case 0x7f: return IP_SCOPE_LOOPBACK; // 127.0.0.0/8 case 0xa9: - if ((ip & 0xffff0000) == 0xa9fe0000) return IP_SCOPE_LINK_LOCAL; // 169.254.0.0/16 + if ((ip & 0xffff0000) == 0xa9fe0000) return IP_SCOPE_LINK_LOCAL; // 169.254.0.0/16 break; case 0xac: - if ((ip & 0xfff00000) == 0xac100000) return IP_SCOPE_PRIVATE; // 172.16.0.0/12 + if ((ip & 0xfff00000) == 0xac100000) return IP_SCOPE_PRIVATE; // 172.16.0.0/12 break; case 0xc0: - if ((ip & 0xffff0000) == 0xc0a80000) return IP_SCOPE_PRIVATE; // 192.168.0.0/16 + if ((ip & 0xffff0000) == 0xc0a80000) return IP_SCOPE_PRIVATE; // 192.168.0.0/16 break; case 0xff: return IP_SCOPE_NONE; // 255.0.0.0/8 (broadcast, or unused/unusable) } switch(ip >> 28) { - case 0xe: return IP_SCOPE_MULTICAST; // 224.0.0.0/4 - case 0xf: return IP_SCOPE_PSEUDOPRIVATE; // 240.0.0.0/4 ("reserved," usually unusable) + case 0xe: return IP_SCOPE_MULTICAST; // 224.0.0.0/4 + case 0xf: return IP_SCOPE_PSEUDOPRIVATE; // 240.0.0.0/4 ("reserved," usually unusable) } return IP_SCOPE_GLOBAL; } break; @@ -86,21 +86,21 @@ InetAddress::IpScope InetAddress::ipScope() const case AF_INET6: { const unsigned char *ip = reinterpret_cast(reinterpret_cast(this)->sin6_addr.s6_addr); if ((ip[0] & 0xf0) == 0xf0) { - if (ip[0] == 0xff) return IP_SCOPE_MULTICAST; // ff00::/8 + if (ip[0] == 0xff) return IP_SCOPE_MULTICAST; // ff00::/8 if ((ip[0] == 0xfe)&&((ip[1] & 0xc0) == 0x80)) { unsigned int k = 2; while ((!ip[k])&&(k < 15)) ++k; if ((k == 15)&&(ip[15] == 0x01)) - return IP_SCOPE_LOOPBACK; // fe80::1/128 - else return IP_SCOPE_LINK_LOCAL; // fe80::/10 + return IP_SCOPE_LOOPBACK; // fe80::1/128 + else return IP_SCOPE_LINK_LOCAL; // fe80::/10 } - if ((ip[0] & 0xfe) == 0xfc) return IP_SCOPE_PRIVATE; // fc00::/7 + if ((ip[0] & 0xfe) == 0xfc) return IP_SCOPE_PRIVATE; // fc00::/7 } unsigned int k = 0; while ((!ip[k])&&(k < 15)) ++k; if (k == 15) { // all 0's except last byte - if (ip[15] == 0x01) return IP_SCOPE_LOOPBACK; // ::1/128 - if (ip[15] == 0x00) return IP_SCOPE_NONE; // ::/128 + if (ip[15] == 0x01) return IP_SCOPE_LOOPBACK; // ::1/128 + if (ip[15] == 0x00) return IP_SCOPE_NONE; // ::/128 } return IP_SCOPE_GLOBAL; } break; diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index 0f4a621e..0a318136 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -68,7 +68,7 @@ public: * Derive the multicast group used for address resolution (ARP/NDP) for an IP * * @param ip IP address (port field is ignored) - * @return Multicat group for ARP/NDP + * @return Multicast group for ARP/NDP */ static inline MulticastGroup deriveMulticastGroupForAddressResolution(const InetAddress &ip) { diff --git a/node/Network.hpp b/node/Network.hpp index 95b5483a..38f03eb3 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -438,6 +438,6 @@ private: AtomicCounter __refCount; }; -} // naemspace ZeroTier +} // namespace ZeroTier #endif diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 44066c86..8900bda5 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -562,7 +562,7 @@ public: char name[ZT_MAX_NETWORK_SHORT_NAME_LENGTH + 1]; /** - * Certficiate of membership (for private networks) + * Certificate of membership (for private networks) */ CertificateOfMembership com; }; diff --git a/node/Packet.hpp b/node/Packet.hpp index 6869691e..8e82bd34 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -150,7 +150,7 @@ * * In cryptography, a "break" means something different from what it means in * common discussion. If a cipher is 256 bits strong and someone finds a way - * to reduce key search to 254 bits, this constitues a "break" in the academic + * to reduce key search to 254 bits, this constitutes a "break" in the academic * literature. 254 bits is still far beyond what can be leveraged to accomplish * a "break" as most people would understand it -- the actual decryption and * reading of traffic. @@ -249,7 +249,7 @@ */ #define ZT_PROTO_MIN_FRAGMENT_LENGTH ZT_PACKET_FRAGMENT_IDX_PAYLOAD -// Field incides for parsing verbs ------------------------------------------- +// Field indices for parsing verbs ------------------------------------------- // Some verbs have variable-length fields. Those aren't fully defined here // yet-- instead they are parsed using relative indexes in IncomingPacket. @@ -734,7 +734,7 @@ public: * Credentials can be for any number of networks. * * The use of a zero byte to terminate the COM section is for legacy - * backward compatiblity. Newer fields are prefixed with a length. + * backward compatibility. Newer fields are prefixed with a length. * * OK/ERROR are not generated. */ @@ -751,7 +751,7 @@ public: * This message requests network configuration from a node capable of * providing it. * - * Respones to this are always whole configs intended for the recipient. + * Responses to this are always whole configs intended for the recipient. * For patches and other updates a NETWORK_CONFIG is sent instead. * * It would be valid and correct as of 1.2.0 to use NETWORK_CONFIG always, @@ -884,7 +884,7 @@ public: * <[6] MAC address of multicast group> * <[4] 32-bit ADI for multicast group> * <[1] flags> - * [<[...] network certficate of membership (DEPRECATED)>] + * [<[...] network certificate of membership (DEPRECATED)>] * [<[...] implicit gather results if flag 0x01 is set>] * * OK flags (same bits as request flags): @@ -933,7 +933,7 @@ public: // 0x11 -- deprecated /** - * An acknowledgement of receipt of a series of recent packets from another + * An acknowledgment of receipt of a series of recent packets from another * peer. This is used to calculate relative throughput values and to detect * packet loss. Only VERB_FRAME and VERB_EXT_FRAME packets are counted. * @@ -967,7 +967,7 @@ public: * The number of possible records per QoS packet is: (1400 * 8) / 72 = 155 * This packet should be sent very rarely (every few seconds) as it can be * somewhat large if the connection is saturated. Future versions might use - * a bloom table to probablistically determine these values in a vastly + * a bloom table to probabilistically determine these values in a vastly * more space-efficient manner. * * Note: The 'internal packet sojourn time' is a slight misnomer as it is a @@ -1000,7 +1000,7 @@ public: * * This message contains a remote trace event. Remote trace events can * be sent to observers configured at the network level for those that - * pertain directly to actiity on a network, or to global observers if + * pertain directly to activity on a network, or to global observers if * locally configured. * * The instance ID is a random 64-bit value generated by each ZeroTier @@ -1297,7 +1297,7 @@ public: * Encrypt/decrypt a separately armored portion of a packet * * This is currently only used to mask portions of HELLO as an extra - * security precation since most of that message is sent in the clear. + * security precaution since most of that message is sent in the clear. * * This must NEVER be used more than once in the same packet, as doing * so will result in re-use of the same key stream. diff --git a/node/Path.hpp b/node/Path.hpp index e7448190..80a7895a 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -353,7 +353,7 @@ public: * Record that we've received a VERB_ACK on this path, also compute throughput if required. * * @param now Current time - * @param ackedBytes Number of bytes awknowledged by other peer + * @param ackedBytes Number of bytes acknowledged by other peer */ inline void receivedAck(int64_t now, int32_t ackedBytes) { @@ -387,7 +387,7 @@ public: } /** - * @return Number of bytes thusfar sent that have not been awknowledged by the remote peer + * @return Number of bytes thus far sent that have not been acknowledged by the remote peer */ inline int64_t unackedSentBytes() { @@ -529,7 +529,7 @@ public: inline char *getName() { return _ifname; } /** - * @return Packet delay varience + * @return Packet delay variance */ inline float packetDelayVariance() { return _packetDelayVariance; } diff --git a/node/Peer.cpp b/node/Peer.cpp index a22bbffe..877ec2fd 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -140,7 +140,7 @@ void Peer::received( if ((!havePath)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) { Mutex::Lock _l(_paths_m); - // Paths are redunant if they duplicate an alive path to the same IP or + // Paths are redundant if they duplicate an alive path to the same IP or // with the same local socket and address family. bool redundant = false; for(unsigned int i=0;i SelfAwareness::getSymmetricNatPredictions() * * Since flows are encrypted and authenticated they could not actually * read or modify traffic, but they could gather meta-data for forensics - * purpsoes or use this as a DOS attack vector. */ + * purposes or use this as a DOS attack vector. */ std::map< uint32_t,unsigned int > maxPortByIp; InetAddress theOneTrueSurface; diff --git a/node/Tag.hpp b/node/Tag.hpp index d2e932c2..1d3e97fa 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -58,7 +58,7 @@ class RuntimeEnvironment; * values. * * Unlike capabilities tags are signed only by the issuer and are never - * transferrable. + * transferable. */ class Tag : public Credential { diff --git a/node/Topology.cpp b/node/Topology.cpp index 7c526b41..7e32f205 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -138,7 +138,7 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) } return SharedPtr(); } - } catch ( ... ) {} // ignore invalid identities or other strage failures + } catch ( ... ) {} // ignore invalid identities or other strange failures return SharedPtr(); } diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index cadd4e6b..8b7fd948 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -366,7 +366,7 @@ std::vector OSUtils::split(const char *s,const char *const sep,cons if (buf.size() > 0) { fields.push_back(buf); buf.clear(); - } // else skip runs of seperators + } // else skip runs of separators } else buf.push_back(*s); } ++s; diff --git a/osdep/WindowsEthernetTap.hpp b/osdep/WindowsEthernetTap.hpp index 1e36bdd8..4f0f89c4 100644 --- a/osdep/WindowsEthernetTap.hpp +++ b/osdep/WindowsEthernetTap.hpp @@ -71,7 +71,7 @@ public: static std::string destroyAllPersistentTapDevices(); /** - * Uninstall a specific persistent tap device by instance ID + * Uninstalls a specific persistent tap device by instance ID * * @param instanceId Device instance ID * @return Empty string on success, otherwise an error message diff --git a/service/OneService.cpp b/service/OneService.cpp index 152dca7b..e56a4827 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -422,7 +422,7 @@ public: unsigned int _primaryPort; volatile unsigned int _udpPortPickerCounter; - // Local configuration and memo-ized information from it + // Local configuration and memoized information from it json _localConfig; Hashtable< uint64_t,std::vector > _v4Hints; Hashtable< uint64_t,std::vector > _v6Hints; @@ -438,7 +438,7 @@ public: * To attempt to handle NAT/gateway craziness we use three local UDP ports: * * [0] is the normal/default port, usually 9993 - * [1] is a port dervied from our ZeroTier address + * [1] is a port derived from our ZeroTier address * [2] is a port computed from the normal/default for use with uPnP/NAT-PMP mappings * * [2] exists because on some gateways trying to do regular NAT-t interferes @@ -1130,16 +1130,7 @@ public: #ifdef __SYNOLOGY__ // Authenticate via Synology's built-in cgi script if (!isAuth) { - /* - fprintf(stderr, "path = %s\n", path.c_str()); - fprintf(stderr, "headers.size=%d\n", headers.size()); - std::map::const_iterator it(headers.begin()); - while(it != headers.end()) { - fprintf(stderr,"header[%s] = %s\n", (it->first).c_str(), (it->second).c_str()); - it++; - } - */ - // parse out url args + // Parse out url args int synotoken_pos = path.find("SynoToken"); int argpos = path.find("?"); if(synotoken_pos != std::string::npos && argpos != std::string::npos) { @@ -1152,10 +1143,7 @@ public: setenv("HTTP_COOKIE", cookie_val.c_str(), true); setenv("HTTP_X_SYNO_TOKEN", synotoken_val.c_str(), true); setenv("REMOTE_ADDR", ah2->second.c_str(),true); - //fprintf(stderr, "HTTP_COOKIE: %s\n",std::getenv ("HTTP_COOKIE")); - //fprintf(stderr, "HTTP_X_SYNO_TOKEN: %s\n",std::getenv ("HTTP_X_SYNO_TOKEN")); - //fprintf(stderr, "REMOTE_ADDR: %s\n",std::getenv ("REMOTE_ADDR")); - // check synology web auth + // Check Synology web auth char user[256], buf[1024]; FILE *fp = NULL; bzero(user, 256); -- cgit v1.2.3 From 17fbb020e733fab8d8be933bb4981927015a10f5 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 14 Jun 2018 16:34:45 -0700 Subject: Added multipath field to zerotier-cli status output. Adjusted how path estimates are computed and cached --- include/ZeroTierOne.h | 57 +++++++++++++++++++++++++++++++----- node/Node.cpp | 11 +++++++ node/Path.hpp | 79 ++++++++++++++++++++++++++++++-------------------- node/Peer.cpp | 4 +-- node/Peer.hpp | 14 +++++---- service/OneService.cpp | 51 +++++++++++++++++++++++++++++++- 6 files changed, 169 insertions(+), 47 deletions(-) (limited to 'include/ZeroTierOne.h') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index b3927c2e..a100afd9 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -449,13 +449,6 @@ enum ZT_MultipathMode * Will cease sending traffic over links that appear to be stale. */ ZT_MULTIPATH_PROPORTIONALLY_BALANCED = 2, - - /** - * Traffic is allocated across a user-defined interface/allocation - * - * Will cease sending traffic over links that appear to be stale. - */ - ZT_MULTIPATH_MANUALLY_BALANCED = 3 }; /** @@ -1221,6 +1214,56 @@ typedef struct */ uint64_t trustedPathId; + /** + * One-way latency + */ + float latency; + + /** + * How much latency varies over time + */ + float packetDelayVariance; + + /** + * How much observed throughput varies over time + */ + float throughputDisturbCoeff; + + /** + * Packet Error Ratio (PER) + */ + float packetErrorRatio; + + /** + * Packet Loss Ratio (PLR) + */ + float packetLossRatio; + + /** + * Stability of the path + */ + float stability; + + /** + * Current throughput (moving average) + */ + uint64_t throughput; + + /** + * Maximum observed throughput for this path + */ + uint64_t maxThroughput; + + /** + * Percentage of traffic allocated to this path + */ + float allocation; + + /** + * Name of physical interface (for monitoring) + */ + char *ifname; + /** * Is path expired? */ diff --git a/node/Node.cpp b/node/Node.cpp index 71e5b6a7..24deeae2 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -474,6 +474,17 @@ ZT_PeerList *Node::peers() const p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); p->paths[p->pathCount].expired = 0; p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0; + p->paths[p->pathCount].latency = (*path)->latency(); + p->paths[p->pathCount].packetDelayVariance = (*path)->packetDelayVariance(); + p->paths[p->pathCount].throughputDisturbCoeff = (*path)->throughputDisturbanceCoefficient(); + p->paths[p->pathCount].packetErrorRatio = (*path)->packetErrorRatio(); + p->paths[p->pathCount].packetLossRatio = (*path)->packetLossRatio(); + p->paths[p->pathCount].stability = (*path)->lastComputedStability(); + p->paths[p->pathCount].throughput = (*path)->meanThroughput(); + p->paths[p->pathCount].maxThroughput = (*path)->maxLifetimeThroughput(); + p->paths[p->pathCount].allocation = (*path)->allocation(); + p->paths[p->pathCount].ifname = (*path)->getName(); + ++p->pathCount; } } diff --git a/node/Path.hpp b/node/Path.hpp index 30655877..6162be20 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -111,15 +111,16 @@ public: _expectingAckAsOf(0), _packetsReceivedSinceLastAck(0), _packetsReceivedSinceLastQoS(0), - _meanThroughput(0.0), _maxLifetimeThroughput(0), + _lastComputedMeanThroughput(0), _bytesAckedSinceLastThroughputEstimation(0), - _meanLatency(0.0), - _packetDelayVariance(0.0), - _packetErrorRatio(0.0), - _packetLossRatio(0), + _lastComputedMeanLatency(0.0), + _lastComputedPacketDelayVariance(0.0), + _lastComputedPacketErrorRatio(0.0), + _lastComputedPacketLossRatio(0), _lastComputedStability(0.0), _lastComputedRelativeQuality(0), + _lastComputedThroughputDistCoeff(0.0), _lastAllocation(0.0) { prepareBuffers(); @@ -142,15 +143,16 @@ public: _expectingAckAsOf(0), _packetsReceivedSinceLastAck(0), _packetsReceivedSinceLastQoS(0), - _meanThroughput(0.0), _maxLifetimeThroughput(0), + _lastComputedMeanThroughput(0), _bytesAckedSinceLastThroughputEstimation(0), - _meanLatency(0.0), - _packetDelayVariance(0.0), - _packetErrorRatio(0.0), - _packetLossRatio(0), + _lastComputedMeanLatency(0.0), + _lastComputedPacketDelayVariance(0.0), + _lastComputedPacketErrorRatio(0.0), + _lastComputedPacketLossRatio(0), _lastComputedStability(0.0), _lastComputedRelativeQuality(0), + _lastComputedThroughputDistCoeff(0.0), _lastAllocation(0.0) { prepareBuffers(); @@ -162,9 +164,11 @@ public: delete _throughputSamples; delete _latencySamples; delete _packetValiditySamples; + delete _throughputDisturbanceSamples; _throughputSamples = NULL; _latencySamples = NULL; _packetValiditySamples = NULL; + _throughputDisturbanceSamples = NULL; } /** @@ -311,7 +315,7 @@ public: inline void recordOutgoingPacket(int64_t now, int64_t packetId, uint16_t payloadLength, Packet::Verb verb) { Mutex::Lock _l(_statistics_m); - if (verb == Packet::VERB_FRAME || verb == Packet::VERB_EXT_FRAME) { + if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) { if (packetId % 2 == 0) { // even -> use for ACK _unackedBytes += payloadLength; // Take note that we're expecting a VERB_ACK on this path as of a specific time @@ -336,7 +340,7 @@ public: inline void recordIncomingPacket(int64_t now, int64_t packetId, uint16_t payloadLength, Packet::Verb verb) { Mutex::Lock _l(_statistics_m); - if (verb == Packet::VERB_FRAME || verb == Packet::VERB_EXT_FRAME) { + if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) { if (packetId % 2 == 0) { // even -> use for ACK _inACKRecords[packetId] = payloadLength; _packetsReceivedSinceLastAck++; @@ -497,14 +501,14 @@ public: inline int64_t ackAge(int64_t now) { return _expectingAckAsOf ? now - _expectingAckAsOf : 0; } /** - * The maximum observed throughput for this path + * The maximum observed throughput (in bits/s) for this path */ inline uint64_t maxLifetimeThroughput() { return _maxLifetimeThroughput; } /** * @return The mean throughput (in bits/s) of this link */ - inline float meanThroughput() { return _meanThroughput; } + inline uint64_t meanThroughput() { return _lastComputedMeanThroughput; } /** * Assign a new relative quality value for this path in the aggregate link @@ -543,22 +547,22 @@ public: /** * @return Packet delay variance */ - inline float packetDelayVariance() { return _packetDelayVariance; } + inline float packetDelayVariance() { return _lastComputedPacketDelayVariance; } /** * @return Previously-computed mean latency */ - inline float meanLatency() { return _meanLatency; } + inline float meanLatency() { return _lastComputedMeanLatency; } /** * @return Packet loss rate (PLR) */ - inline float packetLossRatio() { return _packetLossRatio; } + inline float packetLossRatio() { return _lastComputedPacketLossRatio; } /** * @return Packet error ratio (PER) */ - inline float packetErrorRatio() { return _packetErrorRatio; } + inline float packetErrorRatio() { return _lastComputedPacketErrorRatio; } /** * Record an invalid incoming packet. This packet failed MAC/compression/cipher checks and will now @@ -571,38 +575,46 @@ public: */ inline char *getAddressString() { return _addrString; } + /** + * @return The current throughput disturbance coefficient + */ + inline float throughputDisturbanceCoefficient() { return _lastComputedThroughputDistCoeff; } + /** * Compute and cache stability and performance metrics. The resultant stability coefficient is a measure of how "well behaved" * this path is. This figure is substantially different from (but required for the estimation of the path's overall "quality". * * @param now Current time */ - inline void processBackgroundPathMeasurements(int64_t now, const int64_t peerId) { + inline void processBackgroundPathMeasurements(int64_t now) { if (now - _lastPathQualityComputeTime > ZT_PATH_QUALITY_COMPUTE_INTERVAL) { Mutex::Lock _l(_statistics_m); _lastPathQualityComputeTime = now; address().toString(_addrString); - _meanThroughput = _throughputSamples->mean(); - _meanLatency = _latencySamples->mean(); - _packetDelayVariance = _latencySamples->stddev(); // Similar to "jitter" (SEE: RFC 3393, RFC 4689) + _lastComputedMeanLatency = _latencySamples->mean(); + _lastComputedPacketDelayVariance = _latencySamples->stddev(); // Similar to "jitter" (SEE: RFC 3393, RFC 4689) + _lastComputedMeanThroughput = (uint64_t)_throughputSamples->mean(); // If no packet validity samples, assume PER==0 - _packetErrorRatio = 1 - (_packetValiditySamples->count() ? _packetValiditySamples->mean() : 1); + _lastComputedPacketErrorRatio = 1 - (_packetValiditySamples->count() ? _packetValiditySamples->mean() : 1); // Compute path stability // Normalize measurements with wildly different ranges into a reasonable range - float normalized_pdv = Utils::normalize(_packetDelayVariance, 0, ZT_PATH_MAX_PDV, 0, 10); - float normalized_la = Utils::normalize(_meanLatency, 0, ZT_PATH_MAX_MEAN_LATENCY, 0, 10); + float normalized_pdv = Utils::normalize(_lastComputedPacketDelayVariance, 0, ZT_PATH_MAX_PDV, 0, 10); + float normalized_la = Utils::normalize(_lastComputedMeanLatency, 0, ZT_PATH_MAX_MEAN_LATENCY, 0, 10); float throughput_cv = _throughputSamples->mean() > 0 ? _throughputSamples->stddev() / _throughputSamples->mean() : 1; // Form an exponential cutoff and apply contribution weights float pdv_contrib = exp((-1)*normalized_pdv) * ZT_PATH_CONTRIB_PDV; float latency_contrib = exp((-1)*normalized_la) * ZT_PATH_CONTRIB_LATENCY; + // Throughput Disturbance Coefficient float throughput_disturbance_contrib = exp((-1)*throughput_cv) * ZT_PATH_CONTRIB_THROUGHPUT_DISTURBANCE; + _throughputDisturbanceSamples->push(throughput_cv); + _lastComputedThroughputDistCoeff = _throughputDisturbanceSamples->mean(); // Obey user-defined ignored contributions pdv_contrib = ZT_PATH_CONTRIB_PDV > 0.0 ? pdv_contrib : 1; latency_contrib = ZT_PATH_CONTRIB_LATENCY > 0.0 ? latency_contrib : 1; throughput_disturbance_contrib = ZT_PATH_CONTRIB_THROUGHPUT_DISTURBANCE > 0.0 ? throughput_disturbance_contrib : 1; - // Compute the quality product + // Stability _lastComputedStability = pdv_contrib + latency_contrib + throughput_disturbance_contrib; - _lastComputedStability *= 1 - _packetErrorRatio; + _lastComputedStability *= 1 - _lastComputedPacketErrorRatio; // Prevent QoS records from sticking around for too long std::map::iterator it = _outQoSRecords.begin(); while (it != _outQoSRecords.end()) { @@ -646,6 +658,7 @@ public: _throughputSamples = new RingBuffer(ZT_PATH_QUALITY_METRIC_WIN_SZ); _latencySamples = new RingBuffer(ZT_PATH_QUALITY_METRIC_WIN_SZ); _packetValiditySamples = new RingBuffer(ZT_PATH_QUALITY_METRIC_WIN_SZ); + _throughputDisturbanceSamples = new RingBuffer(ZT_PATH_QUALITY_METRIC_WIN_SZ); memset(_ifname, 0, 16); memset(_addrString, 0, sizeof(_addrString)); } @@ -677,19 +690,20 @@ private: int16_t _packetsReceivedSinceLastAck; int16_t _packetsReceivedSinceLastQoS; - float _meanThroughput; uint64_t _maxLifetimeThroughput; + uint64_t _lastComputedMeanThroughput; uint64_t _bytesAckedSinceLastThroughputEstimation; - volatile float _meanLatency; - float _packetDelayVariance; + float _lastComputedMeanLatency; + float _lastComputedPacketDelayVariance; - float _packetErrorRatio; - float _packetLossRatio; + float _lastComputedPacketErrorRatio; + float _lastComputedPacketLossRatio; // cached estimates float _lastComputedStability; float _lastComputedRelativeQuality; + float _lastComputedThroughputDistCoeff; float _lastAllocation; // cached human-readable strings for tracing purposes @@ -699,6 +713,7 @@ private: RingBuffer *_throughputSamples; RingBuffer *_latencySamples; RingBuffer *_packetValiditySamples; + RingBuffer *_throughputDisturbanceSamples; }; } // namespace ZeroTier diff --git a/node/Peer.cpp b/node/Peer.cpp index 55132bba..0f471b07 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -116,7 +116,7 @@ void Peer::received( } for(unsigned int i=0;iprocessBackgroundPathMeasurements(now, _id.address().toInt()); + _paths[i].p->processBackgroundPathMeasurements(now); } } } @@ -415,7 +415,7 @@ SharedPtr Peer::getAppropriatePath(int64_t now, bool includeExpired) for(unsigned int i=0;iprocessBackgroundPathMeasurements(now, _id.address().toInt()); + _paths[i].p->processBackgroundPathMeasurements(now); } } diff --git a/node/Peer.hpp b/node/Peer.hpp index 9361f665..ddbe6f77 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -353,14 +353,18 @@ public: inline int64_t isActive(int64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** - * @return Latency in milliseconds of best path or 0xffff if unknown / no paths + * @return Latency in milliseconds of best/aggregate path or 0xffff if unknown / no paths */ inline unsigned int latency(const int64_t now) { - SharedPtr bp(getAppropriatePath(now,false)); - if (bp) - return bp->latency(); - return 0xffff; + if (RR->node->getMultipathMode()) { + return (int)computeAggregateLinkMeanLatency(); + } else { + SharedPtr bp(getAppropriatePath(now,false)); + if (bp) + return bp->latency(); + return 0xffff; + } } /** diff --git a/service/OneService.cpp b/service/OneService.cpp index e56a4827..a1a9d981 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -298,6 +298,39 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) pj["paths"] = pa; } +static void _peerAggregateLinkToJson(nlohmann::json &pj,const ZT_Peer *peer) +{ + char tmp[256]; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); + pj["aggregateLinkLatency"] = peer->latency; + + nlohmann::json pa = nlohmann::json::array(); + for(unsigned int i=0;ipathCount;++i) { + //int64_t lastSend = peer->paths[i].lastSend; + //int64_t lastReceive = peer->paths[i].lastReceive; + nlohmann::json j; + j["address"] = reinterpret_cast(&(peer->paths[i].address))->toString(tmp); + //j["lastSend"] = (lastSend < 0) ? 0 : lastSend; + //j["lastReceive"] = (lastReceive < 0) ? 0 : lastReceive; + //j["trustedPathId"] = peer->paths[i].trustedPathId; + //j["active"] = (bool)(peer->paths[i].expired == 0); + //j["expired"] = (bool)(peer->paths[i].expired != 0); + //j["preferred"] = (bool)(peer->paths[i].preferred != 0); + j["latency"] = peer->paths[i].latency; + //j["packetDelayVariance"] = peer->paths[i].packetDelayVariance; + //j["throughputDisturbCoeff"] = peer->paths[i].throughputDisturbCoeff; + //j["packetErrorRatio"] = peer->paths[i].packetErrorRatio; + //j["packetLossRatio"] = peer->paths[i].packetLossRatio; + j["stability"] = peer->paths[i].stability; + j["throughput"] = peer->paths[i].throughput; + //j["maxThroughput"] = peer->paths[i].maxThroughput; + j["allocation"] = peer->paths[i].allocation; + j["ifname"] = peer->paths[i].ifname; + pa.push_back(j); + } + pj["paths"] = pa; +} + static void _moonToJson(nlohmann::json &mj,const World &world) { char tmp[4096]; @@ -1189,7 +1222,23 @@ public: json &settings = res["config"]["settings"]; settings["primaryPort"] = OSUtils::jsonInt(settings["primaryPort"],(uint64_t)_primaryPort) & 0xffff; settings["allowTcpFallbackRelay"] = OSUtils::jsonBool(settings["allowTcpFallbackRelay"],_allowTcpFallbackRelay); - settings["multipathMode"] = OSUtils::jsonInt(settings["multipathMode"],_multipathMode); + + if (_multipathMode) { + json &multipathConfig = res["multipath"]; + ZT_PeerList *pl = _node->peers(); + char peerAddrStr[256]; + if (pl) { + for(unsigned long i=0;ipeerCount;++i) { + if (pl->peers[i].role == ZT_PEER_ROLE_LEAF) { + nlohmann::json pj; + _peerAggregateLinkToJson(pj,&(pl->peers[i])); + OSUtils::ztsnprintf(peerAddrStr,sizeof(peerAddrStr),"%.10llx",pl->peers[i].address); + multipathConfig[peerAddrStr] = (pj); + } + } + } + } + #ifdef ZT_USE_MINIUPNPC settings["portMappingEnabled"] = OSUtils::jsonBool(settings["portMappingEnabled"],true); #else -- cgit v1.2.3 From bdcdccfcc3157e62a4bc8078e583876cbef41223 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Fri, 22 Jun 2018 16:30:20 -0700 Subject: Improved path selection, more efficient traffic allocation, lower QoS/ACK overhead --- include/ZeroTierOne.h | 5 ++ node/Constants.hpp | 15 ++++- node/Node.cpp | 6 +- node/Path.hpp | 20 +++---- node/Peer.cpp | 150 ++++++++++++++++++++++++++----------------------- node/Peer.hpp | 36 ++++++++++-- service/OneService.cpp | 2 +- 7 files changed, 143 insertions(+), 91 deletions(-) (limited to 'include/ZeroTierOne.h') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index a100afd9..5b228e17 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1315,6 +1315,11 @@ typedef struct */ unsigned int pathCount; + /** + * Whether this peer was ever reachable via an aggregate link + */ + bool hadAggregateLink; + /** * Known network paths to peer */ diff --git a/node/Constants.hpp b/node/Constants.hpp index 0d3692f1..420343ad 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -274,6 +274,19 @@ */ #define ZT_MULTIPATH_BINDER_REFRESH_PERIOD 5000 +/** + * Packets are only used for QoS/ACK statistical sampling if their packet ID is divisible by + * this integer. This is to provide a mechanism for both peers to agree on which packets need + * special treatment without having to exchange information. Changing this value would be + * a breaking change and would necessitate a protocol version upgrade. Since each incoming and + * outgoing packet ID is checked against this value its evaluation is of the form: + * (id & (divisor - 1)) == 0, thus the divisor must be a power of 2. + * + * This value is set at (16) so that given a normally-distributed RNG output we will sample + * 1/16th (or ~6.25%) of packets. + */ +#define ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR 0x10 + /** * Time horizon for VERB_QOS_MEASUREMENT and VERB_ACK packet processing cutoff */ @@ -384,7 +397,7 @@ /** * Minimum amount of time between each ACK packet */ -#define ZT_PATH_ACK_INTERVAL 250 +#define ZT_PATH_ACK_INTERVAL 1000 /** * How often an aggregate link statistics report is emitted into this tracing system diff --git a/node/Node.cpp b/node/Node.cpp index 24deeae2..9b10dfdd 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -450,6 +450,7 @@ ZT_PeerList *Node::peers() const for(std::vector< std::pair< Address,SharedPtr > >::iterator pi(peers.begin());pi!=peers.end();++pi) { ZT_Peer *p = &(pl->peers[pl->peerCount++]); p->address = pi->second->address().toInt(); + p->hadAggregateLink = 0; if (pi->second->remoteVersionKnown()) { p->versionMajor = pi->second->remoteVersionMajor(); p->versionMinor = pi->second->remoteVersionMinor(); @@ -466,6 +467,7 @@ ZT_PeerList *Node::peers() const std::vector< SharedPtr > paths(pi->second->paths(_now)); SharedPtr bestp(pi->second->getAppropriatePath(_now,false)); + p->hadAggregateLink |= pi->second->hasAggregateLink(); p->pathCount = 0; for(std::vector< SharedPtr >::iterator path(paths.begin());path!=paths.end();++path) { ZT_FAST_MEMCPY(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); @@ -475,14 +477,14 @@ ZT_PeerList *Node::peers() const p->paths[p->pathCount].expired = 0; p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0; p->paths[p->pathCount].latency = (*path)->latency(); - p->paths[p->pathCount].packetDelayVariance = (*path)->packetDelayVariance(); + p->paths[p->pathCount].packetDelayVariance = (*path)->packetDelayVariance(); p->paths[p->pathCount].throughputDisturbCoeff = (*path)->throughputDisturbanceCoefficient(); p->paths[p->pathCount].packetErrorRatio = (*path)->packetErrorRatio(); p->paths[p->pathCount].packetLossRatio = (*path)->packetLossRatio(); p->paths[p->pathCount].stability = (*path)->lastComputedStability(); p->paths[p->pathCount].throughput = (*path)->meanThroughput(); p->paths[p->pathCount].maxThroughput = (*path)->maxLifetimeThroughput(); - p->paths[p->pathCount].allocation = (*path)->allocation(); + p->paths[p->pathCount].allocation = (float)(*path)->allocation() / (float)255; p->paths[p->pathCount].ifname = (*path)->getName(); ++p->pathCount; diff --git a/node/Path.hpp b/node/Path.hpp index fb202306..cafff8cf 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -121,7 +121,7 @@ public: _lastComputedStability(0.0), _lastComputedRelativeQuality(0), _lastComputedThroughputDistCoeff(0.0), - _lastAllocation(0.0) + _lastAllocation(0) { prepareBuffers(); } @@ -153,7 +153,7 @@ public: _lastComputedStability(0.0), _lastComputedRelativeQuality(0), _lastComputedThroughputDistCoeff(0.0), - _lastAllocation(0.0) + _lastAllocation(0) { prepareBuffers(); _phy->getIfName((PhySocket *)((uintptr_t)_localSocket), _ifname, 16); @@ -316,12 +316,10 @@ public: { Mutex::Lock _l(_statistics_m); if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) { - if (packetId % 2 == 0) { // even -> use for ACK + if ((packetId & (ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR - 1)) == 0) { _unackedBytes += payloadLength; // Take note that we're expecting a VERB_ACK on this path as of a specific time _expectingAckAsOf = ackAge(now) > ZT_PATH_ACK_INTERVAL ? _expectingAckAsOf : now; - } - else { // odd -> use for QoS if (_outQoSRecords.size() < ZT_PATH_MAX_OUTSTANDING_QOS_RECORDS) { _outQoSRecords[packetId] = now; } @@ -341,11 +339,9 @@ public: { Mutex::Lock _l(_statistics_m); if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) { - if (packetId % 2 == 0) { // even -> use for ACK + if ((packetId & (ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR - 1)) == 0) { _inACKRecords[packetId] = payloadLength; _packetsReceivedSinceLastAck++; - } - else { // odd -> use for QoS _inQoSRecords[packetId] = now; _packetsReceivedSinceLastQoS++; } @@ -527,12 +523,12 @@ public: * * @param allocation Percentage of traffic to be sent over this path to a peer */ - inline void updateComponentAllocationOfAggregateLink(float allocation) { _lastAllocation = allocation; } + inline void updateComponentAllocationOfAggregateLink(unsigned char allocation) { _lastAllocation = allocation; } /** * @return Percentage of traffic allocated to this path in the aggregate link */ - inline float allocation() { return _lastAllocation; } + inline unsigned char allocation() { return _lastAllocation; } /** * @return Stability estimates can become expensive to compute, we cache the most recent result. @@ -704,7 +700,9 @@ private: float _lastComputedStability; float _lastComputedRelativeQuality; float _lastComputedThroughputDistCoeff; - float _lastAllocation; + unsigned char _lastAllocation; + + // cached human-readable strings for tracing purposes char _ifname[16]; diff --git a/node/Peer.cpp b/node/Peer.cpp index 1d581ab8..21bbfabe 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -56,6 +56,12 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastSentFullHello(0), _lastACKWindowReset(0), _lastQoSWindowReset(0), + _lastMultipathCompatibilityCheck(0), + _freeRandomByte(0), + _uniqueAlivePathCount(0), + _localMultipathSupported(false), + _remoteMultipathSupported(false), + _canUseMultipath(false), _vProto(0), _vMajor(0), _vMinor(0), @@ -69,6 +75,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastAggregateStatsReport(0), _lastAggregateAllocation(0) { + Utils::getSecureRandom(&_freeRandomByte, 1); if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) throw ZT_EXCEPTION_INVALID_ARGUMENT; _pathChoiceHist = new RingBuffer(ZT_MULTIPATH_PROPORTION_WIN_SZ); @@ -110,7 +117,7 @@ void Peer::received( recordIncomingPacket(tPtr, path, packetId, payloadLength, verb, now); - if (canUseMultipath()) { + if (_canUseMultipath) { if (path->needsToSendQoS(now)) { sendQOS_MEASUREMENT(tPtr, path, path->localSocket(), path->address(), now); } @@ -145,17 +152,23 @@ void Peer::received( // Paths are redundant if they duplicate an alive path to the same IP or // with the same local socket and address family. bool redundant = false; + unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS; for(unsigned int i=0;ialive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual2(path->address())) ) ) { + if ( (_paths[i].p->alive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual2(path->address())) ) ) { redundant = true; break; } + // If the path is the same address and port, simply assume this is a replacement + if ( (_paths[i].p->address().ipsEqual2(path->address()) && (_paths[i].p->address().port() == path->address().port()))) { + replacePath = i; + break; + } } else break; } - - if (!redundant) { - unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS; + // If the path isn't a duplicate of the same localSocket AND we haven't already determined a replacePath, + // then find the worst path and replace it. + if (!redundant && replacePath == ZT_MAX_PEER_NETWORK_PATHS) { int replacePathQuality = 0; for(unsigned int i=0;iaddress().ss_family == path->address().ss_family && _paths[i].p->address().ipsEqual2(path->address())) { - replacePath = i; - break; - } - } - } - } - - if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) { - if (verb == Packet::VERB_OK) { - RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId); - _paths[replacePath].lr = now; - _paths[replacePath].p = path; - _paths[replacePath].priority = 1; - } else { - attemptToContact = true; - } + } + if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) { + if (verb == Packet::VERB_OK) { + RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId); + _paths[replacePath].lr = now; + _paths[replacePath].p = path; + _paths[replacePath].priority = 1; + } else { + attemptToContact = true; } } } @@ -274,7 +273,9 @@ void Peer::received( void Peer::recordOutgoingPacket(const SharedPtr &path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, int64_t now) { - if (localMultipathSupport()) { + // Grab second byte from packetId to use as a source of entropy in the next path selection + _freeRandomByte = (packetId & 0xFF00) >> 8; + if (_canUseMultipath) { path->recordOutgoingPacket(now, packetId, payloadLength, verb); } } @@ -282,7 +283,7 @@ void Peer::recordOutgoingPacket(const SharedPtr &path, const uint64_t pack void Peer::recordIncomingPacket(void *tPtr, const SharedPtr &path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, int64_t now) { - if (localMultipathSupport()) { + if (_canUseMultipath) { if (path->needsToSendAck(now)) { sendACK(tPtr, path, path->localSocket(), path->address(), now); } @@ -323,6 +324,9 @@ void Peer::computeAggregateProportionalAllocation(int64_t now) + (fmax(1, relThroughput[i]) * ZT_PATH_CONTRIB_THROUGHPUT) + relScope * ZT_PATH_CONTRIB_SCOPE; relQuality *= age_contrib; + // Arbitrary cutoffs + relQuality = relQuality > (1.00 / 100.0) ? relQuality : 0.0; + relQuality = relQuality < (99.0 / 100.0) ? relQuality : 1.0; totalRelativeQuality += relQuality; _paths[i].p->updateRelativeQuality(relQuality); } @@ -330,12 +334,12 @@ void Peer::computeAggregateProportionalAllocation(int64_t now) // Convert set of relative performances into an allocation set for(uint16_t i=0;iupdateComponentAllocationOfAggregateLink(_paths[i].p->relativeQuality() / totalRelativeQuality); + _paths[i].p->updateComponentAllocationOfAggregateLink((_paths[i].p->relativeQuality() / totalRelativeQuality) * 255); } } } -float Peer::computeAggregateLinkPacketDelayVariance() +int Peer::computeAggregateLinkPacketDelayVariance() { float pdv = 0.0; for(unsigned int i=0;i Peer::getAppropriatePath(int64_t now, bool includeExpired) * Send traffic across the highest quality path only. This algorithm will still * use the old path quality metric from protocol version 9. */ - if (!canUseMultipath()) { + if (!_canUseMultipath) { long bestPathQuality = 2147483647; for(unsigned int i=0;i Peer::getAppropriatePath(int64_t now, bool includeExpired) } } } - unsigned int r; - Utils::getSecureRandom(&r, 1); + unsigned int r = _freeRandomByte; if (numAlivePaths > 0) { - // pick a random out of the set deemed "alive" int rf = r % numAlivePaths; return _paths[alivePaths[rf]].p; } else if(numStalePaths > 0) { - // resort to trying any non-expired path + // Resort to trying any non-expired path int rf = r % numStalePaths; return _paths[stalePaths[rf]].p; } @@ -461,40 +463,12 @@ SharedPtr Peer::getAppropriatePath(int64_t now, bool includeExpired) * Proportionally allocate traffic according to dynamic path quality measurements */ if (RR->node->getMultipathMode() == ZT_MULTIPATH_PROPORTIONALLY_BALANCED) { - int numAlivePaths = 0; - int numStalePaths = 0; - int alivePaths[ZT_MAX_PEER_NETWORK_PATHS]; - int stalePaths[ZT_MAX_PEER_NETWORK_PATHS]; - memset(&alivePaths, -1, sizeof(alivePaths)); - memset(&stalePaths, -1, sizeof(stalePaths)); - // Attempt to find an excuse not to use the rest of this algorithm - // Alive or Stale? - for(unsigned int i=0;ialive(now)) { - alivePaths[numAlivePaths] = i; - numAlivePaths++; - } else { - stalePaths[numStalePaths] = i; - numStalePaths++; - } - // Record a default path to use as a short-circuit for the rest of the algorithm (if needed) - bestPath = i; - } - } if ((now - _lastAggregateAllocation) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) { _lastAggregateAllocation = now; computeAggregateProportionalAllocation(now); } - if (numAlivePaths == 0 && numStalePaths == 0) { - return SharedPtr(); - } if (numAlivePaths == 1 || numStalePaths == 1) { - return _paths[bestPath].p; - } // Randomly choose path according to their allocations - unsigned int r; - Utils::getSecureRandom(&r, 1); - float rf = (float)(r %= 100) / 100; + float rf = _freeRandomByte; for(int i=0;iallocation()) { @@ -676,6 +650,41 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &o } } +inline void Peer::processBackgroundPeerTasks(int64_t now) +{ + // Determine current multipath compatibility with other peer + if ((now - _lastMultipathCompatibilityCheck) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) { + // Cache number of available paths so that we can short-circuit multipath logic elsewhere + // + // We also take notice of duplicate paths (same IP only) because we may have + // recently received a direct path push from a peer and our list might contain + // a dead path which hasn't been fully recognized as such. In this case we + // don't want the duplicate to trigger execution of multipath code prematurely. + // + // This is done to support the behavior of auto multipath enable/disable + // without user intervention. + int currAlivePathCount = 0; + int duplicatePathsFound = 0; + for (unsigned int i=0;iaddress().ipsEqual2(_paths[j].p->address()) && i != j) { + duplicatePathsFound+=1; + break; + } + } + } + } + _uniqueAlivePathCount = (currAlivePathCount - (duplicatePathsFound / 2)); + _lastMultipathCompatibilityCheck = now; + _localMultipathSupported = ((RR->node->getMultipathMode() != ZT_MULTIPATH_NONE) && (ZT_PROTO_VERSION > 9)); + _remoteMultipathSupported = _vProto > 9; + // If both peers support multipath and more than one path exist, we can use multipath logic + _canUseMultipath = _localMultipathSupported && _remoteMultipathSupported && (_uniqueAlivePathCount > 1); + } +} + void Peer::sendACK(void *tPtr,const SharedPtr &path,const int64_t localSocket,const InetAddress &atAddress,int64_t now) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ACK); @@ -774,14 +783,15 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now) unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) { unsigned int sent = 0; - Mutex::Lock _l(_paths_m); const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD); _lastSentFullHello = now; + processBackgroundPeerTasks(now); + // Emit traces regarding aggregate link status - if (canUseMultipath()) { + if (_canUseMultipath) { int alivePathCount = aggregateLinkPhysicalPathCount(); if ((now - _lastAggregateStatsReport) > ZT_PATH_AGGREGATE_STATS_REPORT_INTERVAL) { _lastAggregateStatsReport = now; diff --git a/node/Peer.hpp b/node/Peer.hpp index ddbe6f77..a32eaad0 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -203,12 +203,12 @@ public: /** * @return The aggregate link Packet Delay Variance (PDV) */ - float computeAggregateLinkPacketDelayVariance(); + int computeAggregateLinkPacketDelayVariance(); /** * @return The aggregate link mean latency */ - float computeAggregateLinkMeanLatency(); + int computeAggregateLinkMeanLatency(); /** * @return The number of currently alive "physical" paths in the aggregate link @@ -357,7 +357,7 @@ public: */ inline unsigned int latency(const int64_t now) { - if (RR->node->getMultipathMode()) { + if (_canUseMultipath) { return (int)computeAggregateLinkMeanLatency(); } else { SharedPtr bp(getAppropriatePath(now,false)); @@ -417,6 +417,14 @@ public: inline bool remoteVersionKnown() const { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); } + /** + * Periodically update known multipath activation constraints. This is done so that we know when and when + * not to use multipath logic. Doing this once every few seconds is sufficient. + * + * @param now Current time + */ + inline void processBackgroundPeerTasks(int64_t now); + /** * Record that the remote peer does have multipath enabled. As is evident by the receipt of a VERB_ACK * or a VERB_QOS_MEASUREMENT packet at some point in the past. Until this flag is set, the local client @@ -427,18 +435,18 @@ public: /** * @return Whether the local client supports and is configured to use multipath */ - inline bool localMultipathSupport() { return ((RR->node->getMultipathMode() != ZT_MULTIPATH_NONE) && (ZT_PROTO_VERSION > 9)); } + inline bool localMultipathSupport() { return _localMultipathSupported; } /** * @return Whether the remote peer supports and is configured to use multipath */ - inline bool remoteMultipathSupport() { return (_remotePeerMultipathEnabled && (_vProto > 9)); } + inline bool remoteMultipathSupport() { return _remoteMultipathSupported; } /** * @return Whether this client can use multipath to communicate with this peer. True if both peers are using * the correct protocol and if both peers have multipath enabled. False if otherwise. */ - inline bool canUseMultipath() { return (localMultipathSupport() && remoteMultipathSupport()); } + inline bool canUseMultipath() { return _canUseMultipath; } /** * @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms @@ -557,6 +565,13 @@ public: return (_QoSCutoffCount < ZT_PATH_QOS_ACK_CUTOFF_LIMIT); } + /** + * @return Whether this peer is reachable via an aggregate link + */ + inline bool hasAggregateLink() { + return _localMultipathSupported && _remoteMultipathSupported && _remotePeerMultipathEnabled; + } + /** * Serialize a peer for storage in local cache * @@ -658,6 +673,15 @@ private: int64_t _lastPathPrune; int64_t _lastACKWindowReset; int64_t _lastQoSWindowReset; + int64_t _lastMultipathCompatibilityCheck; + + unsigned char _freeRandomByte; + + int _uniqueAlivePathCount; + + bool _localMultipathSupported; + bool _remoteMultipathSupported; + bool _canUseMultipath; uint16_t _vProto; uint16_t _vMajor; diff --git a/service/OneService.cpp b/service/OneService.cpp index a1a9d981..9b12f17b 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1229,7 +1229,7 @@ public: char peerAddrStr[256]; if (pl) { for(unsigned long i=0;ipeerCount;++i) { - if (pl->peers[i].role == ZT_PEER_ROLE_LEAF) { + if (pl->peers[i].hadAggregateLink) { nlohmann::json pj; _peerAggregateLinkToJson(pj,&(pl->peers[i])); OSUtils::ztsnprintf(peerAddrStr,sizeof(peerAddrStr),"%.10llx",pl->peers[i].address); -- cgit v1.2.3 From 65b0030342704cdbace07693a22d3e8048f7f244 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Wed, 11 Jul 2018 16:55:13 -0700 Subject: Added basic QoS rule handling --- include/ZeroTierOne.h | 10 ++++++++++ node/Network.cpp | 8 +++++++- rule-compiler/rule-compiler.js | 5 ++++- 3 files changed, 21 insertions(+), 2 deletions(-) (limited to 'include/ZeroTierOne.h') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 5b228e17..e4157b96 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -744,6 +744,11 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_ACTION_BREAK = 5, + /** + * Place a matching frame in the specified QoS bucket + */ + ZT_NETWORK_RULE_ACTION_PRIORITY = 6, + /** * Maximum ID for an ACTION, anything higher is a MATCH */ @@ -934,6 +939,11 @@ typedef struct uint32_t flags; uint16_t length; } fwd; + + /** + * Quality of Service (QoS) bucket we want a frame to be placed in + */ + uint8_t qosBucket; } v; } ZT_VirtualNetworkRule; diff --git a/node/Network.cpp b/node/Network.cpp index a5c2fc3e..69a74281 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -29,6 +29,8 @@ #include #include +#include "../include/ZeroTierDebug.h" + #include "Constants.hpp" #include "../version.h" #include "Network.hpp" @@ -107,7 +109,7 @@ static _doZtFilterResult _doZtFilter( Address &cc, // MUTABLE -- set to TEE destination if TEE action is taken or left alone otherwise unsigned int &ccLength, // MUTABLE -- set to length of packet payload to TEE bool &ccWatch, // MUTABLE -- set to true for WATCH target as opposed to normal TEE - uint8_t &qosBucket) // MUTABLE -- set to the value of the argument provided to the matching action + uint8_t &qosBucket) // MUTABLE -- set to the value of the argument provided to PRIORITY { // Set to true if we are a TEE/REDIRECT/WATCH target bool superAccept = false; @@ -125,6 +127,10 @@ static _doZtFilterResult _doZtFilter( if ((unsigned int)rt <= (unsigned int)ZT_NETWORK_RULE_ACTION__MAX_ID) { if (thisSetMatches) { switch(rt) { + case ZT_NETWORK_RULE_ACTION_PRIORITY: + qosBucket = (rules[rn].v.qosBucket >= 0 || rules[rn].v.qosBucket <= 8) ? rules[rn].v.qosBucket : 4; // 4 = default bucket (no priority) + return DOZTFILTER_ACCEPT; + case ZT_NETWORK_RULE_ACTION_DROP: return DOZTFILTER_DROP; diff --git a/rule-compiler/rule-compiler.js b/rule-compiler/rule-compiler.js index bd84824e..38134b74 100644 --- a/rule-compiler/rule-compiler.js +++ b/rule-compiler/rule-compiler.js @@ -65,7 +65,8 @@ const OPEN_BLOCK_KEYWORDS = { 'tee': true, 'watch': true, 'redirect': true, - 'break': true + 'break': true, + 'priority': true }; // Reserved words that can't be used as tag, capability, or rule set names @@ -81,6 +82,7 @@ const RESERVED_WORDS = { 'watch': true, 'redirect': true, 'break': true, + 'priority': true, 'ztsrc': true, 'ztdest': true, @@ -131,6 +133,7 @@ const KEYWORD_TO_API_MAP = { 'watch': 'ACTION_WATCH', 'redirect': 'ACTION_REDIRECT', 'break': 'ACTION_BREAK', + 'priority': 'ACTION_PRIORITY', 'ztsrc': 'MATCH_SOURCE_ZEROTIER_ADDRESS', 'ztdest': 'MATCH_DEST_ZEROTIER_ADDRESS', -- cgit v1.2.3