diff options
313 files changed, 33040 insertions, 33855 deletions
@@ -4,6 +4,7 @@ /zerotier-cli /zerotier-selftest /zerotier +/nltest # OS-created garbage files from various platforms .DS_Store @@ -50,6 +51,7 @@ enc_temp_folder /world/mkworld /world/*.c25519 zt1-src.tar.gz +/MacEthernetTapAgent # Miscellaneous temporaries, build files, etc. *.log @@ -116,3 +118,5 @@ ext/librethinkdbxx/build .vscode __pycache__ *~ +attic/world/*.c25519 +attic/world/mkworld @@ -2,6 +2,7 @@ * ZeroTier Core and ZeroTier One virtual networking service<br> Adam Ierymenko / adam.ierymenko@zerotier.com + Joseph Henry / joseph.henry@zerotier.com (QoS and multipath) * Java JNI Interface to enable Android application development, and Android app itself (code for that is elsewhere)<br> Grant Limberg / glimberg@gmail.com @@ -45,13 +46,6 @@ ZeroTier includes the following third party code, either in ext/ or incorporated * Home page: https://github.com/nlohmann/json * License grant: MIT - * TunTapOSX by Mattias Nissler - - * Files: ext/tap-mac/tuntap/* - * Home page: http://tuntaposx.sourceforge.net/ - * License grant: BSD attribution no-endorsement - * ZeroTier Modifications: change interface name to zt#, increase max MTU, increase max devices - * tap-windows6 by the OpenVPN project * Files: windows/TapDriver6/* diff --git a/Jenkinsfile b/Jenkinsfile index e729c334..88989327 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,61 +24,61 @@ parallel 'centos7': { throw err } } -}, 'android-ndk': { - node('android-ndk') { - try { - checkout scm +// }, 'android-ndk': { +// node('android-ndk') { +// try { +// checkout scm - stage('Build Android NDK') { - sh "/android/android-ndk-r15b/ndk-build -C $WORKSPACE/java ZT1=${WORKSPACE}" - } - } - catch (err) { - currentBuild.result = "FAILURE" - mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Android NDK (<${env.BUILD_URL}|Open>)" +// stage('Build Android NDK') { +// sh "/android/android-ndk-r15b/ndk-build -C $WORKSPACE/java ZT1=${WORKSPACE}" +// } +// } +// catch (err) { +// currentBuild.result = "FAILURE" +// mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Android NDK (<${env.BUILD_URL}|Open>)" - throw err - } - } -}, 'macOS': { - node('macOS') { - try { - checkout scm +// throw err +// } +// } +// }, 'macOS': { +// node('macOS') { +// try { +// checkout scm - stage('Build macOS') { - sh 'make -f make-mac.mk' - } +// stage('Build macOS') { +// sh 'make -f make-mac.mk' +// } - stage('Build macOS UI') { - sh 'cd macui && xcodebuild -target "ZeroTier One" -configuration Debug' - } - } - catch (err) { - currentBuild.result = "FAILURE" - mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on macOS (<${env.BUILD_URL}|Open>)" +// stage('Build macOS UI') { +// sh 'cd macui && xcodebuild -target "ZeroTier One" -configuration Debug' +// } +// } +// catch (err) { +// currentBuild.result = "FAILURE" +// mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on macOS (<${env.BUILD_URL}|Open>)" - throw err - } - } -}, 'windows': { - node('windows') { - try { - checkout scm +// throw err +// } +// } +// }, 'windows': { +// node('windows') { +// try { +// checkout scm - stage('Build Windows') { - bat '''CALL "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64 -git clean -dfx -msbuild windows\\ZeroTierOne.sln -''' - } - } - catch (err) { - currentBuild.result = "FAILURE" - mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Windows (<${env.BUILD_URL}|Open>)" +// stage('Build Windows') { +// bat '''CALL "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64 +// git clean -dfx +// msbuild windows\\ZeroTierOne.sln +// ''' +// } +// } +// catch (err) { +// currentBuild.result = "FAILURE" +// mattermostSend color: '#ff0000', message: "${env.JOB_NAME} broken on Windows (<${env.BUILD_URL}|Open>)" - throw err - } - } +// throw err +// } +// } } mattermostSend color: "#00ff00", message: "${env.JOB_NAME} #${env.BUILD_NUMBER} Complete (<${env.BUILD_URL}|Show More...>)" diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 2464f8f7..67bd2e58 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,18 @@ ZeroTier Release Notes ====== +# 2019-??-?? -- Version 1.4.0 (currently in prerelease as 1.2.99) + + * Added support for concurrent multipath (multiple paths at once) with traffic weighting by link quality and faster recovery from lost links. + * Added under-the-hood support for QoS (not yet exposed) that will eventually be configurable via our rules engine. + * Modified credential push and direct path push timings and algorithms to somewhat reduce "chattiness" of the protocol when idle. More radical background overhead reductions will have to wait for the 2.x line. + * Mac version no longer requires a kernel extension, instead making use of the [feth interfaces](https://apple.stackexchange.com/questions/337715/fake-ethernet-interfaces-feth-if-fake-anyone-ever-seen-this). + * Removed our beta/half-baked integration of Central with the Windows UI. We're going to do a whole new UI of some kind in the future at least for Windows and Mac. + * Fixed stack overflow issues on Linux versions using musl libc. + * Fixed some alignment problems reported on ARM and ARM64, but some reports we could not reproduce so please report any issues with exact chip, OS/distro, and ZeroTier version in use. + * Fixed numerous other small issues and bugs. + * Windows now sets the adapter name such that it is consistent in both the Windows UI and command line utilities. + # 2018-07-27 -- Version 1.2.12 * Fixed a bug that caused exits to take a long time on Mac due to huge numbers of redundant attempts to delete managed routes. @@ -43,7 +55,7 @@ ZeroTier Release Notes * Fixed two very rare multithreading issues that were only observed on certain systems * Platform-Specific Changes * MacOS - * Installer now loads the kernel extension right away so that High Sierra users will see the prompt to authorize it. This is done in the "Security & Privacy" preference pane and must be done driectly on the console (not via remote desktop). On High Sierra and newer kexts must be authorized at the console via security settings system preferences pane. + * Installer now loads the kernel extension right away so that High Sierra users will see the prompt to authorize it. This is done in the "Security & Privacy" preference pane and must be done directly on the console (not via remote desktop). On High Sierra and newer kexts must be authorized at the console via security settings system preferences pane. * Windows * The Windows installer should now install the driver without requiring a special prompt in most cases. This should make it easier for our packages to be accepted into and updated in the Chocolatey repository and should make it easier to perform remote installs across groups of machines using IT management and provisioning tools. * The Windows official packages are now signed with an EV certificate (with hardware key). @@ -85,7 +97,7 @@ The largest new feature in 1.2.0, and the product of many months of work, is our Rules allow you to filter packets on your network and vector traffic to security observers. Security observation can be performed in-band using REDIRECT or out of band using TEE. -Tags and capabilites provide advanced methods for implementing fine grained permission structures and micro-segmentation schemes without bloating the size and complexity of your rules table. +Tags and capabilities provide advanced methods for implementing fine grained permission structures and micro-segmentation schemes without bloating the size and complexity of your rules table. See the [rules engine announcement blog post](https://www.zerotier.com/blog/?p=927) for an in-depth discussion of theory and implementation. The [manual](https://www.zerotier.com/manual.shtml) contains detailed information on rule, tag, and capability use, and the `rule-compiler/` subfolder of the ZeroTier source tree contains a JavaScript function to compile rules in our human-readable rule definition language into rules suitable for import into a network controller. (ZeroTier Central uses this same script to compile rules on [my.zerotier.com](https://my.zerotier.com/).) @@ -168,7 +180,7 @@ A special kind of public network called an ad-hoc network may be accessed by joi | Start of port range (hex) Reserved ZeroTier address prefix indicating a controller-less network -Ad-hoc networks are public (no access control) networks that have no network controller. Instead their configuration and other credentials are generated locally. Ad-hoc networks permit only IPv6 UDP and TCP unicast traffic (no multicast or broadcast) using 6plane format NDP-emulated IPv6 addresses. In addition an ad-hoc network ID encodes an IP port range. UDP packets and TCP SYN (connection open) packets are only allowed to desintation ports within the encoded range. +Ad-hoc networks are public (no access control) networks that have no network controller. Instead their configuration and other credentials are generated locally. Ad-hoc networks permit only IPv6 UDP and TCP unicast traffic (no multicast or broadcast) using 6plane format NDP-emulated IPv6 addresses. In addition an ad-hoc network ID encodes an IP port range. UDP packets and TCP SYN (connection open) packets are only allowed to destination ports within the encoded range. For example `ff00160016000000` is an ad-hoc network allowing only SSH, while `ff0000ffff000000` is an ad-hoc network allowing any UDP or TCP port. @@ -183,7 +195,7 @@ If you have data in an old SQLite3 controller we've included a NodeJS script in ## Major Bug Fixes in 1.2.0 * **The Windows HyperV 100% CPU bug is FINALLY DEAD**: This long-running problem turns out to have been an issue with Windows itself, but one we were triggering by placing invalid data into the Windows registry. Microsoft is aware of the issue but we've also fixed the triggering problem on our side. ZeroTier should now co-exist quite well with HyperV and should now be able to be bridged with a HyperV virtual switch. - * **Segmenation faults on musl-libc based Linux systems**: Alpine Linux and some embedded Linux systems that use musl libc (a minimal libc) experienced segmentation faults. These were due to a smaller default stack size. A work-around that sets the stack size for new threads has been added. + * **Segmentation faults on musl-libc based Linux systems**: Alpine Linux and some embedded Linux systems that use musl libc (a minimal libc) experienced segmentation faults. These were due to a smaller default stack size. A work-around that sets the stack size for new threads has been added. * **Windows firewall blocks local JSON API**: On some Windows systems the firewall likes to block 127.0.0.1:9993 for mysterious reasons. This is now fixed in the installer via the addition of another firewall exemption rule. * **UI crash on embedded Windows due to missing fonts**: The MSI installer now ships fonts and will install them if they are not present, so this should be fixed. diff --git a/artwork/AppIcon_90x90.png b/artwork/AppIcon_90x90.png Binary files differnew file mode 100644 index 00000000..0a618bbb --- /dev/null +++ b/artwork/AppIcon_90x90.png diff --git a/attic/Cluster.cpp b/attic/Cluster.cpp deleted file mode 100644 index 119aec29..00000000 --- a/attic/Cluster.cpp +++ /dev/null @@ -1,1042 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifdef ZT_ENABLE_CLUSTER - -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <math.h> - -#include <map> -#include <algorithm> -#include <set> -#include <utility> -#include <list> -#include <stdexcept> - -#include "../version.h" - -#include "Cluster.hpp" -#include "RuntimeEnvironment.hpp" -#include "MulticastGroup.hpp" -#include "CertificateOfMembership.hpp" -#include "Salsa20.hpp" -#include "Poly1305.hpp" -#include "Identity.hpp" -#include "Topology.hpp" -#include "Packet.hpp" -#include "Switch.hpp" -#include "Node.hpp" -#include "Network.hpp" -#include "Array.hpp" - -namespace ZeroTier { - -static inline double _dist3d(int x1,int y1,int z1,int x2,int y2,int z2) - throw() -{ - double dx = ((double)x2 - (double)x1); - double dy = ((double)y2 - (double)y1); - double dz = ((double)z2 - (double)z1); - return sqrt((dx * dx) + (dy * dy) + (dz * dz)); -} - -// An entry in _ClusterSendQueue -struct _ClusterSendQueueEntry -{ - uint64_t timestamp; - Address fromPeerAddress; - Address toPeerAddress; - // if we ever support larger transport MTUs this must be increased - unsigned char data[ZT_CLUSTER_SEND_QUEUE_DATA_MAX]; - unsigned int len; - bool unite; -}; - -// A multi-index map with entry memory pooling -- this allows our queue to -// be O(log(N)) and is complex enough that it makes the code a lot cleaner -// to break it out from Cluster. -class _ClusterSendQueue -{ -public: - _ClusterSendQueue() : - _poolCount(0) {} - ~_ClusterSendQueue() {} // memory is automatically freed when _chunks is destroyed - - inline void enqueue(uint64_t now,const Address &from,const Address &to,const void *data,unsigned int len,bool unite) - { - if (len > ZT_CLUSTER_SEND_QUEUE_DATA_MAX) - return; - - Mutex::Lock _l(_lock); - - // Delete oldest queue entry for this sender if this enqueue() would take them over the per-sender limit - { - std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator qi(_bySrc.lower_bound(std::pair<Address,_ClusterSendQueueEntry *>(from,(_ClusterSendQueueEntry *)0))); - std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator oldest(qi); - unsigned long countForSender = 0; - while ((qi != _bySrc.end())&&(qi->first == from)) { - if (qi->second->timestamp < oldest->second->timestamp) - oldest = qi; - ++countForSender; - ++qi; - } - if (countForSender >= ZT_CLUSTER_MAX_QUEUE_PER_SENDER) { - _byDest.erase(std::pair<Address,_ClusterSendQueueEntry *>(oldest->second->toPeerAddress,oldest->second)); - _pool[_poolCount++] = oldest->second; - _bySrc.erase(oldest); - } - } - - _ClusterSendQueueEntry *e; - if (_poolCount > 0) { - e = _pool[--_poolCount]; - } else { - if (_chunks.size() >= ZT_CLUSTER_MAX_QUEUE_CHUNKS) - return; // queue is totally full! - _chunks.push_back(Array<_ClusterSendQueueEntry,ZT_CLUSTER_QUEUE_CHUNK_SIZE>()); - e = &(_chunks.back().data[0]); - for(unsigned int i=1;i<ZT_CLUSTER_QUEUE_CHUNK_SIZE;++i) - _pool[_poolCount++] = &(_chunks.back().data[i]); - } - - e->timestamp = now; - e->fromPeerAddress = from; - e->toPeerAddress = to; - memcpy(e->data,data,len); - e->len = len; - e->unite = unite; - - _bySrc.insert(std::pair<Address,_ClusterSendQueueEntry *>(from,e)); - _byDest.insert(std::pair<Address,_ClusterSendQueueEntry *>(to,e)); - } - - inline void expire(uint64_t now) - { - Mutex::Lock _l(_lock); - for(std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator qi(_bySrc.begin());qi!=_bySrc.end();) { - if ((now - qi->second->timestamp) > ZT_CLUSTER_QUEUE_EXPIRATION) { - _byDest.erase(std::pair<Address,_ClusterSendQueueEntry *>(qi->second->toPeerAddress,qi->second)); - _pool[_poolCount++] = qi->second; - _bySrc.erase(qi++); - } else ++qi; - } - } - - /** - * Get and dequeue entries for a given destination address - * - * After use these entries must be returned with returnToPool()! - * - * @param dest Destination address - * @param results Array to fill with results - * @param maxResults Size of results[] in pointers - * @return Number of actual results returned - */ - inline unsigned int getByDest(const Address &dest,_ClusterSendQueueEntry **results,unsigned int maxResults) - { - unsigned int count = 0; - Mutex::Lock _l(_lock); - std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator qi(_byDest.lower_bound(std::pair<Address,_ClusterSendQueueEntry *>(dest,(_ClusterSendQueueEntry *)0))); - while ((qi != _byDest.end())&&(qi->first == dest)) { - _bySrc.erase(std::pair<Address,_ClusterSendQueueEntry *>(qi->second->fromPeerAddress,qi->second)); - results[count++] = qi->second; - if (count == maxResults) - break; - _byDest.erase(qi++); - } - return count; - } - - /** - * Return entries to pool after use - * - * @param entries Array of entries - * @param count Number of entries - */ - inline void returnToPool(_ClusterSendQueueEntry **entries,unsigned int count) - { - Mutex::Lock _l(_lock); - for(unsigned int i=0;i<count;++i) - _pool[_poolCount++] = entries[i]; - } - -private: - std::list< Array<_ClusterSendQueueEntry,ZT_CLUSTER_QUEUE_CHUNK_SIZE> > _chunks; - _ClusterSendQueueEntry *_pool[ZT_CLUSTER_QUEUE_CHUNK_SIZE * ZT_CLUSTER_MAX_QUEUE_CHUNKS]; - unsigned long _poolCount; - std::set< std::pair<Address,_ClusterSendQueueEntry *> > _bySrc; - std::set< std::pair<Address,_ClusterSendQueueEntry *> > _byDest; - Mutex _lock; -}; - -Cluster::Cluster( - const RuntimeEnvironment *renv, - uint16_t id, - const std::vector<InetAddress> &zeroTierPhysicalEndpoints, - int32_t x, - int32_t y, - int32_t z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg) : - RR(renv), - _sendQueue(new _ClusterSendQueue()), - _sendFunction(sendFunction), - _sendFunctionArg(sendFunctionArg), - _addressToLocationFunction(addressToLocationFunction), - _addressToLocationFunctionArg(addressToLocationFunctionArg), - _x(x), - _y(y), - _z(z), - _id(id), - _zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints), - _members(new _Member[ZT_CLUSTER_MAX_MEMBERS]), - _lastFlushed(0), - _lastCleanedRemotePeers(0), - _lastCleanedQueue(0) -{ - uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; - - // Generate master secret by hashing the secret from our Identity key pair - RR->identity.sha512PrivateKey(_masterSecret); - - // Generate our inbound message key, which is the master secret XORed with our ID and hashed twice - memcpy(stmp,_masterSecret,sizeof(stmp)); - stmp[0] ^= Utils::hton(id); - SHA512::hash(stmp,stmp,sizeof(stmp)); - SHA512::hash(stmp,stmp,sizeof(stmp)); - memcpy(_key,stmp,sizeof(_key)); - Utils::burn(stmp,sizeof(stmp)); -} - -Cluster::~Cluster() -{ - Utils::burn(_masterSecret,sizeof(_masterSecret)); - Utils::burn(_key,sizeof(_key)); - delete [] _members; - delete _sendQueue; -} - -void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) -{ - Buffer<ZT_CLUSTER_MAX_MESSAGE_LENGTH> dmsg; - { - // FORMAT: <[16] iv><[8] MAC><... data> - if ((len < 24)||(len > ZT_CLUSTER_MAX_MESSAGE_LENGTH)) - return; - - // 16-byte IV: first 8 bytes XORed with key, last 8 bytes used as Salsa20 64-bit IV - char keytmp[32]; - memcpy(keytmp,_key,32); - for(int i=0;i<8;++i) - keytmp[i] ^= reinterpret_cast<const char *>(msg)[i]; - Salsa20 s20(keytmp,reinterpret_cast<const char *>(msg) + 8); - Utils::burn(keytmp,sizeof(keytmp)); - - // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") - char polykey[ZT_POLY1305_KEY_LEN]; - memset(polykey,0,sizeof(polykey)); - s20.crypt12(polykey,polykey,sizeof(polykey)); - - // Compute 16-byte MAC - char mac[ZT_POLY1305_MAC_LEN]; - Poly1305::compute(mac,reinterpret_cast<const char *>(msg) + 24,len - 24,polykey); - - // Check first 8 bytes of MAC against 64-bit MAC in stream - if (!Utils::secureEq(mac,reinterpret_cast<const char *>(msg) + 16,8)) - return; - - // Decrypt! - dmsg.setSize(len - 24); - s20.crypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size()); - } - - if (dmsg.size() < 4) - return; - const uint16_t fromMemberId = dmsg.at<uint16_t>(0); - unsigned int ptr = 2; - if (fromMemberId == _id) // sanity check: we don't talk to ourselves - return; - const uint16_t toMemberId = dmsg.at<uint16_t>(ptr); - ptr += 2; - if (toMemberId != _id) // sanity check: message not for us? - return; - - { // make sure sender is actually considered a member - Mutex::Lock _l3(_memberIds_m); - if (std::find(_memberIds.begin(),_memberIds.end(),fromMemberId) == _memberIds.end()) - return; - } - - try { - while (ptr < dmsg.size()) { - const unsigned int mlen = dmsg.at<uint16_t>(ptr); ptr += 2; - const unsigned int nextPtr = ptr + mlen; - if (nextPtr > dmsg.size()) - break; - - int mtype = -1; - try { - switch((StateMessageType)(mtype = (int)dmsg[ptr++])) { - default: - break; - - case CLUSTER_MESSAGE_ALIVE: { - _Member &m = _members[fromMemberId]; - Mutex::Lock mlck(m.lock); - ptr += 7; // skip version stuff, not used yet - m.x = dmsg.at<int32_t>(ptr); ptr += 4; - m.y = dmsg.at<int32_t>(ptr); ptr += 4; - m.z = dmsg.at<int32_t>(ptr); ptr += 4; - ptr += 8; // skip local clock, not used - m.load = dmsg.at<uint64_t>(ptr); ptr += 8; - m.peers = dmsg.at<uint64_t>(ptr); ptr += 8; - ptr += 8; // skip flags, unused -#ifdef ZT_TRACE - std::string addrs; -#endif - unsigned int physicalAddressCount = dmsg[ptr++]; - m.zeroTierPhysicalEndpoints.clear(); - for(unsigned int i=0;i<physicalAddressCount;++i) { - m.zeroTierPhysicalEndpoints.push_back(InetAddress()); - ptr += m.zeroTierPhysicalEndpoints.back().deserialize(dmsg,ptr); - if (!(m.zeroTierPhysicalEndpoints.back())) { - m.zeroTierPhysicalEndpoints.pop_back(); - } -#ifdef ZT_TRACE - else { - if (addrs.length() > 0) - addrs.push_back(','); - addrs.append(m.zeroTierPhysicalEndpoints.back().toString()); - } -#endif - } -#ifdef ZT_TRACE - if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) { - TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str()); - } -#endif - m.lastReceivedAliveAnnouncement = RR->node->now(); - } break; - - case CLUSTER_MESSAGE_HAVE_PEER: { - Identity id; - ptr += id.deserialize(dmsg,ptr); - if (id) { - { - Mutex::Lock _l(_remotePeers_m); - _RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(id.address(),(unsigned int)fromMemberId)]; - if (!rp.lastHavePeerReceived) { - RR->topology->saveIdentity((void *)0,id); - RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH); - } - rp.lastHavePeerReceived = RR->node->now(); - } - - _ClusterSendQueueEntry *q[16384]; // 16384 is "tons" - unsigned int qc = _sendQueue->getByDest(id.address(),q,16384); - for(unsigned int i=0;i<qc;++i) - this->relayViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite); - _sendQueue->returnToPool(q,qc); - - TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc); - } - } break; - - case CLUSTER_MESSAGE_WANT_PEER: { - const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - SharedPtr<Peer> peer(RR->topology->getPeerNoCache(zeroTierAddress)); - if ( (peer) && (peer->hasLocalClusterOptimalPath(RR->node->now())) ) { - Buffer<1024> buf; - peer->identity().serialize(buf); - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); - } - } break; - - case CLUSTER_MESSAGE_REMOTE_PACKET: { - const unsigned int plen = dmsg.at<uint16_t>(ptr); ptr += 2; - if (plen) { - Packet remotep(dmsg.field(ptr,plen),plen); ptr += plen; - //TRACE("remote %s from %s via %u (%u bytes)",Packet::verbString(remotep.verb()),remotep.source().toString().c_str(),fromMemberId,plen); - switch(remotep.verb()) { - case Packet::VERB_WHOIS: _doREMOTE_WHOIS(fromMemberId,remotep); break; - case Packet::VERB_MULTICAST_GATHER: _doREMOTE_MULTICAST_GATHER(fromMemberId,remotep); break; - default: break; // ignore things we don't care about across cluster - } - } - } break; - - case CLUSTER_MESSAGE_PROXY_UNITE: { - const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const unsigned int numRemotePeerPaths = dmsg[ptr++]; - InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max - for(unsigned int i=0;i<numRemotePeerPaths;++i) - ptr += remotePeerPaths[i].deserialize(dmsg,ptr); - - TRACE("[%u] requested that we unite local %s with remote %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str()); - - const uint64_t now = RR->node->now(); - SharedPtr<Peer> localPeer(RR->topology->getPeerNoCache(localPeerAddress)); - if ((localPeer)&&(numRemotePeerPaths > 0)) { - InetAddress bestLocalV4,bestLocalV6; - localPeer->getRendezvousAddresses(now,bestLocalV4,bestLocalV6); - - InetAddress bestRemoteV4,bestRemoteV6; - for(unsigned int i=0;i<numRemotePeerPaths;++i) { - if ((bestRemoteV4)&&(bestRemoteV6)) - break; - switch(remotePeerPaths[i].ss_family) { - case AF_INET: - if (!bestRemoteV4) - bestRemoteV4 = remotePeerPaths[i]; - break; - case AF_INET6: - if (!bestRemoteV6) - bestRemoteV6 = remotePeerPaths[i]; - break; - } - } - - Packet rendezvousForLocal(localPeerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS); - rendezvousForLocal.append((uint8_t)0); - remotePeerAddress.appendTo(rendezvousForLocal); - - Buffer<2048> rendezvousForRemote; - remotePeerAddress.appendTo(rendezvousForRemote); - rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS); - rendezvousForRemote.addSize(2); // space for actual packet payload length - rendezvousForRemote.append((uint8_t)0); // flags == 0 - localPeerAddress.appendTo(rendezvousForRemote); - - bool haveMatch = false; - if ((bestLocalV6)&&(bestRemoteV6)) { - haveMatch = true; - - rendezvousForLocal.append((uint16_t)bestRemoteV6.port()); - rendezvousForLocal.append((uint8_t)16); - rendezvousForLocal.append(bestRemoteV6.rawIpData(),16); - - rendezvousForRemote.append((uint16_t)bestLocalV6.port()); - rendezvousForRemote.append((uint8_t)16); - rendezvousForRemote.append(bestLocalV6.rawIpData(),16); - rendezvousForRemote.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 16)); - } else if ((bestLocalV4)&&(bestRemoteV4)) { - haveMatch = true; - - rendezvousForLocal.append((uint16_t)bestRemoteV4.port()); - rendezvousForLocal.append((uint8_t)4); - rendezvousForLocal.append(bestRemoteV4.rawIpData(),4); - - rendezvousForRemote.append((uint16_t)bestLocalV4.port()); - rendezvousForRemote.append((uint8_t)4); - rendezvousForRemote.append(bestLocalV4.rawIpData(),4); - rendezvousForRemote.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 4)); - } - - if (haveMatch) { - { - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size()); - } - RR->sw->send((void *)0,rendezvousForLocal,true); - } - } - } break; - - case CLUSTER_MESSAGE_PROXY_SEND: { - const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const Packet::Verb verb = (Packet::Verb)dmsg[ptr++]; - const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2; - Packet outp(rcpt,RR->identity.address(),verb); - outp.append(dmsg.field(ptr,len),len); ptr += len; - RR->sw->send((void *)0,outp,true); - //TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len); - } break; - - case CLUSTER_MESSAGE_NETWORK_CONFIG: { - const SharedPtr<Network> network(RR->node->network(dmsg.at<uint64_t>(ptr))); - if (network) { - // Copy into a Packet just to conform to Network API. Eventually - // will want to refactor. - network->handleConfigChunk((void *)0,0,Address(),Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(dmsg),ptr); - } - } break; - } - } catch ( ... ) { - TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype); - // drop invalids - } - - ptr = nextPtr; - } - } catch ( ... ) { - TRACE("invalid message (outer loop), discarding"); - // drop invalids - } -} - -void Cluster::broadcastHavePeer(const Identity &id) -{ - Buffer<1024> buf; - id.serialize(buf); - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); - } -} - -void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) -{ - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len); - } -} - -int Cluster::checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret) -{ - const uint64_t now = RR->node->now(); - mostRecentTs = 0; - int mostRecentMemberId = -1; - { - Mutex::Lock _l2(_remotePeers_m); - std::map< std::pair<Address,unsigned int>,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0))); - for(;;) { - if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) - break; - else if (rpe->second.lastHavePeerReceived > mostRecentTs) { - mostRecentTs = rpe->second.lastHavePeerReceived; - memcpy(peerSecret,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH); - mostRecentMemberId = (int)rpe->first.second; - } - ++rpe; - } - } - - const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; - if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { - if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT) - mostRecentMemberId = -1; - - bool sendWantPeer = true; - { - Mutex::Lock _l(_remotePeers_m); - _RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(toPeerAddress,(unsigned int)_id)]; - if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { - rp.lastSentWantPeer = now; - } else { - sendWantPeer = false; // don't flood WANT_PEER - } - } - if (sendWantPeer) { - char tmp[ZT_ADDRESS_LENGTH]; - toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); - { - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); - } - } - } - } - - return mostRecentMemberId; -} - -bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len) -{ - if ((mostRecentMemberId < 0)||(mostRecentMemberId >= ZT_CLUSTER_MAX_MEMBERS)) // sanity check - return false; - Mutex::Lock _l2(_members[mostRecentMemberId].lock); - for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { - for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { - if (i1->ss_family == i2->ss_family) { - TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); - RR->node->putPacket((void *)0,*i1,*i2,data,len); - return true; - } - } - } - return false; -} - -void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) -{ - if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check - return; - - const uint64_t now = RR->node->now(); - - uint64_t mostRecentTs = 0; - int mostRecentMemberId = -1; - { - Mutex::Lock _l2(_remotePeers_m); - std::map< std::pair<Address,unsigned int>,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0))); - for(;;) { - if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) - break; - else if (rpe->second.lastHavePeerReceived > mostRecentTs) { - mostRecentTs = rpe->second.lastHavePeerReceived; - mostRecentMemberId = (int)rpe->first.second; - } - ++rpe; - } - } - - const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; - if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { - // Enqueue and wait if peer seems alive, but do WANT_PEER to refresh homing - const bool enqueueAndWait = ((ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId < 0)); - - // Poll everyone with WANT_PEER if the age of our most recent entry is - // approaching expiration (or has expired, or does not exist). - bool sendWantPeer = true; - { - Mutex::Lock _l(_remotePeers_m); - _RemotePeer &rp = _remotePeers[std::pair<Address,unsigned int>(toPeerAddress,(unsigned int)_id)]; - if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { - rp.lastSentWantPeer = now; - } else { - sendWantPeer = false; // don't flood WANT_PEER - } - } - if (sendWantPeer) { - char tmp[ZT_ADDRESS_LENGTH]; - toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); - { - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); - } - } - } - - // If there isn't a good place to send via, then enqueue this for retrying - // later and return after having broadcasted a WANT_PEER. - if (enqueueAndWait) { - TRACE("relayViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str()); - _sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite); - return; - } - } - - if (mostRecentMemberId >= 0) { - Buffer<1024> buf; - if (unite) { - InetAddress v4,v6; - if (fromPeerAddress) { - SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress)); - if (fromPeer) - fromPeer->getRendezvousAddresses(now,v4,v6); - } - uint8_t addrCount = 0; - if (v4) - ++addrCount; - if (v6) - ++addrCount; - if (addrCount) { - toPeerAddress.appendTo(buf); - fromPeerAddress.appendTo(buf); - buf.append(addrCount); - if (v4) - v4.serialize(buf); - if (v6) - v6.serialize(buf); - } - } - - { - Mutex::Lock _l2(_members[mostRecentMemberId].lock); - if (buf.size() > 0) - _send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size()); - - for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { - for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { - if (i1->ss_family == i2->ss_family) { - TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); - RR->node->putPacket((void *)0,*i1,*i2,data,len); - return; - } - } - } - - TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId); - } - } -} - -void Cluster::sendDistributedQuery(const Packet &pkt) -{ - Buffer<4096> buf; - buf.append((uint16_t)pkt.size()); - buf.append(pkt.data(),pkt.size()); - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_REMOTE_PACKET,buf.data(),buf.size()); - } -} - -void Cluster::doPeriodicTasks() -{ - const uint64_t now = RR->node->now(); - - if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) { - _lastFlushed = now; - - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - - if ((now - _members[*mid].lastAnnouncedAliveTo) >= ((ZT_CLUSTER_TIMEOUT / 2) - 1000)) { - _members[*mid].lastAnnouncedAliveTo = now; - - Buffer<2048> alive; - alive.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR); - alive.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR); - alive.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); - alive.append((uint8_t)ZT_PROTO_VERSION); - if (_addressToLocationFunction) { - alive.append((int32_t)_x); - alive.append((int32_t)_y); - alive.append((int32_t)_z); - } else { - alive.append((int32_t)0); - alive.append((int32_t)0); - alive.append((int32_t)0); - } - alive.append((uint64_t)now); - alive.append((uint64_t)0); // TODO: compute and send load average - alive.append((uint64_t)RR->topology->countActive(now)); - alive.append((uint64_t)0); // unused/reserved flags - alive.append((uint8_t)_zeroTierPhysicalEndpoints.size()); - for(std::vector<InetAddress>::const_iterator pe(_zeroTierPhysicalEndpoints.begin());pe!=_zeroTierPhysicalEndpoints.end();++pe) - pe->serialize(alive); - _send(*mid,CLUSTER_MESSAGE_ALIVE,alive.data(),alive.size()); - } - - _flush(*mid); - } - } - - if ((now - _lastCleanedRemotePeers) >= (ZT_PEER_ACTIVITY_TIMEOUT * 2)) { - _lastCleanedRemotePeers = now; - - Mutex::Lock _l(_remotePeers_m); - for(std::map< std::pair<Address,unsigned int>,_RemotePeer >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) { - if ((now - rp->second.lastHavePeerReceived) >= ZT_PEER_ACTIVITY_TIMEOUT) - _remotePeers.erase(rp++); - else ++rp; - } - } - - if ((now - _lastCleanedQueue) >= ZT_CLUSTER_QUEUE_EXPIRATION) { - _lastCleanedQueue = now; - _sendQueue->expire(now); - } -} - -void Cluster::addMember(uint16_t memberId) -{ - if ((memberId >= ZT_CLUSTER_MAX_MEMBERS)||(memberId == _id)) - return; - - Mutex::Lock _l2(_members[memberId].lock); - - { - Mutex::Lock _l(_memberIds_m); - if (std::find(_memberIds.begin(),_memberIds.end(),memberId) != _memberIds.end()) - return; - _memberIds.push_back(memberId); - std::sort(_memberIds.begin(),_memberIds.end()); - } - - _members[memberId].clear(); - - // Generate this member's message key from the master and its ID - uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; - memcpy(stmp,_masterSecret,sizeof(stmp)); - stmp[0] ^= Utils::hton(memberId); - SHA512::hash(stmp,stmp,sizeof(stmp)); - SHA512::hash(stmp,stmp,sizeof(stmp)); - memcpy(_members[memberId].key,stmp,sizeof(_members[memberId].key)); - Utils::burn(stmp,sizeof(stmp)); - - // Prepare q - _members[memberId].q.clear(); - char iv[16]; - Utils::getSecureRandom(iv,16); - _members[memberId].q.append(iv,16); - _members[memberId].q.addSize(8); // room for MAC - _members[memberId].q.append((uint16_t)_id); - _members[memberId].q.append((uint16_t)memberId); -} - -void Cluster::removeMember(uint16_t memberId) -{ - Mutex::Lock _l(_memberIds_m); - std::vector<uint16_t> newMemberIds; - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - if (*mid != memberId) - newMemberIds.push_back(*mid); - } - _memberIds = newMemberIds; -} - -bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload) -{ - if (_addressToLocationFunction) { - // Pick based on location if it can be determined - int px = 0,py = 0,pz = 0; - if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) { - TRACE("no geolocation data for %s",peerPhysicalAddress.toIpString().c_str()); - return false; - } - - // Find member closest to this peer - const uint64_t now = RR->node->now(); - std::vector<InetAddress> best; - const double currentDistance = _dist3d(_x,_y,_z,px,py,pz); - double bestDistance = (offload ? 2147483648.0 : currentDistance); -#ifdef ZT_TRACE - unsigned int bestMember = _id; -#endif - { - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - _Member &m = _members[*mid]; - Mutex::Lock _ml(m.lock); - - // Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to - if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) { - const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz); - if (mdist < bestDistance) { - bestDistance = mdist; -#ifdef ZT_TRACE - bestMember = *mid; -#endif - best = m.zeroTierPhysicalEndpoints; - } - } - } - } - - // Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family - for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) { - if (a->ss_family == peerPhysicalAddress.ss_family) { - TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str()); - redirectTo = *a; - return true; - } - } - TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance); - return false; - } else { - // TODO: pick based on load if no location info? - return false; - } -} - -bool Cluster::isClusterPeerFrontplane(const InetAddress &ip) const -{ - Mutex::Lock _l(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - for(std::vector<InetAddress>::const_iterator i2(_members[*mid].zeroTierPhysicalEndpoints.begin());i2!=_members[*mid].zeroTierPhysicalEndpoints.end();++i2) { - if (ip == *i2) - return true; - } - } - return false; -} - -void Cluster::status(ZT_ClusterStatus &status) const -{ - const uint64_t now = RR->node->now(); - memset(&status,0,sizeof(ZT_ClusterStatus)); - - status.myId = _id; - - { - ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); - s->id = _id; - s->alive = 1; - s->x = _x; - s->y = _y; - s->z = _z; - s->load = 0; // TODO - s->peers = RR->topology->countActive(now); - for(std::vector<InetAddress>::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) { - if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check - break; - memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); - } - } - - { - Mutex::Lock _l1(_memberIds_m); - for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check - break; - - _Member &m = _members[*mid]; - Mutex::Lock ml(m.lock); - - ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); - s->id = *mid; - s->msSinceLastHeartbeat = (unsigned int)std::min((uint64_t)(~((unsigned int)0)),(now - m.lastReceivedAliveAnnouncement)); - s->alive = (s->msSinceLastHeartbeat < ZT_CLUSTER_TIMEOUT) ? 1 : 0; - s->x = m.x; - s->y = m.y; - s->z = m.z; - s->load = m.load; - s->peers = m.peers; - for(std::vector<InetAddress>::const_iterator ep(m.zeroTierPhysicalEndpoints.begin());ep!=m.zeroTierPhysicalEndpoints.end();++ep) { - if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check - break; - memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); - } - } - } -} - -void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len) -{ - if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check - return; - _Member &m = _members[memberId]; - // assumes m.lock is locked! - if ((m.q.size() + len + 3) > ZT_CLUSTER_MAX_MESSAGE_LENGTH) - _flush(memberId); - m.q.append((uint16_t)(len + 1)); - m.q.append((uint8_t)type); - m.q.append(msg,len); -} - -void Cluster::_flush(uint16_t memberId) -{ - _Member &m = _members[memberId]; - // assumes m.lock is locked! - if (m.q.size() > (24 + 2 + 2)) { // 16-byte IV + 8-byte MAC + 2 byte from-member-ID + 2 byte to-member-ID - // Create key from member's key and IV - char keytmp[32]; - memcpy(keytmp,m.key,32); - for(int i=0;i<8;++i) - keytmp[i] ^= m.q[i]; - Salsa20 s20(keytmp,m.q.field(8,8)); - Utils::burn(keytmp,sizeof(keytmp)); - - // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") - char polykey[ZT_POLY1305_KEY_LEN]; - memset(polykey,0,sizeof(polykey)); - s20.crypt12(polykey,polykey,sizeof(polykey)); - - // Encrypt m.q in place - s20.crypt12(reinterpret_cast<const char *>(m.q.data()) + 24,const_cast<char *>(reinterpret_cast<const char *>(m.q.data())) + 24,m.q.size() - 24); - - // Add MAC for authentication (encrypt-then-MAC) - char mac[ZT_POLY1305_MAC_LEN]; - Poly1305::compute(mac,reinterpret_cast<const char *>(m.q.data()) + 24,m.q.size() - 24,polykey); - memcpy(m.q.field(16,8),mac,8); - - // Send! - _sendFunction(_sendFunctionArg,memberId,m.q.data(),m.q.size()); - - // Prepare for more - m.q.clear(); - char iv[16]; - Utils::getSecureRandom(iv,16); - m.q.append(iv,16); - m.q.addSize(8); // room for MAC - m.q.append((uint16_t)_id); // from member ID - m.q.append((uint16_t)memberId); // to member ID - } -} - -void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep) -{ - if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) { - Identity queried(RR->topology->getIdentity((void *)0,Address(remotep.payload(),ZT_ADDRESS_LENGTH))); - if (queried) { - Buffer<1024> routp; - remotep.source().appendTo(routp); - routp.append((uint8_t)Packet::VERB_OK); - routp.addSize(2); // space for length - routp.append((uint8_t)Packet::VERB_WHOIS); - routp.append(remotep.packetId()); - queried.serialize(routp); - routp.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); - - TRACE("responding to remote WHOIS from %s @ %u with identity of %s",remotep.source().toString().c_str(),(unsigned int)fromMemberId,queried.address().toString().c_str()); - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); - } - } -} - -void Cluster::_doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep) -{ - const uint64_t nwid = remotep.at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); - const MulticastGroup mg(MAC(remotep.field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),remotep.at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); - unsigned int gatherLimit = remotep.at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); - const Address remotePeerAddress(remotep.source()); - - if (gatherLimit) { - Buffer<ZT_PROTO_MAX_PACKET_LENGTH> routp; - remotePeerAddress.appendTo(routp); - routp.append((uint8_t)Packet::VERB_OK); - routp.addSize(2); // space for length - routp.append((uint8_t)Packet::VERB_MULTICAST_GATHER); - routp.append(remotep.packetId()); - routp.append(nwid); - mg.mac().appendTo(routp); - routp.append((uint32_t)mg.adi()); - - if (gatherLimit > ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5)) - gatherLimit = ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5); - if (RR->mc->gather(remotePeerAddress,nwid,mg,routp,gatherLimit)) { - routp.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); - - TRACE("responding to remote MULTICAST_GATHER from %s @ %u with %u bytes",remotePeerAddress.toString().c_str(),(unsigned int)fromMemberId,routp.size()); - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); - } - } -} - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER diff --git a/attic/Cluster.hpp b/attic/Cluster.hpp deleted file mode 100644 index 74b091f5..00000000 --- a/attic/Cluster.hpp +++ /dev/null @@ -1,463 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CLUSTER_HPP -#define ZT_CLUSTER_HPP - -#ifdef ZT_ENABLE_CLUSTER - -#include <map> - -#include "Constants.hpp" -#include "../include/ZeroTierOne.h" -#include "Address.hpp" -#include "InetAddress.hpp" -#include "SHA512.hpp" -#include "Utils.hpp" -#include "Buffer.hpp" -#include "Mutex.hpp" -#include "SharedPtr.hpp" -#include "Hashtable.hpp" -#include "Packet.hpp" -#include "SharedPtr.hpp" - -/** - * Timeout for cluster members being considered "alive" - * - * A cluster member is considered dead and will no longer have peers - * redirected to it if we have not heard a heartbeat in this long. - */ -#define ZT_CLUSTER_TIMEOUT 5000 - -/** - * Desired period between doPeriodicTasks() in milliseconds - */ -#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 20 - -/** - * How often to flush outgoing message queues (maximum interval) - */ -#define ZT_CLUSTER_FLUSH_PERIOD ZT_CLUSTER_PERIODIC_TASK_PERIOD - -/** - * Maximum number of queued outgoing packets per sender address - */ -#define ZT_CLUSTER_MAX_QUEUE_PER_SENDER 16 - -/** - * Expiration time for send queue entries - */ -#define ZT_CLUSTER_QUEUE_EXPIRATION 3000 - -/** - * Chunk size for allocating queue entries - * - * Queue entries are allocated in chunks of this many and are added to a pool. - * ZT_CLUSTER_MAX_QUEUE_GLOBAL must be evenly divisible by this. - */ -#define ZT_CLUSTER_QUEUE_CHUNK_SIZE 32 - -/** - * Maximum number of chunks to ever allocate - * - * This is a global sanity limit to prevent resource exhaustion attacks. It - * works out to about 600mb of RAM. You'll never see this on a normal edge - * node. We're unlikely to see this on a root server unless someone is DOSing - * us. In that case cluster relaying will be affected but other functions - * should continue to operate normally. - */ -#define ZT_CLUSTER_MAX_QUEUE_CHUNKS 8194 - -/** - * Max data per queue entry - */ -#define ZT_CLUSTER_SEND_QUEUE_DATA_MAX 1500 - -/** - * We won't send WANT_PEER to other members more than every (ms) per recipient - */ -#define ZT_CLUSTER_WANT_PEER_EVERY 1000 - -namespace ZeroTier { - -class RuntimeEnvironment; -class MulticastGroup; -class Peer; -class Identity; - -// Internal class implemented inside Cluster.cpp -class _ClusterSendQueue; - -/** - * Multi-homing cluster state replication and packet relaying - * - * Multi-homing means more than one node sharing the same ZeroTier identity. - * There is nothing in the protocol to prevent this, but to make it work well - * requires the devices sharing an identity to cooperate and share some - * information. - * - * There are three use cases we want to fulfill: - * - * (1) Multi-homing of root servers with handoff for efficient routing, - * HA, and load balancing across many commodity nodes. - * (2) Multi-homing of network controllers for the same reason. - * (3) Multi-homing of nodes on virtual networks, such as domain servers - * and other important endpoints. - * - * These use cases are in order of escalating difficulty. The initial - * version of Cluster is aimed at satisfying the first, though you are - * free to try #2 and #3. - */ -class Cluster -{ -public: - /** - * State message types - */ - enum StateMessageType - { - CLUSTER_MESSAGE_NOP = 0, - - /** - * This cluster member is alive: - * <[2] version minor> - * <[2] version major> - * <[2] version revision> - * <[1] protocol version> - * <[4] X location (signed 32-bit)> - * <[4] Y location (signed 32-bit)> - * <[4] Z location (signed 32-bit)> - * <[8] local clock at this member> - * <[8] load average> - * <[8] number of peers> - * <[8] flags (currently unused, must be zero)> - * <[1] number of preferred ZeroTier endpoints> - * <[...] InetAddress(es) of preferred ZeroTier endpoint(s)> - * - * Cluster members constantly broadcast an alive heartbeat and will only - * receive peer redirects if they've done so within the timeout. - */ - CLUSTER_MESSAGE_ALIVE = 1, - - /** - * Cluster member has this peer: - * <[...] serialized identity of peer> - * - * This is typically sent in response to WANT_PEER but can also be pushed - * to prepopulate if this makes sense. - */ - CLUSTER_MESSAGE_HAVE_PEER = 2, - - /** - * Cluster member wants this peer: - * <[5] ZeroTier address of peer> - * - * Members that have a direct link to this peer will respond with - * HAVE_PEER. - */ - CLUSTER_MESSAGE_WANT_PEER = 3, - - /** - * A remote packet that we should also possibly respond to: - * <[2] 16-bit length of remote packet> - * <[...] remote packet payload> - * - * Cluster members may relay requests by relaying the request packet. - * These may include requests such as WHOIS and MULTICAST_GATHER. The - * packet must be already decrypted, decompressed, and authenticated. - * - * This can only be used for small request packets as per the cluster - * message size limit, but since these are the only ones in question - * this is fine. - * - * If a response is generated it is sent via PROXY_SEND. - */ - CLUSTER_MESSAGE_REMOTE_PACKET = 4, - - /** - * Request that VERB_RENDEZVOUS be sent to a peer that we have: - * <[5] ZeroTier address of peer on recipient's side> - * <[5] ZeroTier address of peer on sender's side> - * <[1] 8-bit number of sender's peer's active path addresses> - * <[...] series of serialized InetAddresses of sender's peer's paths> - * - * This requests that we perform NAT-t introduction between a peer that - * we have and one on the sender's side. The sender furnishes contact - * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours - * directly and with PROXY_SEND to theirs. - */ - CLUSTER_MESSAGE_PROXY_UNITE = 5, - - /** - * Request that a cluster member send a packet to a locally-known peer: - * <[5] ZeroTier address of recipient> - * <[1] packet verb> - * <[2] length of packet payload> - * <[...] packet payload> - * - * This differs from RELAY in that it requests the receiving cluster - * member to actually compose a ZeroTier Packet from itself to the - * provided recipient. RELAY simply says "please forward this blob." - * RELAY is used to implement peer-to-peer relaying with RENDEZVOUS, - * while PROXY_SEND is used to implement proxy sending (which right - * now is only used to send RENDEZVOUS). - */ - CLUSTER_MESSAGE_PROXY_SEND = 6, - - /** - * Replicate a network config for a network we belong to: - * <[...] network config chunk> - * - * This is used by clusters to avoid every member having to query - * for the same netconf for networks all members belong to. - * - * The first field of a network config chunk is the network ID, - * so this can be checked to look up the network on receipt. - */ - CLUSTER_MESSAGE_NETWORK_CONFIG = 7 - }; - - /** - * Construct a new cluster - */ - Cluster( - const RuntimeEnvironment *renv, - uint16_t id, - const std::vector<InetAddress> &zeroTierPhysicalEndpoints, - int32_t x, - int32_t y, - int32_t z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg); - - ~Cluster(); - - /** - * @return This cluster member's ID - */ - inline uint16_t id() const throw() { return _id; } - - /** - * Handle an incoming intra-cluster message - * - * @param data Message data - * @param len Message length (max: ZT_CLUSTER_MAX_MESSAGE_LENGTH) - */ - void handleIncomingStateMessage(const void *msg,unsigned int len); - - /** - * Broadcast that we have a given peer - * - * This should be done when new peers are first contacted. - * - * @param id Identity of peer - */ - void broadcastHavePeer(const Identity &id); - - /** - * Broadcast a network config chunk to other members of cluster - * - * @param chunk Chunk data - * @param len Length of chunk - */ - void broadcastNetworkConfigChunk(const void *chunk,unsigned int len); - - /** - * If the cluster has this peer, prepare the packet to send via cluster - * - * Note that outp is only armored (or modified at all) if the return value is a member ID. - * - * @param toPeerAddress Value of outp.destination(), simply to save additional lookup - * @param ts Result: set to time of last HAVE_PEER from the cluster - * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes - * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() - */ - int checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret); - - /** - * Send data via cluster front plane (packet head or fragment) - * - * @param haveMemberId Member ID that has this peer as returned by prepSendviaCluster() - * @param toPeerAddress Destination peer address - * @param data Packet or packet fragment data - * @param len Length of packet or fragment - * @return True if packet was sent (and outp was modified via armoring) - */ - bool sendViaCluster(int haveMemberId,const Address &toPeerAddress,const void *data,unsigned int len); - - /** - * Relay a packet via the cluster - * - * This is used in the outgoing packet and relaying logic in Switch to - * relay packets to other cluster members. It isn't PROXY_SEND-- that is - * used internally in Cluster to send responses to peer queries. - * - * @param fromPeerAddress Source peer address (if known, should be NULL for fragments) - * @param toPeerAddress Destination peer address - * @param data Packet or packet fragment data - * @param len Length of packet or fragment - * @param unite If true, also request proxy unite across cluster - */ - void relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite); - - /** - * Send a distributed query to other cluster members - * - * Some queries such as WHOIS or MULTICAST_GATHER need a response from other - * cluster members. Replies (if any) will be sent back to the peer via - * PROXY_SEND across the cluster. - * - * @param pkt Packet to distribute - */ - void sendDistributedQuery(const Packet &pkt); - - /** - * Call every ~ZT_CLUSTER_PERIODIC_TASK_PERIOD milliseconds. - */ - void doPeriodicTasks(); - - /** - * Add a member ID to this cluster - * - * @param memberId Member ID - */ - void addMember(uint16_t memberId); - - /** - * Remove a member ID from this cluster - * - * @param memberId Member ID to remove - */ - void removeMember(uint16_t memberId); - - /** - * Find a better cluster endpoint for this peer (if any) - * - * @param redirectTo InetAddress to be set to a better endpoint (if there is one) - * @param peerAddress Address of peer to (possibly) redirect - * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address()) - * @param offload Always redirect if possible -- can be used to offload peers during shutdown - * @return True if redirectTo was set to a new address, false if redirectTo was not modified - */ - bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload); - - /** - * @param ip Address to check - * @return True if this is a cluster frontplane address (excluding our addresses) - */ - bool isClusterPeerFrontplane(const InetAddress &ip) const; - - /** - * Fill out ZT_ClusterStatus structure (from core API) - * - * @param status Reference to structure to hold result (anything there is replaced) - */ - void status(ZT_ClusterStatus &status) const; - -private: - void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len); - void _flush(uint16_t memberId); - - void _doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep); - void _doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep); - - // These are initialized in the constructor and remain immutable ------------ - uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; - unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; - const RuntimeEnvironment *RR; - _ClusterSendQueue *const _sendQueue; - void (*_sendFunction)(void *,unsigned int,const void *,unsigned int); - void *_sendFunctionArg; - int (*_addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *); - void *_addressToLocationFunctionArg; - const int32_t _x; - const int32_t _y; - const int32_t _z; - const uint16_t _id; - const std::vector<InetAddress> _zeroTierPhysicalEndpoints; - // end immutable fields ----------------------------------------------------- - - struct _Member - { - unsigned char key[ZT_PEER_SECRET_KEY_LENGTH]; - - uint64_t lastReceivedAliveAnnouncement; - uint64_t lastAnnouncedAliveTo; - - uint64_t load; - uint64_t peers; - int32_t x,y,z; - - std::vector<InetAddress> zeroTierPhysicalEndpoints; - - Buffer<ZT_CLUSTER_MAX_MESSAGE_LENGTH> q; - - Mutex lock; - - inline void clear() - { - lastReceivedAliveAnnouncement = 0; - lastAnnouncedAliveTo = 0; - load = 0; - peers = 0; - x = 0; - y = 0; - z = 0; - zeroTierPhysicalEndpoints.clear(); - q.clear(); - } - - _Member() { this->clear(); } - ~_Member() { Utils::burn(key,sizeof(key)); } - }; - _Member *const _members; - - std::vector<uint16_t> _memberIds; - Mutex _memberIds_m; - - struct _RemotePeer - { - _RemotePeer() : lastHavePeerReceived(0),lastSentWantPeer(0) {} - ~_RemotePeer() { Utils::burn(key,ZT_PEER_SECRET_KEY_LENGTH); } - uint64_t lastHavePeerReceived; - uint64_t lastSentWantPeer; - uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; // secret key from identity agreement - }; - std::map< std::pair<Address,unsigned int>,_RemotePeer > _remotePeers; // we need ordered behavior and lower_bound here - Mutex _remotePeers_m; - - uint64_t _lastFlushed; - uint64_t _lastCleanedRemotePeers; - uint64_t _lastCleanedQueue; -}; - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -#endif diff --git a/attic/ClusterDefinition.hpp b/attic/ClusterDefinition.hpp deleted file mode 100644 index b6317ff7..00000000 --- a/attic/ClusterDefinition.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CLUSTERDEFINITION_HPP -#define ZT_CLUSTERDEFINITION_HPP - -#ifdef ZT_ENABLE_CLUSTER - -#include <vector> -#include <algorithm> - -#include "../node/Constants.hpp" -#include "../node/Utils.hpp" -#include "../node/NonCopyable.hpp" -#include "../osdep/OSUtils.hpp" - -#include "ClusterGeoIpService.hpp" - -namespace ZeroTier { - -/** - * Parser for cluster definition file - */ -class ClusterDefinition : NonCopyable -{ -public: - struct MemberDefinition - { - MemberDefinition() : id(0),x(0),y(0),z(0) { name[0] = (char)0; } - - unsigned int id; - int x,y,z; - char name[256]; - InetAddress clusterEndpoint; - std::vector<InetAddress> zeroTierEndpoints; - }; - - /** - * Load and initialize cluster definition and GeoIP data if any - * - * @param myAddress My ZeroTier address - * @param pathToClusterFile Path to cluster definition file - * @throws std::runtime_error Invalid cluster definition or unable to load data - */ - ClusterDefinition(uint64_t myAddress,const char *pathToClusterFile) - { - std::string cf; - if (!OSUtils::readFile(pathToClusterFile,cf)) - return; - - char myAddressStr[64]; - Utils::ztsnprintf(myAddressStr,sizeof(myAddressStr),"%.10llx",myAddress); - - std::vector<std::string> lines(OSUtils::split(cf.c_str(),"\r\n","","")); - for(std::vector<std::string>::iterator l(lines.begin());l!=lines.end();++l) { - std::vector<std::string> fields(OSUtils::split(l->c_str()," \t","","")); - if ((fields.size() < 5)||(fields[0][0] == '#')||(fields[0] != myAddressStr)) - continue; - - // <address> geo <CSV path> <ip start column> <ip end column> <latitutde column> <longitude column> - if (fields[1] == "geo") { - if ((fields.size() >= 7)&&(OSUtils::fileExists(fields[2].c_str()))) { - int ipStartColumn = Utils::strToInt(fields[3].c_str()); - int ipEndColumn = Utils::strToInt(fields[4].c_str()); - int latitudeColumn = Utils::strToInt(fields[5].c_str()); - int longitudeColumn = Utils::strToInt(fields[6].c_str()); - if (_geo.load(fields[2].c_str(),ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn) <= 0) - throw std::runtime_error(std::string("failed to load geo-ip data from ")+fields[2]); - } - continue; - } - - // <address> <ID> <name> <backplane IP/port(s)> <ZT frontplane IP/port(s)> <x,y,z> - int id = Utils::strToUInt(fields[1].c_str()); - if ((id < 0)||(id > ZT_CLUSTER_MAX_MEMBERS)) - throw std::runtime_error(std::string("invalid cluster member ID: ")+fields[1]); - MemberDefinition &md = _md[id]; - - md.id = (unsigned int)id; - if (fields.size() >= 6) { - std::vector<std::string> xyz(OSUtils::split(fields[5].c_str(),",","","")); - md.x = (xyz.size() > 0) ? Utils::strToInt(xyz[0].c_str()) : 0; - md.y = (xyz.size() > 1) ? Utils::strToInt(xyz[1].c_str()) : 0; - md.z = (xyz.size() > 2) ? Utils::strToInt(xyz[2].c_str()) : 0; - } - Utils::scopy(md.name,sizeof(md.name),fields[2].c_str()); - md.clusterEndpoint.fromString(fields[3]); - if (!md.clusterEndpoint) - continue; - std::vector<std::string> zips(OSUtils::split(fields[4].c_str(),",","","")); - for(std::vector<std::string>::iterator zip(zips.begin());zip!=zips.end();++zip) { - InetAddress i; - i.fromString(*zip); - if (i) - md.zeroTierEndpoints.push_back(i); - } - - _ids.push_back((unsigned int)id); - } - - std::sort(_ids.begin(),_ids.end()); - } - - /** - * @return All member definitions in this cluster by ID (ID is array index) - */ - inline const MemberDefinition &operator[](unsigned int id) const throw() { return _md[id]; } - - /** - * @return Number of members in this cluster - */ - inline unsigned int size() const throw() { return (unsigned int)_ids.size(); } - - /** - * @return IDs of members in this cluster sorted by ID - */ - inline const std::vector<unsigned int> &ids() const throw() { return _ids; } - - /** - * @return GeoIP service for this cluster - */ - inline ClusterGeoIpService &geo() throw() { return _geo; } - - /** - * @return A vector (new copy) containing all cluster members - */ - inline std::vector<MemberDefinition> members() const - { - std::vector<MemberDefinition> m; - for(std::vector<unsigned int>::const_iterator i(_ids.begin());i!=_ids.end();++i) - m.push_back(_md[*i]); - return m; - } - -private: - MemberDefinition _md[ZT_CLUSTER_MAX_MEMBERS]; - std::vector<unsigned int> _ids; - ClusterGeoIpService _geo; -}; - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -#endif diff --git a/attic/ClusterGeoIpService.cpp b/attic/ClusterGeoIpService.cpp deleted file mode 100644 index 2dcc9179..00000000 --- a/attic/ClusterGeoIpService.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifdef ZT_ENABLE_CLUSTER - -#include <math.h> - -#include <cmath> - -#include "ClusterGeoIpService.hpp" - -#include "../node/Utils.hpp" -#include "../osdep/OSUtils.hpp" - -#define ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY 10000 - -namespace ZeroTier { - -ClusterGeoIpService::ClusterGeoIpService() : - _pathToCsv(), - _ipStartColumn(-1), - _ipEndColumn(-1), - _latitudeColumn(-1), - _longitudeColumn(-1), - _lastFileCheckTime(0), - _csvModificationTime(0), - _csvFileSize(0) -{ -} - -ClusterGeoIpService::~ClusterGeoIpService() -{ -} - -bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z) -{ - Mutex::Lock _l(_lock); - - if ((_pathToCsv.length() > 0)&&((OSUtils::now() - _lastFileCheckTime) > ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY)) { - _lastFileCheckTime = OSUtils::now(); - if ((_csvFileSize != OSUtils::getFileSize(_pathToCsv.c_str()))||(_csvModificationTime != OSUtils::getLastModified(_pathToCsv.c_str()))) - _load(_pathToCsv.c_str(),_ipStartColumn,_ipEndColumn,_latitudeColumn,_longitudeColumn); - } - - /* We search by looking up the upper bound of the sorted vXdb vectors - * and then iterating down for a matching IP range. We stop when we hit - * the beginning or an entry whose start and end are before the IP we - * are searching. */ - - if ((ip.ss_family == AF_INET)&&(_v4db.size() > 0)) { - _V4E key; - key.start = Utils::ntoh((uint32_t)(reinterpret_cast<const struct sockaddr_in *>(&ip)->sin_addr.s_addr)); - std::vector<_V4E>::const_iterator i(std::upper_bound(_v4db.begin(),_v4db.end(),key)); - while (i != _v4db.begin()) { - --i; - if ((key.start >= i->start)&&(key.start <= i->end)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((key.start > i->start)&&(key.start > i->end)) - break; - } - } else if ((ip.ss_family == AF_INET6)&&(_v6db.size() > 0)) { - _V6E key; - memcpy(key.start,reinterpret_cast<const struct sockaddr_in6 *>(&ip)->sin6_addr.s6_addr,16); - std::vector<_V6E>::const_iterator i(std::upper_bound(_v6db.begin(),_v6db.end(),key)); - while (i != _v6db.begin()) { - --i; - const int s_vs_s = memcmp(key.start,i->start,16); - const int s_vs_e = memcmp(key.start,i->end,16); - if ((s_vs_s >= 0)&&(s_vs_e <= 0)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((s_vs_s > 0)&&(s_vs_e > 0)) - break; - } - } - - return false; -} - -void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - std::vector<std::string> ls(OSUtils::split(line,",\t","\\","\"'")); - if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&& - ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&& - ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&& - ((longitudeColumn >= 0)&&(longitudeColumn < (int)ls.size())) ) { - InetAddress ipStart(ls[ipStartColumn].c_str(),0); - InetAddress ipEnd(ls[ipEndColumn].c_str(),0); - const double lat = strtod(ls[latitudeColumn].c_str(),(char **)0); - const double lon = strtod(ls[longitudeColumn].c_str(),(char **)0); - - if ((ipStart.ss_family == ipEnd.ss_family)&&(ipStart)&&(ipEnd)&&(std::isfinite(lat))&&(std::isfinite(lon))) { - const double latRadians = lat * 0.01745329251994; // PI / 180 - const double lonRadians = lon * 0.01745329251994; // PI / 180 - const double cosLat = cos(latRadians); - const int x = (int)round((-6371.0) * cosLat * cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers - const int y = (int)round(6371.0 * sin(latRadians)); - const int z = (int)round(6371.0 * cosLat * sin(lonRadians)); - - if (ipStart.ss_family == AF_INET) { - v4db.push_back(_V4E()); - v4db.back().start = Utils::ntoh((uint32_t)(reinterpret_cast<const struct sockaddr_in *>(&ipStart)->sin_addr.s_addr)); - v4db.back().end = Utils::ntoh((uint32_t)(reinterpret_cast<const struct sockaddr_in *>(&ipEnd)->sin_addr.s_addr)); - v4db.back().lat = (float)lat; - v4db.back().lon = (float)lon; - v4db.back().x = x; - v4db.back().y = y; - v4db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } else if (ipStart.ss_family == AF_INET6) { - v6db.push_back(_V6E()); - memcpy(v6db.back().start,reinterpret_cast<const struct sockaddr_in6 *>(&ipStart)->sin6_addr.s6_addr,16); - memcpy(v6db.back().end,reinterpret_cast<const struct sockaddr_in6 *>(&ipEnd)->sin6_addr.s6_addr,16); - v6db.back().lat = (float)lat; - v6db.back().lon = (float)lon; - v6db.back().x = x; - v6db.back().y = y; - v6db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } - } - } -} - -long ClusterGeoIpService::_load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - // assumes _lock is locked - - FILE *f = fopen(pathToCsv,"rb"); - if (!f) - return -1; - - std::vector<_V4E> v4db; - std::vector<_V6E> v6db; - v4db.reserve(16777216); - v6db.reserve(16777216); - - char buf[4096]; - char linebuf[1024]; - unsigned int lineptr = 0; - for(;;) { - int n = (int)fread(buf,1,sizeof(buf),f); - if (n <= 0) - break; - for(int i=0;i<n;++i) { - if ((buf[i] == '\r')||(buf[i] == '\n')||(buf[i] == (char)0)) { - if (lineptr) { - linebuf[lineptr] = (char)0; - _parseLine(linebuf,v4db,v6db,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); - } - lineptr = 0; - } else if (lineptr < (unsigned int)sizeof(linebuf)) - linebuf[lineptr++] = buf[i]; - } - } - if (lineptr) { - linebuf[lineptr] = (char)0; - _parseLine(linebuf,v4db,v6db,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); - } - - fclose(f); - - if ((v4db.size() > 0)||(v6db.size() > 0)) { - std::sort(v4db.begin(),v4db.end()); - std::sort(v6db.begin(),v6db.end()); - - _pathToCsv = pathToCsv; - _ipStartColumn = ipStartColumn; - _ipEndColumn = ipEndColumn; - _latitudeColumn = latitudeColumn; - _longitudeColumn = longitudeColumn; - - _lastFileCheckTime = OSUtils::now(); - _csvModificationTime = OSUtils::getLastModified(pathToCsv); - _csvFileSize = OSUtils::getFileSize(pathToCsv); - - _v4db.swap(v4db); - _v6db.swap(v6db); - - return (long)(_v4db.size() + _v6db.size()); - } else { - return 0; - } -} - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -/* -int main(int argc,char **argv) -{ - char buf[1024]; - - ZeroTier::ClusterGeoIpService gip; - printf("loading...\n"); - gip.load("/Users/api/Code/ZeroTier/Infrastructure/root-servers/zerotier-one/cluster-geoip.csv",0,1,5,6); - printf("... done!\n"); fflush(stdout); - - while (gets(buf)) { // unsafe, testing only - ZeroTier::InetAddress addr(buf,0); - printf("looking up: %s\n",addr.toString().c_str()); fflush(stdout); - int x = 0,y = 0,z = 0; - if (gip.locate(addr,x,y,z)) { - //printf("%s: %d,%d,%d\n",addr.toString().c_str(),x,y,z); fflush(stdout); - } else { - printf("%s: not found!\n",addr.toString().c_str()); fflush(stdout); - } - } - - return 0; -} -*/ diff --git a/attic/ClusterGeoIpService.hpp b/attic/ClusterGeoIpService.hpp deleted file mode 100644 index 380f944f..00000000 --- a/attic/ClusterGeoIpService.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CLUSTERGEOIPSERVICE_HPP -#define ZT_CLUSTERGEOIPSERVICE_HPP - -#ifdef ZT_ENABLE_CLUSTER - -#include <stdint.h> -#include <stdio.h> -#include <string.h> -#include <stdlib.h> - -#include <vector> -#include <string> -#include <algorithm> - -#include "../node/Constants.hpp" -#include "../node/Mutex.hpp" -#include "../node/NonCopyable.hpp" -#include "../node/InetAddress.hpp" - -namespace ZeroTier { - -/** - * Loads a GeoIP CSV into memory for fast lookup, reloading as needed - * - * This was designed around the CSV from https://db-ip.com but can be used - * with any similar GeoIP CSV database that is presented in the form of an - * IP range and lat/long coordinates. - * - * It loads the whole database into memory, which can be kind of large. If - * the CSV file changes, the changes are loaded automatically. - */ -class ClusterGeoIpService : NonCopyable -{ -public: - ClusterGeoIpService(); - ~ClusterGeoIpService(); - - /** - * Load or reload CSV file - * - * CSV column indexes start at zero. CSVs can be quoted with single or - * double quotes. Whitespace before or after commas is ignored. Backslash - * may be used for escaping whitespace as well. - * - * @param pathToCsv Path to (uncompressed) CSV file - * @param ipStartColumn Column with IP range start - * @param ipEndColumn Column with IP range end (inclusive) - * @param latitudeColumn Column with latitude - * @param longitudeColumn Column with longitude - * @return Number of valid records loaded or -1 on error (invalid file, not found, etc.) - */ - inline long load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) - { - Mutex::Lock _l(_lock); - return _load(pathToCsv,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); - } - - /** - * Attempt to locate an IP - * - * This returns true if x, y, and z are set. If the return value is false - * the values of x, y, and z are undefined. - * - * @param ip IPv4 or IPv6 address - * @param x Reference to variable to receive X - * @param y Reference to variable to receive Y - * @param z Reference to variable to receive Z - * @return True if coordinates were set - */ - bool locate(const InetAddress &ip,int &x,int &y,int &z); - - /** - * @return True if IP database/service is available for queries (otherwise locate() will always be false) - */ - inline bool available() const - { - Mutex::Lock _l(_lock); - return ((_v4db.size() + _v6db.size()) > 0); - } - -private: - struct _V4E - { - uint32_t start; - uint32_t end; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V4E &e) const { return (start < e.start); } - }; - - struct _V6E - { - uint8_t start[16]; - uint8_t end[16]; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V6E &e) const { return (memcmp(start,e.start,16) < 0); } - }; - - static void _parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - long _load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - - std::string _pathToCsv; - int _ipStartColumn; - int _ipEndColumn; - int _latitudeColumn; - int _longitudeColumn; - - uint64_t _lastFileCheckTime; - uint64_t _csvModificationTime; - int64_t _csvFileSize; - - std::vector<_V4E> _v4db; - std::vector<_V6E> _v6db; - - Mutex _lock; -}; - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -#endif diff --git a/attic/FCV.hpp b/attic/FCV.hpp deleted file mode 100644 index 0fb9e250..00000000 --- a/attic/FCV.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#include "Constants.hpp" - -namespace ZeroTier { - -/** - * A really simple fixed capacity vector - * - * This class does no bounds checking, so the user must ensure that - * no more than C elements are ever added and that accesses are in - * bounds. - * - * @tparam T Type to contain - * @tparam C Capacity of vector - */ -template<typename T,unsigned long C> -class FCV -{ -public: - FCV() : _s(0) {} - ~FCV() { clear(); } - - FCV(const FCV &v) : - _s(v._s) - { - for(unsigned long i=0;i<_s;++i) { - new (reinterpret_cast<T *>(_mem + (sizeof(T) * i))) T(reinterpret_cast<const T *>(v._mem)[i]); - } - } - - inline FCV &operator=(const FCV &v) - { - clear(); - _s = v._s; - for(unsigned long i=0;i<_s;++i) { - new (reinterpret_cast<T *>(_mem + (sizeof(T) * i))) T(reinterpret_cast<const T *>(v._mem)[i]); - } - return *this; - } - - typedef T * iterator; - typedef const T * const_iterator; - typedef unsigned long size_type; - - inline iterator begin() { return (T *)_mem; } - inline iterator end() { return (T *)(_mem + (sizeof(T) * _s)); } - inline iterator begin() const { return (const T *)_mem; } - inline iterator end() const { return (const T *)(_mem + (sizeof(T) * _s)); } - - inline T &operator[](const size_type i) { return reinterpret_cast<T *>(_mem)[i]; } - inline const T &operator[](const size_type i) const { return reinterpret_cast<const T *>(_mem)[i]; } - - inline T &front() { return reinterpret_cast<T *>(_mem)[0]; } - inline const T &front() const { return reinterpret_cast<const T *>(_mem)[0]; } - inline T &back() { return reinterpret_cast<T *>(_mem)[_s - 1]; } - inline const T &back() const { return reinterpret_cast<const T *>(_mem)[_s - 1]; } - - inline void push_back(const T &v) { new (reinterpret_cast<T *>(_mem + (sizeof(T) * _s++))) T(v); } - inline void pop_back() { reinterpret_cast<T *>(_mem + (sizeof(T) * --_s))->~T(); } - - inline size_type size() const { return _s; } - inline size_type capacity() const { return C; } - - inline void clear() - { - for(unsigned long i=0;i<_s;++i) - reinterpret_cast<T *>(_mem + (sizeof(T) * i))->~T(); - _s = 0; - } - -private: - char _mem[sizeof(T) * C]; - unsigned long _s; -}; - -} // namespace ZeroTier diff --git a/attic/OSXEthernetTap.cpp.pcap-with-bridge-test b/attic/OSXEthernetTap.cpp.pcap-with-bridge-test deleted file mode 100644 index baae0a4b..00000000 --- a/attic/OSXEthernetTap.cpp.pcap-with-bridge-test +++ /dev/null @@ -1,650 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ - */ - -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <errno.h> - -#include <unistd.h> -#include <signal.h> - -#include <fcntl.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/ioctl.h> -#include <sys/wait.h> -#include <sys/select.h> -#include <sys/cdefs.h> -#include <sys/uio.h> -#include <sys/param.h> -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <net/route.h> -#include <net/if.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> -#include <netinet6/in6_var.h> -#include <netinet/in_var.h> -#include <netinet/icmp6.h> - -#include <pcap/pcap.h> - -// OSX compile fix... in6_var defines this in a struct which namespaces it for C++ ... why?!? -struct prf_ra { - u_char onlink : 1; - u_char autonomous : 1; - u_char reserved : 6; -} prf_ra; - -#include <netinet6/nd6.h> -#include <ifaddrs.h> - -// These are KERNEL_PRIVATE... why? -#ifndef SIOCAUTOCONF_START -#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq) /* accept rtadvd on this interface */ -#endif -#ifndef SIOCAUTOCONF_STOP -#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq) /* stop accepting rtadv for this interface */ -#endif - -#ifndef ETH_ALEN -#define ETH_ALEN 6 -#endif - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -// This source is from: -// http://www.opensource.apple.com/source/Libinfo/Libinfo-406.17/gen.subproj/getifmaddrs.c?txt -// It's here because OSX 10.6 does not have this convenience function. - -#define SALIGN (sizeof(uint32_t) - 1) -#define SA_RLEN(sa) ((sa)->sa_len ? (((sa)->sa_len + SALIGN) & ~SALIGN) : \ -(SALIGN + 1)) -#define MAX_SYSCTL_TRY 5 -#define RTA_MASKS (RTA_GATEWAY | RTA_IFP | RTA_IFA) - -/* FreeBSD uses NET_RT_IFMALIST and RTM_NEWMADDR from <sys/socket.h> */ -/* We can use NET_RT_IFLIST2 and RTM_NEWMADDR2 on Darwin */ -//#define DARWIN_COMPAT - -//#ifdef DARWIN_COMPAT -#define GIM_SYSCTL_MIB NET_RT_IFLIST2 -#define GIM_RTM_ADDR RTM_NEWMADDR2 -//#else -//#define GIM_SYSCTL_MIB NET_RT_IFMALIST -//#define GIM_RTM_ADDR RTM_NEWMADDR -//#endif - -// Not in 10.6 includes so use our own -struct _intl_ifmaddrs { - struct _intl_ifmaddrs *ifma_next; - struct sockaddr *ifma_name; - struct sockaddr *ifma_addr; - struct sockaddr *ifma_lladdr; -}; - -static inline int _intl_getifmaddrs(struct _intl_ifmaddrs **pif) -{ - int icnt = 1; - int dcnt = 0; - int ntry = 0; - size_t len; - size_t needed; - int mib[6]; - int i; - char *buf; - char *data; - char *next; - char *p; - struct ifma_msghdr2 *ifmam; - struct _intl_ifmaddrs *ifa, *ift; - struct rt_msghdr *rtm; - struct sockaddr *sa; - - mib[0] = CTL_NET; - mib[1] = PF_ROUTE; - mib[2] = 0; /* protocol */ - mib[3] = 0; /* wildcard address family */ - mib[4] = GIM_SYSCTL_MIB; - mib[5] = 0; /* no flags */ - do { - if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0) - return (-1); - if ((buf = (char *)malloc(needed)) == NULL) - return (-1); - if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) { - if (errno != ENOMEM || ++ntry >= MAX_SYSCTL_TRY) { - free(buf); - return (-1); - } - free(buf); - buf = NULL; - } - } while (buf == NULL); - - for (next = buf; next < buf + needed; next += rtm->rtm_msglen) { - rtm = (struct rt_msghdr *)(void *)next; - if (rtm->rtm_version != RTM_VERSION) - continue; - switch (rtm->rtm_type) { - case GIM_RTM_ADDR: - ifmam = (struct ifma_msghdr2 *)(void *)rtm; - if ((ifmam->ifmam_addrs & RTA_IFA) == 0) - break; - icnt++; - p = (char *)(ifmam + 1); - for (i = 0; i < RTAX_MAX; i++) { - if ((RTA_MASKS & ifmam->ifmam_addrs & - (1 << i)) == 0) - continue; - sa = (struct sockaddr *)(void *)p; - len = SA_RLEN(sa); - dcnt += len; - p += len; - } - break; - } - } - - data = (char *)malloc(sizeof(struct _intl_ifmaddrs) * icnt + dcnt); - if (data == NULL) { - free(buf); - return (-1); - } - - ifa = (struct _intl_ifmaddrs *)(void *)data; - data += sizeof(struct _intl_ifmaddrs) * icnt; - - memset(ifa, 0, sizeof(struct _intl_ifmaddrs) * icnt); - ift = ifa; - - for (next = buf; next < buf + needed; next += rtm->rtm_msglen) { - rtm = (struct rt_msghdr *)(void *)next; - if (rtm->rtm_version != RTM_VERSION) - continue; - - switch (rtm->rtm_type) { - case GIM_RTM_ADDR: - ifmam = (struct ifma_msghdr2 *)(void *)rtm; - if ((ifmam->ifmam_addrs & RTA_IFA) == 0) - break; - - p = (char *)(ifmam + 1); - for (i = 0; i < RTAX_MAX; i++) { - if ((RTA_MASKS & ifmam->ifmam_addrs & - (1 << i)) == 0) - continue; - sa = (struct sockaddr *)(void *)p; - len = SA_RLEN(sa); - switch (i) { - case RTAX_GATEWAY: - ift->ifma_lladdr = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - case RTAX_IFP: - ift->ifma_name = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - case RTAX_IFA: - ift->ifma_addr = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - default: - data += len; - break; - } - p += len; - } - ift->ifma_next = ift + 1; - ift = ift->ifma_next; - break; - } - } - - free(buf); - - if (ift > ifa) { - ift--; - ift->ifma_next = NULL; - *pif = ifa; - } else { - *pif = NULL; - free(ifa); - } - return (0); -} - -static inline void _intl_freeifmaddrs(struct _intl_ifmaddrs *ifmp) -{ - free(ifmp); -} - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- - -#include <string> -#include <map> -#include <set> -#include <algorithm> - -#include "../node/Constants.hpp" -#include "../node/Utils.hpp" -#include "../node/Mutex.hpp" -#include "../node/Dictionary.hpp" -#include "OSUtils.hpp" -#include "OSXEthernetTap.hpp" - -// ff:ff:ff:ff:ff:ff with no ADI -static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0); - -static inline bool _setIpv6Stuff(const char *ifname,bool performNUD,bool acceptRouterAdverts) -{ - struct in6_ndireq nd; - struct in6_ifreq ifr; - - int s = socket(AF_INET6,SOCK_DGRAM,0); - if (s <= 0) - return false; - - memset(&nd,0,sizeof(nd)); - strncpy(nd.ifname,ifname,sizeof(nd.ifname)); - - if (ioctl(s,SIOCGIFINFO_IN6,&nd)) { - close(s); - return false; - } - - unsigned long oldFlags = (unsigned long)nd.ndi.flags; - - if (performNUD) - nd.ndi.flags |= ND6_IFF_PERFORMNUD; - else nd.ndi.flags &= ~ND6_IFF_PERFORMNUD; - - if (oldFlags != (unsigned long)nd.ndi.flags) { - if (ioctl(s,SIOCSIFINFO_FLAGS,&nd)) { - close(s); - return false; - } - } - - memset(&ifr,0,sizeof(ifr)); - strncpy(ifr.ifr_name,ifname,sizeof(ifr.ifr_name)); - if (ioctl(s,acceptRouterAdverts ? SIOCAUTOCONF_START : SIOCAUTOCONF_STOP,&ifr)) { - close(s); - return false; - } - - close(s); - return true; -} - -namespace ZeroTier { - -static std::set<std::string> globalDeviceNames; -static Mutex globalTapCreateLock; - -OSXEthernetTap::OSXEthernetTap( - const char *homePath, - const MAC &mac, - unsigned int mtu, - unsigned int metric, - uint64_t nwid, - const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), - void *arg) : - _handler(handler), - _arg(arg), - _pcap((void *)0), - _nwid(nwid), - _mac(mac), - _homePath(homePath), - _mtu(mtu), - _metric(metric), - _enabled(true) -{ - char errbuf[PCAP_ERRBUF_SIZE]; - char devname[64],ethaddr[64],mtustr[32],metstr[32],nwids[32]; - - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - - if (mtu > 2800) - throw std::runtime_error("max tap MTU is 2800"); - - Mutex::Lock _gl(globalTapCreateLock); - - std::string desiredDevice; - Dictionary devmap; - { - std::string devmapbuf; - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),devmapbuf)) { - devmap.fromString(devmapbuf); - desiredDevice = devmap.get(nwids,""); - } - } - - if ((desiredDevice.length() >= 9)&&(desiredDevice.substr(0,6) == "bridge")) { - // length() >= 9 matches bridge### or bridge#### - _dev = desiredDevice; - } else { - if (globalDeviceNames.size() >= (10000 - 128)) // sanity check... this would be nuts - throw std::runtime_error("too many devices!"); - unsigned int pseudoBridgeNo = (unsigned int)((nwid ^ (nwid >> 32)) % (10000 - 128)) + 128; // range: bridge128 to bridge9999 - sprintf(devname,"bridge%u",pseudoBridgeNo); - while (globalDeviceNames.count(std::string(devname)) > 0) { - ++pseudoBridgeNo; - if (pseudoBridgeNo > 9999) - pseudoBridgeNo = 64; - sprintf(devname,"bridge%u",pseudoBridgeNo); - } - _dev = devname; - } - - // Configure MAC address and MTU, bring interface up - long cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"create",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - if (exitcode != 0) - throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface"); - } else throw std::runtime_error("unable to fork()"); - Utils::snprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::snprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::snprintf(metstr,sizeof(metstr),"%u",_metric); - cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - if (exitcode != 0) - throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface"); - } else throw std::runtime_error("unable to fork()"); - - _setIpv6Stuff(_dev.c_str(),true,false); - - _pcap = (void *)pcap_create(_dev.c_str(),errbuf); - if (!_pcap) { - cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"destroy",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - } - throw std::runtime_error((std::string("pcap_create() on new bridge device failed: ") + errbuf).c_str()); - } - pcap_set_promisc(reinterpret_cast<pcap_t *>(_pcap),1); - pcap_set_timeout(reinterpret_cast<pcap_t *>(_pcap),120000); - pcap_set_immediate_mode(reinterpret_cast<pcap_t *>(_pcap),1); - if (pcap_set_buffer_size(reinterpret_cast<pcap_t *>(_pcap),1024 * 1024 * 16) != 0) // 16MB - fprintf(stderr,"WARNING: pcap_set_buffer_size() failed!\n"); - if (pcap_set_snaplen(reinterpret_cast<pcap_t *>(_pcap),4096) != 0) - fprintf(stderr,"WARNING: pcap_set_snaplen() failed!\n"); - if (pcap_activate(reinterpret_cast<pcap_t *>(_pcap)) != 0) { - pcap_close(reinterpret_cast<pcap_t *>(_pcap)); - cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"destroy",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - } - throw std::runtime_error("pcap_activate() on new bridge device failed."); - } - - globalDeviceNames.insert(_dev); - - devmap[nwids] = _dev; - OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),devmap.toString()); - - _thread = Thread::start(this); -} - -OSXEthernetTap::~OSXEthernetTap() -{ - _enabled = false; - - Mutex::Lock _gl(globalTapCreateLock); - globalDeviceNames.erase(_dev); - - long cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"destroy",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - if (exitcode == 0) { - // Destroying the interface nukes pcap and terminates the thread. - Thread::join(_thread); - } - } - - pcap_close(reinterpret_cast<pcap_t *>(_pcap)); -} - -static bool ___removeIp(const std::string &_dev,const InetAddress &ip) -{ - long cpid = (long)vfork(); - if (cpid == 0) { - execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"inet",ip.toIpString().c_str(),"-alias",(const char *)0); - _exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } - return false; // never reached, make compiler shut up about return value -} - -bool OSXEthernetTap::addIp(const InetAddress &ip) -{ - if (!ip) - return false; - - std::vector<InetAddress> allIps(ips()); - if (std::binary_search(allIps.begin(),allIps.end(),ip)) - return true; - - // Remove and reconfigure if address is the same but netmask is different - for(std::vector<InetAddress>::iterator i(allIps.begin());i!=allIps.end();++i) { - if ((i->ipsEqual(ip))&&(i->netmaskBits() != ip.netmaskBits())) { - if (___removeIp(_dev,*i)) - break; - } - } - - long cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),ip.isV4() ? "inet" : "inet6",ip.toString().c_str(),"alias",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } // else return false... - - return false; -} - -bool OSXEthernetTap::removeIp(const InetAddress &ip) -{ - if (!ip) - return true; - std::vector<InetAddress> allIps(ips()); - if (!std::binary_search(allIps.begin(),allIps.end(),ip)) { - if (___removeIp(_dev,ip)) - return true; - } - return false; -} - -std::vector<InetAddress> OSXEthernetTap::ips() const -{ - struct ifaddrs *ifa = (struct ifaddrs *)0; - if (getifaddrs(&ifa)) - return std::vector<InetAddress>(); - - std::vector<InetAddress> r; - - struct ifaddrs *p = ifa; - while (p) { - if ((!strcmp(p->ifa_name,_dev.c_str()))&&(p->ifa_addr)&&(p->ifa_netmask)&&(p->ifa_addr->sa_family == p->ifa_netmask->sa_family)) { - switch(p->ifa_addr->sa_family) { - case AF_INET: { - struct sockaddr_in *sin = (struct sockaddr_in *)p->ifa_addr; - struct sockaddr_in *nm = (struct sockaddr_in *)p->ifa_netmask; - r.push_back(InetAddress(&(sin->sin_addr.s_addr),4,Utils::countBits((uint32_t)nm->sin_addr.s_addr))); - } break; - case AF_INET6: { - struct sockaddr_in6 *sin = (struct sockaddr_in6 *)p->ifa_addr; - struct sockaddr_in6 *nm = (struct sockaddr_in6 *)p->ifa_netmask; - uint32_t b[4]; - memcpy(b,nm->sin6_addr.s6_addr,sizeof(b)); - r.push_back(InetAddress(sin->sin6_addr.s6_addr,16,Utils::countBits(b[0]) + Utils::countBits(b[1]) + Utils::countBits(b[2]) + Utils::countBits(b[3]))); - } break; - } - } - p = p->ifa_next; - } - - if (ifa) - freeifaddrs(ifa); - - std::sort(r.begin(),r.end()); - std::unique(r.begin(),r.end()); - - return r; -} - -void OSXEthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) -{ - char putBuf[4096]; - if ((len <= _mtu)&&(_enabled)) { - to.copyTo(putBuf,6); - from.copyTo(putBuf + 6,6); - *((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType); - memcpy(putBuf + 14,data,len); - len += 14; - int r = pcap_inject(reinterpret_cast<pcap_t *>(_pcap),putBuf,len); - if (r <= 0) { - printf("%s: pcap_inject() failed\n",_dev.c_str()); - return; - } - printf("%s: inject %s -> %s etherType==%u len=%u r==%d\n",_dev.c_str(),from.toString().c_str(),to.toString().c_str(),etherType,len,r); - } -} - -std::string OSXEthernetTap::deviceName() const -{ - return _dev; -} - -void OSXEthernetTap::setFriendlyName(const char *friendlyName) -{ -} - -void OSXEthernetTap::scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed) -{ - std::vector<MulticastGroup> newGroups; - - struct _intl_ifmaddrs *ifmap = (struct _intl_ifmaddrs *)0; - if (!_intl_getifmaddrs(&ifmap)) { - struct _intl_ifmaddrs *p = ifmap; - while (p) { - if (p->ifma_addr->sa_family == AF_LINK) { - struct sockaddr_dl *in = (struct sockaddr_dl *)p->ifma_name; - struct sockaddr_dl *la = (struct sockaddr_dl *)p->ifma_addr; - if ((la->sdl_alen == 6)&&(in->sdl_nlen <= _dev.length())&&(!memcmp(_dev.data(),in->sdl_data,in->sdl_nlen))) - newGroups.push_back(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen,6),0)); - } - p = p->ifma_next; - } - _intl_freeifmaddrs(ifmap); - } - - std::vector<InetAddress> allIps(ips()); - for(std::vector<InetAddress>::iterator ip(allIps.begin());ip!=allIps.end();++ip) - newGroups.push_back(MulticastGroup::deriveMulticastGroupForAddressResolution(*ip)); - - std::sort(newGroups.begin(),newGroups.end()); - std::unique(newGroups.begin(),newGroups.end()); - - for(std::vector<MulticastGroup>::iterator m(newGroups.begin());m!=newGroups.end();++m) { - if (!std::binary_search(_multicastGroups.begin(),_multicastGroups.end(),*m)) - added.push_back(*m); - } - for(std::vector<MulticastGroup>::iterator m(_multicastGroups.begin());m!=_multicastGroups.end();++m) { - if (!std::binary_search(newGroups.begin(),newGroups.end(),*m)) - removed.push_back(*m); - } - - _multicastGroups.swap(newGroups); -} - -static void _pcapHandler(u_char *ptr,const struct pcap_pkthdr *hdr,const u_char *data) -{ - OSXEthernetTap *tap = reinterpret_cast<OSXEthernetTap *>(ptr); - if (hdr->caplen > 14) { - MAC to(data,6); - MAC from(data + 6,6); - if (from == tap->_mac) { - unsigned int etherType = ntohs(((const uint16_t *)data)[6]); - printf("%s: %s -> %s etherType==%u len==%u\n",tap->_dev.c_str(),from.toString().c_str(),to.toString().c_str(),etherType,(unsigned int)hdr->caplen); - // TODO: VLAN support - tap->_handler(tap->_arg,tap->_nwid,from,to,etherType,0,(const void *)(data + 14),hdr->len - 14); - } - } -} - -void OSXEthernetTap::threadMain() - throw() -{ - pcap_loop(reinterpret_cast<pcap_t *>(_pcap),-1,&_pcapHandler,reinterpret_cast<u_char *>(this)); -} - -} // namespace ZeroTier diff --git a/attic/OSXEthernetTap.cpp.utun-work-in-progress b/attic/OSXEthernetTap.cpp.utun-work-in-progress deleted file mode 100644 index f40483e8..00000000 --- a/attic/OSXEthernetTap.cpp.utun-work-in-progress +++ /dev/null @@ -1,831 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ - */ - -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <errno.h> - -#include <unistd.h> -#include <signal.h> - -#include <fcntl.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/ioctl.h> -#include <sys/wait.h> -#include <sys/select.h> -#include <sys/cdefs.h> -#include <sys/uio.h> -#include <sys/param.h> -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <sys/sys_domain.h> -#include <sys/kern_control.h> -#include <net/if_utun.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <net/route.h> -#include <net/if.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> -#include <netinet6/in6_var.h> -#include <netinet/in_var.h> -#include <netinet/icmp6.h> - -// OSX compile fix... in6_var defines this in a struct which namespaces it for C++ ... why?!? -struct prf_ra { - u_char onlink : 1; - u_char autonomous : 1; - u_char reserved : 6; -} prf_ra; - -#include <netinet6/nd6.h> -#include <ifaddrs.h> - -// These are KERNEL_PRIVATE... why? -#ifndef SIOCAUTOCONF_START -#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq) /* accept rtadvd on this interface */ -#endif -#ifndef SIOCAUTOCONF_STOP -#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq) /* stop accepting rtadv for this interface */ -#endif - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -// This source is from: -// http://www.opensource.apple.com/source/Libinfo/Libinfo-406.17/gen.subproj/getifmaddrs.c?txt -// It's here because OSX 10.6 does not have this convenience function. - -#define SALIGN (sizeof(uint32_t) - 1) -#define SA_RLEN(sa) ((sa)->sa_len ? (((sa)->sa_len + SALIGN) & ~SALIGN) : \ -(SALIGN + 1)) -#define MAX_SYSCTL_TRY 5 -#define RTA_MASKS (RTA_GATEWAY | RTA_IFP | RTA_IFA) - -/* FreeBSD uses NET_RT_IFMALIST and RTM_NEWMADDR from <sys/socket.h> */ -/* We can use NET_RT_IFLIST2 and RTM_NEWMADDR2 on Darwin */ -//#define DARWIN_COMPAT - -//#ifdef DARWIN_COMPAT -#define GIM_SYSCTL_MIB NET_RT_IFLIST2 -#define GIM_RTM_ADDR RTM_NEWMADDR2 -//#else -//#define GIM_SYSCTL_MIB NET_RT_IFMALIST -//#define GIM_RTM_ADDR RTM_NEWMADDR -//#endif - -// Not in 10.6 includes so use our own -struct _intl_ifmaddrs { - struct _intl_ifmaddrs *ifma_next; - struct sockaddr *ifma_name; - struct sockaddr *ifma_addr; - struct sockaddr *ifma_lladdr; -}; - -static inline int _intl_getifmaddrs(struct _intl_ifmaddrs **pif) -{ - int icnt = 1; - int dcnt = 0; - int ntry = 0; - size_t len; - size_t needed; - int mib[6]; - int i; - char *buf; - char *data; - char *next; - char *p; - struct ifma_msghdr2 *ifmam; - struct _intl_ifmaddrs *ifa, *ift; - struct rt_msghdr *rtm; - struct sockaddr *sa; - - mib[0] = CTL_NET; - mib[1] = PF_ROUTE; - mib[2] = 0; /* protocol */ - mib[3] = 0; /* wildcard address family */ - mib[4] = GIM_SYSCTL_MIB; - mib[5] = 0; /* no flags */ - do { - if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0) - return (-1); - if ((buf = (char *)malloc(needed)) == NULL) - return (-1); - if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) { - if (errno != ENOMEM || ++ntry >= MAX_SYSCTL_TRY) { - free(buf); - return (-1); - } - free(buf); - buf = NULL; - } - } while (buf == NULL); - - for (next = buf; next < buf + needed; next += rtm->rtm_msglen) { - rtm = (struct rt_msghdr *)(void *)next; - if (rtm->rtm_version != RTM_VERSION) - continue; - switch (rtm->rtm_type) { - case GIM_RTM_ADDR: - ifmam = (struct ifma_msghdr2 *)(void *)rtm; - if ((ifmam->ifmam_addrs & RTA_IFA) == 0) - break; - icnt++; - p = (char *)(ifmam + 1); - for (i = 0; i < RTAX_MAX; i++) { - if ((RTA_MASKS & ifmam->ifmam_addrs & - (1 << i)) == 0) - continue; - sa = (struct sockaddr *)(void *)p; - len = SA_RLEN(sa); - dcnt += len; - p += len; - } - break; - } - } - - data = (char *)malloc(sizeof(struct _intl_ifmaddrs) * icnt + dcnt); - if (data == NULL) { - free(buf); - return (-1); - } - - ifa = (struct _intl_ifmaddrs *)(void *)data; - data += sizeof(struct _intl_ifmaddrs) * icnt; - - memset(ifa, 0, sizeof(struct _intl_ifmaddrs) * icnt); - ift = ifa; - - for (next = buf; next < buf + needed; next += rtm->rtm_msglen) { - rtm = (struct rt_msghdr *)(void *)next; - if (rtm->rtm_version != RTM_VERSION) - continue; - - switch (rtm->rtm_type) { - case GIM_RTM_ADDR: - ifmam = (struct ifma_msghdr2 *)(void *)rtm; - if ((ifmam->ifmam_addrs & RTA_IFA) == 0) - break; - - p = (char *)(ifmam + 1); - for (i = 0; i < RTAX_MAX; i++) { - if ((RTA_MASKS & ifmam->ifmam_addrs & - (1 << i)) == 0) - continue; - sa = (struct sockaddr *)(void *)p; - len = SA_RLEN(sa); - switch (i) { - case RTAX_GATEWAY: - ift->ifma_lladdr = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - case RTAX_IFP: - ift->ifma_name = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - case RTAX_IFA: - ift->ifma_addr = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - default: - data += len; - break; - } - p += len; - } - ift->ifma_next = ift + 1; - ift = ift->ifma_next; - break; - } - } - - free(buf); - - if (ift > ifa) { - ift--; - ift->ifma_next = NULL; - *pif = ifa; - } else { - *pif = NULL; - free(ifa); - } - return (0); -} - -static inline void _intl_freeifmaddrs(struct _intl_ifmaddrs *ifmp) -{ - free(ifmp); -} - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- - -#include <string> -#include <map> -#include <set> -#include <algorithm> - -#include "../node/Constants.hpp" -#include "../node/Utils.hpp" -#include "../node/Mutex.hpp" -#include "../node/Dictionary.hpp" -#include "Arp.hpp" -#include "OSUtils.hpp" -#include "OSXEthernetTap.hpp" - -// ff:ff:ff:ff:ff:ff with no ADI -static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0); - -static inline bool _setIpv6Stuff(const char *ifname,bool performNUD,bool acceptRouterAdverts) -{ - struct in6_ndireq nd; - struct in6_ifreq ifr; - - int s = socket(AF_INET6,SOCK_DGRAM,0); - if (s <= 0) - return false; - - memset(&nd,0,sizeof(nd)); - strncpy(nd.ifname,ifname,sizeof(nd.ifname)); - - if (ioctl(s,SIOCGIFINFO_IN6,&nd)) { - close(s); - return false; - } - - unsigned long oldFlags = (unsigned long)nd.ndi.flags; - - if (performNUD) - nd.ndi.flags |= ND6_IFF_PERFORMNUD; - else nd.ndi.flags &= ~ND6_IFF_PERFORMNUD; - - if (oldFlags != (unsigned long)nd.ndi.flags) { - if (ioctl(s,SIOCSIFINFO_FLAGS,&nd)) { - close(s); - return false; - } - } - - memset(&ifr,0,sizeof(ifr)); - strncpy(ifr.ifr_name,ifname,sizeof(ifr.ifr_name)); - if (ioctl(s,acceptRouterAdverts ? SIOCAUTOCONF_START : SIOCAUTOCONF_STOP,&ifr)) { - close(s); - return false; - } - - close(s); - return true; -} - -// Create an OSX-native utun device (utun# where # is desiredNumber) -// Adapted from public domain utun example code by Jonathan Levin -static int _make_utun(int desiredNumber) -{ - struct sockaddr_ctl sc; - struct ctl_info ctlInfo; - struct ifreq ifr; - - memset(&ctlInfo, 0, sizeof(ctlInfo)); - if (strlcpy(ctlInfo.ctl_name, UTUN_CONTROL_NAME, sizeof(ctlInfo.ctl_name)) >= sizeof(ctlInfo.ctl_name)) { - return -1; - } - - int fd = socket(PF_SYSTEM, SOCK_DGRAM, SYSPROTO_CONTROL); - if (fd == -1) - return -1; - if (ioctl(fd, CTLIOCGINFO, &ctlInfo) == -1) { - close(fd); - return -1; - } - - sc.sc_id = ctlInfo.ctl_id; - sc.sc_len = sizeof(sc); - sc.sc_family = AF_SYSTEM; - sc.ss_sysaddr = AF_SYS_CONTROL; - sc.sc_unit = desiredNumber + 1; - - if (connect(fd, (struct sockaddr *)&sc, sizeof(sc)) == -1) { - close(fd); - return -1; - } - - memset(&ifr,0,sizeof(ifr)); - sprintf(ifr.ifr_name,"utun%d",desiredNumber); - if (ioctl(fd,SIOCGIFFLAGS,(void *)&ifr) < 0) { - printf("SIOCGIFFLAGS failed\n"); - } - ifr.ifr_flags &= ~IFF_POINTOPOINT; - if (ioctl(fd,SIOCSIFFLAGS,(void *)&ifr) < 0) { - printf("clear IFF_POINTOPOINT failed\n"); - } - - return fd; -} - -namespace ZeroTier { - -static long globalTapsRunning = 0; -static Mutex globalTapCreateLock; - -OSXEthernetTap::OSXEthernetTap( - const char *homePath, - const MAC &mac, - unsigned int mtu, - unsigned int metric, - uint64_t nwid, - const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), - void *arg) : - _handler(handler), - _arg(arg), - _arp((Arp *)0), - _nwid(nwid), - _homePath(homePath), - _mtu(mtu), - _metric(metric), - _fd(0), - _utun(false), - _enabled(true) -{ - char devpath[64],ethaddr[64],mtustr[32],metstr[32],nwids[32]; - struct stat stattmp; - - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - - if (mtu > 2800) - throw std::runtime_error("max tap MTU is 2800"); - - Mutex::Lock _gl(globalTapCreateLock); - - // Read remembered previous device name, if any -- we'll try to reuse - Dictionary devmap; - std::string desiredDevice; - { - std::string devmapbuf; - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),devmapbuf)) { - devmap.fromString(devmapbuf); - desiredDevice = devmap.get(nwids,""); - } - } - - if (::stat((_homePath + ZT_PATH_SEPARATOR_S + "tap.kext").c_str(),&stattmp) == 0) { - // Try to init kext if it's there, otherwise revert to utun mode - - if (::stat("/dev/zt0",&stattmp)) { - long kextpid = (long)vfork(); - if (kextpid == 0) { - ::chdir(homePath); - OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); - ::execl("/sbin/kextload","/sbin/kextload","-q","-repository",homePath,"tap.kext",(const char *)0); - ::_exit(-1); - } else if (kextpid > 0) { - int exitcode = -1; - ::waitpid(kextpid,&exitcode,0); - } - ::usleep(500); // give tap device driver time to start up and try again - if (::stat("/dev/zt0",&stattmp)) - _utun = true; - } - - if (!_utun) { - // See if we can re-use the last device we had. - bool recalledDevice = false; - if (desiredDevice.length() > 2) { - Utils::snprintf(devpath,sizeof(devpath),"/dev/%s",desiredDevice.c_str()); - if (stat(devpath,&stattmp) == 0) { - _fd = ::open(devpath,O_RDWR); - if (_fd > 0) { - _dev = desiredDevice; - recalledDevice = true; - } - } - } - - // Open the first unused tap device if we didn't recall a previous one. - if (!recalledDevice) { - for(int i=0;i<64;++i) { - Utils::snprintf(devpath,sizeof(devpath),"/dev/zt%d",i); - if (stat(devpath,&stattmp)) { - _utun = true; - break; - } - _fd = ::open(devpath,O_RDWR); - if (_fd > 0) { - char foo[16]; - Utils::snprintf(foo,sizeof(foo),"zt%d",i); - _dev = foo; - break; - } - } - } - if (_fd <= 0) - _utun = true; - } - } else { - _utun = true; - } - - if (_utun) { - // Use OSX built-in utun device if kext is not available or doesn't work - - int utunNo = 0; - - if ((desiredDevice.length() > 4)&&(desiredDevice.substr(0,4) == "utun")) { - utunNo = Utils::strToInt(desiredDevice.substr(4).c_str()); - if (utunNo >= 0) - _fd = _make_utun(utunNo); - } - - if (_fd <= 0) { - // Start at utun8 to leave lower utuns unused since other stuff might - // want them -- OpenVPN, cjdns, etc. I'm not sure if those are smart - // enough to scan upward like this. - for(utunNo=8;utunNo<=256;++utunNo) { - if ((_fd = _make_utun(utunNo)) > 0) - break; - } - } - - if (_fd <= 0) - throw std::runtime_error("unable to find/load ZeroTier tap driver OR use built-in utun driver in OSX; permission or system problem or too many open devices?"); - - Utils::snprintf(devpath,sizeof(devpath),"utun%d",utunNo); - _dev = devpath; - - // Configure address and bring it up - Utils::snprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::snprintf(metstr,sizeof(metstr),"%u",_metric); - long cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",mtustr,"metric",metstr,"up",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - if (exitcode) { - ::close(_fd); - throw std::runtime_error("ifconfig failure activating utun interface"); - } - } - - } else { - // Use our ZeroTier OSX tun/tap driver for zt# Ethernet tap device - - if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) { - ::close(_fd); - throw std::runtime_error("unable to set flags on file descriptor for TAP device"); - } - - // Configure MAC address and MTU, bring interface up - Utils::snprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::snprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::snprintf(metstr,sizeof(metstr),"%u",_metric); - long cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - if (exitcode) { - ::close(_fd); - throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface"); - } - } - - _setIpv6Stuff(_dev.c_str(),true,false); - } - - // Set close-on-exec so that devices cannot persist if we fork/exec for update - fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC); - - ::pipe(_shutdownSignalPipe); - - ++globalTapsRunning; - - devmap[nwids] = _dev; - OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),devmap.toString()); - - _thread = Thread::start(this); -} - -OSXEthernetTap::~OSXEthernetTap() -{ - Mutex::Lock _gl(globalTapCreateLock); - - ::write(_shutdownSignalPipe[1],(const void *)this,1); // writing a byte causes thread to exit - Thread::join(_thread); - - ::close(_fd); - ::close(_shutdownSignalPipe[0]); - ::close(_shutdownSignalPipe[1]); - - if (_utun) { - delete _arp; - } else { - if (--globalTapsRunning <= 0) { - globalTapsRunning = 0; // sanity check -- should not be possible - - char tmp[16384]; - sprintf(tmp,"%s/%s",_homePath.c_str(),"tap.kext"); - long kextpid = (long)vfork(); - if (kextpid == 0) { - OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); - ::execl("/sbin/kextunload","/sbin/kextunload",tmp,(const char *)0); - ::_exit(-1); - } else if (kextpid > 0) { - int exitcode = -1; - ::waitpid(kextpid,&exitcode,0); - } - } - } -} - -void OSXEthernetTap::setEnabled(bool en) -{ - _enabled = en; - // TODO: interface status change -} - -bool OSXEthernetTap::enabled() const -{ - return _enabled; -} - -static bool ___removeIp(const std::string &_dev,const InetAddress &ip) -{ - long cpid = (long)vfork(); - if (cpid == 0) { - execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"inet",ip.toIpString().c_str(),"-alias",(const char *)0); - _exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } - return false; // never reached, make compiler shut up about return value -} - -bool OSXEthernetTap::addIp(const InetAddress &ip) -{ - if (!ip) - return false; - - std::vector<InetAddress> allIps(ips()); - if (std::binary_search(allIps.begin(),allIps.end(),ip)) - return true; - - // Remove and reconfigure if address is the same but netmask is different - for(std::vector<InetAddress>::iterator i(allIps.begin());i!=allIps.end();++i) { - if ((i->ipsEqual(ip))&&(i->netmaskBits() != ip.netmaskBits())) { - if (___removeIp(_dev,*i)) - break; - } - } - - if (_utun) { - long cpid = (long)vfork(); - if (cpid == 0) { - if (ip.ss_family == AF_INET6) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"inet6",ip.toString().c_str(),"alias",(const char *)0); - } else { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),ip.toString().c_str(),ip.toIpString().c_str(),"alias",(const char *)0); - } - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - - if (exitcode == 0) { - if (ip.ss_family == AF_INET) { - // Add route to network over tun for IPv4 -- otherwise it behaves - // as a simple point to point tunnel instead of a true route. - cpid = (long)vfork(); - if (cpid == 0) { - ::close(STDERR_FILENO); - ::close(STDOUT_FILENO); - ::execl("/sbin/route","/sbin/route","add",ip.network().toString().c_str(),ip.toIpString().c_str(),(const char *)0); - ::exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } - } else return true; - } - } - } else { - long cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),ip.isV4() ? "inet" : "inet6",ip.toString().c_str(),"alias",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } - } - - return false; -} - -bool OSXEthernetTap::removeIp(const InetAddress &ip) -{ - if (!ip) - return true; - std::vector<InetAddress> allIps(ips()); - if (!std::binary_search(allIps.begin(),allIps.end(),ip)) { - if (___removeIp(_dev,ip)) - return true; - } - return false; -} - -std::vector<InetAddress> OSXEthernetTap::ips() const -{ - struct ifaddrs *ifa = (struct ifaddrs *)0; - if (getifaddrs(&ifa)) - return std::vector<InetAddress>(); - - std::vector<InetAddress> r; - - struct ifaddrs *p = ifa; - while (p) { - if ((!strcmp(p->ifa_name,_dev.c_str()))&&(p->ifa_addr)&&(p->ifa_netmask)&&(p->ifa_addr->sa_family == p->ifa_netmask->sa_family)) { - switch(p->ifa_addr->sa_family) { - case AF_INET: { - struct sockaddr_in *sin = (struct sockaddr_in *)p->ifa_addr; - struct sockaddr_in *nm = (struct sockaddr_in *)p->ifa_netmask; - r.push_back(InetAddress(&(sin->sin_addr.s_addr),4,Utils::countBits((uint32_t)nm->sin_addr.s_addr))); - } break; - case AF_INET6: { - struct sockaddr_in6 *sin = (struct sockaddr_in6 *)p->ifa_addr; - struct sockaddr_in6 *nm = (struct sockaddr_in6 *)p->ifa_netmask; - uint32_t b[4]; - memcpy(b,nm->sin6_addr.s6_addr,sizeof(b)); - r.push_back(InetAddress(sin->sin6_addr.s6_addr,16,Utils::countBits(b[0]) + Utils::countBits(b[1]) + Utils::countBits(b[2]) + Utils::countBits(b[3]))); - } break; - } - } - p = p->ifa_next; - } - - if (ifa) - freeifaddrs(ifa); - - std::sort(r.begin(),r.end()); - std::unique(r.begin(),r.end()); - - return r; -} - -void OSXEthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) -{ - char putBuf[4096]; - if ((_fd > 0)&&(len <= _mtu)&&(_enabled)) { - to.copyTo(putBuf,6); - from.copyTo(putBuf + 6,6); - *((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType); - memcpy(putBuf + 14,data,len); - len += 14; - ::write(_fd,putBuf,len); - } -} - -std::string OSXEthernetTap::deviceName() const -{ - return _dev; -} - -void OSXEthernetTap::setFriendlyName(const char *friendlyName) -{ -} - -void OSXEthernetTap::scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed) -{ - std::vector<MulticastGroup> newGroups; - - struct _intl_ifmaddrs *ifmap = (struct _intl_ifmaddrs *)0; - if (!_intl_getifmaddrs(&ifmap)) { - struct _intl_ifmaddrs *p = ifmap; - while (p) { - if (p->ifma_addr->sa_family == AF_LINK) { - struct sockaddr_dl *in = (struct sockaddr_dl *)p->ifma_name; - struct sockaddr_dl *la = (struct sockaddr_dl *)p->ifma_addr; - if ((la->sdl_alen == 6)&&(in->sdl_nlen <= _dev.length())&&(!memcmp(_dev.data(),in->sdl_data,in->sdl_nlen))) - newGroups.push_back(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen,6),0)); - } - p = p->ifma_next; - } - _intl_freeifmaddrs(ifmap); - } - - std::vector<InetAddress> allIps(ips()); - for(std::vector<InetAddress>::iterator ip(allIps.begin());ip!=allIps.end();++ip) - newGroups.push_back(MulticastGroup::deriveMulticastGroupForAddressResolution(*ip)); - - std::sort(newGroups.begin(),newGroups.end()); - std::unique(newGroups.begin(),newGroups.end()); - - for(std::vector<MulticastGroup>::iterator m(newGroups.begin());m!=newGroups.end();++m) { - if (!std::binary_search(_multicastGroups.begin(),_multicastGroups.end(),*m)) - added.push_back(*m); - } - for(std::vector<MulticastGroup>::iterator m(_multicastGroups.begin());m!=_multicastGroups.end();++m) { - if (!std::binary_search(newGroups.begin(),newGroups.end(),*m)) - removed.push_back(*m); - } - - _multicastGroups.swap(newGroups); -} - -void OSXEthernetTap::threadMain() - throw() -{ - fd_set readfds,nullfds; - MAC to,from; - int n,nfds,r; - char getBuf[8194]; - - Thread::sleep(500); - - FD_ZERO(&readfds); - FD_ZERO(&nullfds); - nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1; - - r = 0; - for(;;) { - FD_SET(_shutdownSignalPipe[0],&readfds); - FD_SET(_fd,&readfds); - select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0); - - if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread - break; - - if (FD_ISSET(_fd,&readfds)) { - n = (int)::read(_fd,getBuf + r,sizeof(getBuf) - r); - if (n < 0) { - if ((errno != EINTR)&&(errno != ETIMEDOUT)) - break; - } else { - // Some tap drivers like to send the ethernet frame and the - // payload in two chunks, so handle that by accumulating - // data until we have at least a frame. - r += n; - if (r > 14) { - if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms - r = _mtu + 14; - - if (_enabled) { - to.setTo(getBuf,6); - from.setTo(getBuf + 6,6); - unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); - // TODO: VLAN support - _handler(_arg,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); - } - - r = 0; - } - } - } - } -} - -} // namespace ZeroTier diff --git a/attic/OSXEthernetTap.hpp.pcap-with-bridge-test b/attic/OSXEthernetTap.hpp.pcap-with-bridge-test deleted file mode 100644 index 33f1948c..00000000 --- a/attic/OSXEthernetTap.hpp.pcap-with-bridge-test +++ /dev/null @@ -1,96 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ - */ - -#ifndef ZT_OSXETHERNETTAP_HPP -#define ZT_OSXETHERNETTAP_HPP - -#include <stdio.h> -#include <stdlib.h> - -#include <stdexcept> -#include <string> -#include <vector> - -#include "../node/Constants.hpp" -#include "../node/MAC.hpp" -#include "../node/InetAddress.hpp" -#include "../node/MulticastGroup.hpp" - -#include "Thread.hpp" - -namespace ZeroTier { - -/** - * OSX Ethernet tap using ZeroTier kernel extension zt# devices - */ -class OSXEthernetTap -{ -public: - OSXEthernetTap( - const char *homePath, - const MAC &mac, - unsigned int mtu, - unsigned int metric, - uint64_t nwid, - const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), - void *arg); - - ~OSXEthernetTap(); - - inline void setEnabled(bool en) { _enabled = en; } - inline bool enabled() const { return _enabled; } - bool addIp(const InetAddress &ip); - bool removeIp(const InetAddress &ip); - std::vector<InetAddress> ips() const; - void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len); - std::string deviceName() const; - void setFriendlyName(const char *friendlyName); - void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed); - - void threadMain() - throw(); - - // Private members of OSXEthernetTap have public visibility to be accessable - // from an internal bounce function; don't modify directly. - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); - void *_arg; - void *_pcap; // pcap_t * - uint64_t _nwid; - MAC _mac; - Thread _thread; - std::string _homePath; - std::string _dev; - std::vector<MulticastGroup> _multicastGroups; - unsigned int _mtu; - unsigned int _metric; - volatile bool _enabled; -}; - -} // namespace ZeroTier - -#endif diff --git a/attic/OSXEthernetTap.hpp.utun-work-in-progress b/attic/OSXEthernetTap.hpp.utun-work-in-progress deleted file mode 100644 index 8ece87b3..00000000 --- a/attic/OSXEthernetTap.hpp.utun-work-in-progress +++ /dev/null @@ -1,101 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ - */ - -#ifndef ZT_OSXETHERNETTAP_HPP -#define ZT_OSXETHERNETTAP_HPP - -#include <stdio.h> -#include <stdlib.h> - -#include <stdexcept> -#include <string> -#include <vector> - -#include "../node/Constants.hpp" -#include "../node/MAC.hpp" -#include "../node/InetAddress.hpp" -#include "../node/MulticastGroup.hpp" - -#include "Thread.hpp" - -namespace ZeroTier { - -class Arp; - -/** - * OSX Ethernet tap supporting either ZeroTier tun/tap kext or OSX-native utun - */ -class OSXEthernetTap -{ -public: - OSXEthernetTap( - const char *homePath, - const MAC &mac, - unsigned int mtu, - unsigned int metric, - uint64_t nwid, - const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), - void *arg); - - ~OSXEthernetTap(); - - void setEnabled(bool en); - bool enabled() const; - bool addIp(const InetAddress &ip); - bool removeIp(const InetAddress &ip); - std::vector<InetAddress> ips() const; - void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len); - std::string deviceName() const; - void setFriendlyName(const char *friendlyName); - void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed); - - inline bool isNativeUtun() const { return _utun; } - - void threadMain() - throw(); - -private: - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); - void *_arg; - Arp *_arp; // created and used if utun is enabled - uint64_t _nwid; - Thread _thread; - std::string _homePath; - std::string _dev; - std::vector<MulticastGroup> _multicastGroups; - unsigned int _mtu; - unsigned int _metric; - int _fd; - int _shutdownSignalPipe[2]; - bool _utun; - volatile bool _enabled; -}; - -} // namespace ZeroTier - -#endif diff --git a/attic/README.md b/attic/README.md deleted file mode 100644 index 768bccd4..00000000 --- a/attic/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Retired Code and Miscellaneous Junk -====== - -This directory is for old code that isn't used but we don't want to lose track of, and for anything else random like debug scripts. diff --git a/attic/kubernetes_docs/.zerotierCliSettings b/attic/kubernetes_docs/.zerotierCliSettings deleted file mode 100644 index 0e7df9b6..00000000 --- a/attic/kubernetes_docs/.zerotierCliSettings +++ /dev/null @@ -1,18 +0,0 @@ -{ - "configVersion": 1, - "defaultCentral": "@my.zerotier.com", - "defaultController": "@my.zerotier.com", - "defaultOne": "@local", - "things": { - "local": { - "auth": "local_service_auth_token_replaced_automatically", - "type": "one", - "url": "http://127.0.0.1:9993/" - }, - "my.zerotier.com": { - "auth": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", - "type": "central", - "url": "https://my.zerotier.com/" - } - } -} diff --git a/attic/kubernetes_docs/Dockerfile b/attic/kubernetes_docs/Dockerfile deleted file mode 100644 index 6437a2bb..00000000 --- a/attic/kubernetes_docs/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM node:4.4 -EXPOSE 8080/tcp 9993/udp - -# Install ZT network conf files -RUN mkdir -p /var/lib/zerotier-one/networks.d -ADD *.conf /var/lib/zerotier-one/networks.d/ -ADD *.conf / -ADD zerotier-one / -ADD zerotier-cli / -ADD .zerotierCliSettings / - -# Install App -ADD server.js / - -# script which will start/auth VM on ZT network -ADD entrypoint.sh / -RUN chmod -v +x /entrypoint.sh - -CMD ["./entrypoint.sh"]
\ No newline at end of file diff --git a/attic/kubernetes_docs/README.md b/attic/kubernetes_docs/README.md deleted file mode 100644 index 482e77e5..00000000 --- a/attic/kubernetes_docs/README.md +++ /dev/null @@ -1,150 +0,0 @@ -Kubernetes + ZeroTier -==== - -A self-authorizing Kubernetes cluster deployment over a private ZeroTier network. - -This is a quick tutorial for setting up a Kubernetes deployment which can self-authorize each new replica onto your private ZeroTier network with no additional configuration needed when you scale. The Kubernetes-specific instructions and content is based on the [hellonode](http://kubernetes.io/docs/hellonode/) tutorial. All of the files discussed below can be found [here](); - - - -## Preliminary tasks - -**Step 1: Go to [my.zerotier.com](https://my.zerotier.com) and generate a network controller API key. This key will be used by ZeroTier to automatically authorize new instances of your VMs to join your secure deployment network during replication.** - -**Step 2: Create a new `private` network. Take note of the network ID, henceforth: `nwid`** - -**Step 3: Follow the instructions from the [hellonode](ttp://kubernetes.io/docs/hellonode/) tutorial to set up your development system.** - -*** -## Construct docker image - -**Step 4: Create necessary files for inclusion into image, your resultant directory should contain:** - - - `ztkube/<nwid>.conf` - - `ztkube/Dockerfile` - - `ztkube/entrypoint.sh` - - `ztkube/server.js` - - `ztkube/zerotier-cli` - - `ztkube/zerotier-one` - -Start by creating a build directory to copy all required files into `mkdir ztkube`. Then build the following: - - `make one` - - `make cli` - -Add the following files to the `ztkube` directory. These files will be compiled into the Docker image. - - - Create an empty `<nwid>.conf` file to specify the private deployment network you created in *Step 2*: - - - Create a CLI tool config file `.zerotierCliSettings` which should only contain your network controller API key to authorize new devices on your network (the local service API key will be filled in automatically). In this example the default controller is hosted by us at [my.zerotier.com](https://my.zerotier.com). Alternatively, you can host your own network controller but you'll need to modify the CLI config file accordingly. - -``` -{ - "configVersion": 1, - "defaultCentral": "@my.zerotier.com", - "defaultController": "@my.zerotier.com", - "defaultOne": "@local", - "things": { - "local": { - "auth": "local_service_auth_token_replaced_automatically", - "type": "one", - "url": "http://127.0.0.1:9993/" - }, - "my.zerotier.com": { - "auth": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", - "type": "central", - "url": "https://my.zerotier.com/" - } - } -} -``` - - - - Create a `Dockerfile` which will copy the ZeroTier service as well as the ZeroTier CLI to the image: - -``` -FROM node:4.4 -EXPOSE 8080/tcp 9993/udp - -# Install ZT network conf files -RUN mkdir -p /var/lib/zerotier-one/networks.d -ADD *.conf /var/lib/zerotier-one/networks.d/ -ADD *.conf / -ADD zerotier-one / -ADD zerotier-cli / -ADD .zerotierCliSettings / - -# Install App -ADD server.js / - -# script which will start/auth VM on ZT network -ADD entrypoint.sh / -RUN chmod -v +x /entrypoint.sh - -CMD ["./entrypoint.sh"] -``` - - - Create the `entrypoint.sh` script which will start the ZeroTier service in the VM, attempt to join your deployment network and automatically authorize the new VM if your network is set to private: - -``` -#!/bin/bash - -echo '*** ZeroTier-Kubernetes self-auth test script' -chown -R daemon /var/lib/zerotier-one -chgrp -R daemon /var/lib/zerotier-one -su daemon -s /bin/bash -c '/zerotier-one -d -U -p9993 >>/tmp/zerotier-one.out 2>&1' -dev="" -nwconf=$(ls *.conf) -nwid="${nwconf%.*}" - -sleep 10 -dev=$(cat /var/lib/zerotier-one/identity.public| cut -d ':' -f 1) - -echo '*** Joining' -./zerotier-cli join "$nwid".conf -# Fill out local service auth token -AUTHTOKEN=$(cat /var/lib/zerotier-one/authtoken.secret) -sed "s|\local_service_auth_token_replaced_automatically|${AUTHTOKEN}|" .zerotierCliSettings > /root/.zerotierCliSettings -echo '*** Authorizing' -./zerotier-cli net-auth @my.zerotier.com "$nwid" "$dev" -echo '*** Cleaning up' # Remove controller auth token -rm -rf .zerotierCliSettings /root/.zerotierCliSettings -node server.js -``` - -**Step 5: Build the image:** - - - `docker build -t gcr.io/$PROJECT_ID/hello-node .` - - - -**Step 6: Push the docker image to your *Container Registry*** - - - `gcloud docker push gcr.io/$PROJECT_ID/hello-node:v1` - -*** -## Deploy! - -**Step 7: Create Kubernetes Cluster** - - - `gcloud config set compute/zone us-central1-a` - - - `gcloud container clusters create hello-world` - - - `gcloud container clusters get-credentials hello-world` - - - -**Step 8: Create your pod** - - - `kubectl run hello-node --image=gcr.io/$PROJECT_ID/hello-node:v1 --port=8080` - - - -**Step 9: Scale** - - - `kubectl scale deployment hello-node --replicas=4` - -*** -## Verify - -Now, after a minute or so you can use `zerotier-cli net-members <nwid>` to show all of your VM instances on your ZeroTier deployment network. If you haven't [configured your local CLI](https://github.com/zerotier/ZeroTierOne/tree/dev/cli), you can simply log into [my.zerotier.com](https://my.zerotier.com), go to *Networks -> nwid* to check that your VMs are indeed members of your private network. You should also note that the `entrypoint.sh` script will automatically delete your network controller API key once it has authorized your VM. This is merely a security measure and can be removed if needed. diff --git a/attic/kubernetes_docs/entrypoint.sh b/attic/kubernetes_docs/entrypoint.sh deleted file mode 100644 index 80cd278e..00000000 --- a/attic/kubernetes_docs/entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -echo '*** ZeroTier-Kubernetes self-auth test script' -chown -R daemon /var/lib/zerotier-one -chgrp -R daemon /var/lib/zerotier-one -su daemon -s /bin/bash -c '/zerotier-one -d -U -p9993 >>/tmp/zerotier-one.out 2>&1' -dev="" -nwconf=$(ls *.conf) -nwid="${nwconf%.*}" - -sleep 10 -dev=$(cat /var/lib/zerotier-one/identity.public| cut -d ':' -f 1) - -echo '*** Joining' -./zerotier-cli join "$nwid".conf -# Fill out local service auth token -AUTHTOKEN=$(cat /var/lib/zerotier-one/authtoken.secret) -sed "s|\local_service_auth_token_replaced_automatically|${AUTHTOKEN}|" .zerotierCliSettings > /root/.zerotierCliSettings -echo '*** Authorizing' -./zerotier-cli net-auth @my.zerotier.com "$nwid" "$dev" -echo '*** Cleaning up' # Remove controller auth token -rm -rf .zerotierCliSettings /root/.zerotierCliSettings -node server.js
\ No newline at end of file diff --git a/attic/kubernetes_docs/server.js b/attic/kubernetes_docs/server.js deleted file mode 100644 index a4b08bb8..00000000 --- a/attic/kubernetes_docs/server.js +++ /dev/null @@ -1,8 +0,0 @@ -var http = require('http'); -var handleRequest = function(request, response) { - console.log('Received request for URL: ' + request.url); - response.writeHead(200); - response.end('Hello World!'); -}; -var www = http.createServer(handleRequest); -www.listen(8080); diff --git a/attic/lat_lon_to_xyz.js b/attic/lat_lon_to_xyz.js deleted file mode 100644 index 692a3687..00000000 --- a/attic/lat_lon_to_xyz.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' - -/* This is a utility to convert latitude/longitude into X,Y,Z coordinates as used by clustering. */ - -if (process.argv.length !== 4) { - console.log('Usage: node lat_lon_to_xyz.js <latitude> <longitude'); - process.exit(1); -} - -var lat = parseFloat(process.argv[2])||0.0; -var lon = parseFloat(process.argv[3])||0.0; - -var latRadians = lat * 0.01745329251994; // PI / 180 -var lonRadians = lon * 0.01745329251994; // PI / 180 -var cosLat = Math.cos(latRadians); - -console.log({ - lat: lat, - lon: lon, - x: Math.round((-6371.0) * cosLat * Math.cos(lonRadians)), - y: Math.round(6371.0 * Math.sin(latRadians)), - z: Math.round(6371.0 * cosLat * Math.sin(lonRadians)) -}); - -process.exit(0); diff --git a/attic/world/build.sh b/attic/world/build.sh index b783702c..aeb3b87f 100755 --- a/attic/world/build.sh +++ b/attic/world/build.sh @@ -1 +1,3 @@ -c++ -I.. -o mkworld ../node/C25519.cpp ../node/Salsa20.cpp ../node/SHA512.cpp ../node/Identity.cpp ../node/Utils.cpp ../node/InetAddress.cpp ../osdep/OSUtils.cpp mkworld.cpp +#!/bin/bash + +c++ -std=c++11 -I../.. -I.. -O -o mkworld ../../node/C25519.cpp ../../node/Salsa20.cpp ../../node/SHA512.cpp ../../node/Identity.cpp ../../node/Utils.cpp ../../node/InetAddress.cpp ../../osdep/OSUtils.cpp mkworld.cpp -lm diff --git a/attic/world/mkworld.cpp b/attic/world/mkworld.cpp index 647ddd21..b8cb027b 100644 --- a/attic/world/mkworld.cpp +++ b/attic/world/mkworld.cpp @@ -61,11 +61,11 @@ int main(int argc,char **argv) current = previous; OSUtils::writeFile("previous.c25519",previous); OSUtils::writeFile("current.c25519",current); - fprintf(stderr,"INFO: created initial world keys: previous.c25519 and current.c25519 (both initially the same)"ZT_EOL_S); + fprintf(stderr,"INFO: created initial world keys: previous.c25519 and current.c25519 (both initially the same)" ZT_EOL_S); } if ((previous.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))||(current.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))) { - fprintf(stderr,"FATAL: previous.c25519 or current.c25519 empty or invalid"ZT_EOL_S); + fprintf(stderr,"FATAL: previous.c25519 or current.c25519 empty or invalid" ZT_EOL_S); return 1; } C25519::Pair previousKP; @@ -81,7 +81,12 @@ int main(int argc,char **argv) std::vector<World::Root> roots; const uint64_t id = ZT_WORLD_ID_EARTH; - const uint64_t ts = 1532555817048ULL; // July 25th, 2018 + const uint64_t ts = 1562631342273ULL; // July 8th, 2019 + + roots.push_back(World::Root()); + roots.back().identity = Identity("3a46f1bf30:0:76e66fab33e28549a62ee2064d1843273c2c300ba45c3f20bef02dbad225723bb59a9bb4b13535730961aeecf5a163ace477cceb0727025b99ac14a5166a09a3"); + roots.back().stableEndpoints.push_back(InetAddress("185.180.13.82/9993")); + roots.back().stableEndpoints.push_back(InetAddress("2a02:6ea0:c815::/9993")); // Alice roots.push_back(World::Root()); @@ -118,7 +123,7 @@ int main(int argc,char **argv) // END WORLD DEFINITION // ========================================================================= - fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts); + fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu" ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts); World nw = World::make(World::TYPE_PLANET,id,ts,currentKP.pub,roots,previousKP); @@ -127,15 +132,15 @@ int main(int argc,char **argv) World testw; testw.deserialize(outtmp,0); if (testw != nw) { - fprintf(stderr,"FATAL: serialization test failed!"ZT_EOL_S); + fprintf(stderr,"FATAL: serialization test failed!" ZT_EOL_S); return 1; } OSUtils::writeFile("world.bin",std::string((const char *)outtmp.data(),outtmp.size())); - fprintf(stderr,"INFO: world.bin written with %u bytes of binary world data."ZT_EOL_S,outtmp.size()); + fprintf(stderr,"INFO: world.bin written with %u bytes of binary world data." ZT_EOL_S,outtmp.size()); fprintf(stdout,ZT_EOL_S); - fprintf(stdout,"#define ZT_DEFAULT_WORLD_LENGTH %u"ZT_EOL_S,outtmp.size()); + fprintf(stdout,"#define ZT_DEFAULT_WORLD_LENGTH %u" ZT_EOL_S,outtmp.size()); fprintf(stdout,"static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {"); for(unsigned int i=0;i<outtmp.size();++i) { const unsigned char *d = (const unsigned char *)outtmp.data(); @@ -143,7 +148,7 @@ int main(int argc,char **argv) fprintf(stdout,","); fprintf(stdout,"0x%.2x",(unsigned int)d[i]); } - fprintf(stdout,"};"ZT_EOL_S); + fprintf(stdout,"};" ZT_EOL_S); return 0; } diff --git a/attic/world/old/earth-2015-11-16.bin b/attic/world/old/earth-2015-11-16.bin Binary files differdeleted file mode 100644 index 910ff144..00000000 --- a/attic/world/old/earth-2015-11-16.bin +++ /dev/null diff --git a/attic/world/old/earth-2015-11-20.bin b/attic/world/old/earth-2015-11-20.bin Binary files differdeleted file mode 100644 index 198682e5..00000000 --- a/attic/world/old/earth-2015-11-20.bin +++ /dev/null diff --git a/attic/world/old/earth-2015-12-17.bin b/attic/world/old/earth-2015-12-17.bin Binary files differdeleted file mode 100644 index 20fadb56..00000000 --- a/attic/world/old/earth-2015-12-17.bin +++ /dev/null diff --git a/attic/world/old/earth-2016-01-13.bin b/attic/world/old/earth-2016-01-13.bin Binary files differdeleted file mode 100644 index 5dea4d21..00000000 --- a/attic/world/old/earth-2016-01-13.bin +++ /dev/null diff --git a/attic/world/world.bin b/attic/world/world.bin Binary files differindex bbafb43a..fff7e2a9 100644 --- a/attic/world/world.bin +++ b/attic/world/world.bin diff --git a/attic/world/world.c b/attic/world/world.c index 58ec9f38..ecf30e6f 100644 --- a/attic/world/world.c +++ b/attic/world/world.c @@ -1,3 +1,3 @@ -#define ZT_DEFAULT_WORLD_LENGTH 634 -static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x64,0xd3,0x71,0xf0,0x58,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0xbf,0xfd,0xd5,0x32,0xf7,0x15,0x6e,0x02,0x6f,0xb9,0x01,0x0d,0xb5,0x7b,0x04,0xd8,0x3a,0xc5,0x17,0x39,0x04,0x36,0xfd,0x9d,0xc6,0x3d,0xa8,0xf3,0x8e,0x79,0xe7,0xc8,0x77,0x8d,0xcc,0x79,0xb8,0xab,0xc6,0x98,0x7c,0x9f,0x34,0x25,0x14,0xe1,0x2f,0xd7,0x97,0x11,0xec,0x34,0x4c,0x9f,0x0f,0xb4,0x85,0x0d,0x9b,0x11,0xd1,0xc2,0xce,0x00,0xc4,0x0a,0x13,0x4b,0xcb,0xc3,0xae,0x2e,0x16,0x00,0x4b,0xdc,0x90,0x5e,0x7e,0x9b,0x44,0x07,0x15,0x36,0x61,0x3c,0x64,0xaa,0xe9,0x46,0x78,0x3c,0xa7,0x18,0xc8,0xd8,0x02,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x0c,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x7d,0x00,0x01,0x00,0x00,0x00,0x00,0x27,0x09,0x04,0x9a,0x42,0xc5,0x21,0x27,0x09,0x06,0x2c,0x0f,0xf8,0x50,0x01,0x54,0x01,0x97,0x00,0x33,0xcc,0x08,0xf8,0xfa,0xcc,0x08,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x54,0x60,0x01,0x00,0xfc,0xcc,0x08,0x27,0x09,0x04,0x83,0xff,0x06,0x10,0x27,0x09,0x06,0x28,0x03,0xeb,0x80,0x00,0x00,0x00,0x0e,0x00,0x02,0x60,0x01,0x00,0xfc,0xcc,0x08,0x27,0x09,0x04,0x6b,0xaa,0xc5,0x0e,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x02,0x00,0xe0,0x01,0x08,0xfe,0xcc,0x08,0x27,0x09,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0xb7,0x40,0x01,0x08,0xfe,0xcc,0x08,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x0c,0x04,0x2d,0x20,0xc6,0x82,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x64,0x00,0x81,0xc3,0x54,0x00,0x00,0xff,0xfe,0x18,0x1d,0x61,0x27,0x09,0x04,0x2e,0x65,0xa0,0xf9,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x03,0x00,0xd0,0x00,0x6a,0x30,0x01,0x78,0x00,0xcd,0x08,0x27,0x09,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x68,0x00,0x83,0xa4,0x00,0x64,0xcd,0x08,0x80,0x01,0xcd,0x08,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x26,0x70,0x01,0xfe,0x15,0xc4,0xf5,0x27,0x09}; +#define ZT_DEFAULT_WORLD_LENGTH 732 +static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x6b,0xd4,0x16,0x08,0xc1,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0x16,0x93,0xf4,0xe5,0xbd,0x20,0xda,0x10,0xad,0xc7,0x05,0xf4,0x99,0xfe,0x04,0x08,0x9b,0xe0,0x9e,0x77,0x1d,0x9f,0x47,0x16,0xaa,0x92,0x4f,0x10,0x16,0x3d,0xc7,0xec,0xd3,0x90,0x9e,0xd1,0x74,0xfc,0xb3,0xb5,0x07,0x9c,0x4d,0x95,0xc5,0x17,0x8b,0x3d,0x0b,0x60,0x76,0xe8,0x51,0xbb,0xb6,0x3d,0x74,0xb5,0x21,0x83,0x7b,0x95,0x1d,0x02,0x9b,0xcd,0xaf,0x5c,0x3e,0x96,0xdf,0x37,0x2c,0x56,0x6d,0xfa,0x75,0x0f,0xda,0x55,0x85,0x13,0xf4,0x76,0x1a,0x66,0x4d,0x3b,0x8d,0xcf,0x12,0xc9,0x34,0xb9,0x0d,0x61,0x03,0x3a,0x46,0xf1,0xbf,0x30,0x00,0x76,0xe6,0x6f,0xab,0x33,0xe2,0x85,0x49,0xa6,0x2e,0xe2,0x06,0x4d,0x18,0x43,0x27,0x3c,0x2c,0x30,0x0b,0xa4,0x5c,0x3f,0x20,0xbe,0xf0,0x2d,0xba,0xd2,0x25,0x72,0x3b,0xb5,0x9a,0x9b,0xb4,0xb1,0x35,0x35,0x73,0x09,0x61,0xae,0xec,0xf5,0xa1,0x63,0xac,0xe4,0x77,0xcc,0xeb,0x07,0x27,0x02,0x5b,0x99,0xac,0x14,0xa5,0x16,0x6a,0x09,0xa3,0x00,0x02,0x04,0xb9,0xb4,0x0d,0x52,0x27,0x09,0x06,0x2a,0x02,0x6e,0xa0,0xc8,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x27,0x09,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x0c,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9a,0x42,0xc5,0x21,0x27,0x09,0x06,0x2c,0x0f,0xf8,0x50,0x01,0x54,0x01,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x33,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0x83,0xff,0x06,0x10,0x27,0x09,0x06,0x28,0x03,0xeb,0x80,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x6b,0xaa,0xc5,0x0e,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x00,0x00,0x00,0x00,0x02,0x00,0xe0,0x01,0x27,0x09,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0xb7,0x40,0x01,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x0c,0x04,0x2d,0x20,0xc6,0x82,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x64,0x00,0x81,0xc3,0x54,0x00,0x00,0xff,0xfe,0x18,0x1d,0x61,0x27,0x09,0x04,0x2e,0x65,0xa0,0xf9,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x03,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x6a,0x30,0x01,0x27,0x09,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x68,0x00,0x83,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x64,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09}; diff --git a/controller/DB.cpp b/controller/DB.cpp index 61eed0e9..bb734dc8 100644 --- a/controller/DB.cpp +++ b/controller/DB.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "DB.hpp" @@ -96,8 +104,7 @@ void DB::cleanMember(nlohmann::json &member) member.erase("lastRequestMetaData"); } -DB::DB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) : - _controller(nc), +DB::DB(const Identity &myId,const char *path) : _myId(myId), _myAddress(myId.address()), _path((path) ? path : "") @@ -107,9 +114,7 @@ DB::DB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path _myAddressStr = tmp; } -DB::~DB() -{ -} +DB::~DB() {} bool DB::get(const uint64_t networkId,nlohmann::json &network) { @@ -221,7 +226,7 @@ void DB::networks(std::vector<uint64_t> &networks) networks.push_back(n->first); } -void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool push) +void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool initialized) { uint64_t memberId = 0; uint64_t networkId = 0; @@ -305,8 +310,12 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool pu } } - if (push) - _controller->onNetworkMemberUpdate(networkId,memberId); + if (initialized) { + std::lock_guard<std::mutex> ll(_changeListeners_l); + for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) { + (*i)->onNetworkMemberUpdate(networkId,memberId,memberConfig); + } + } } else if (memberId) { if (nw) { std::lock_guard<std::mutex> l(nw->lock); @@ -324,20 +333,24 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool pu } } - if ((push)&&((wasAuth)&&(!isAuth)&&(networkId)&&(memberId))) - _controller->onNetworkMemberDeauthorize(networkId,memberId); + if ((initialized)&&((wasAuth)&&(!isAuth)&&(networkId)&&(memberId))) { + std::lock_guard<std::mutex> ll(_changeListeners_l); + for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) { + (*i)->onNetworkMemberDeauthorize(networkId,memberId); + } + } } -void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool push) +void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool initialized) { if (networkConfig.is_object()) { const std::string ids = networkConfig["id"]; - const uint64_t id = Utils::hexStrToU64(ids.c_str()); - if (id) { + const uint64_t networkId = Utils::hexStrToU64(ids.c_str()); + if (networkId) { std::shared_ptr<_Network> nw; { std::lock_guard<std::mutex> l(_networks_l); - std::shared_ptr<_Network> &nw2 = _networks[id]; + std::shared_ptr<_Network> &nw2 = _networks[networkId]; if (!nw2) nw2.reset(new _Network); nw = nw2; @@ -346,15 +359,19 @@ void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool std::lock_guard<std::mutex> l2(nw->lock); nw->config = networkConfig; } - if (push) - _controller->onNetworkUpdate(id); + if (initialized) { + std::lock_guard<std::mutex> ll(_changeListeners_l); + for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) { + (*i)->onNetworkUpdate(networkId,networkConfig); + } + } } } else if (old.is_object()) { const std::string ids = old["id"]; - const uint64_t id = Utils::hexStrToU64(ids.c_str()); - if (id) { + const uint64_t networkId = Utils::hexStrToU64(ids.c_str()); + if (networkId) { std::lock_guard<std::mutex> l(_networks_l); - _networks.erase(id); + _networks.erase(networkId); } } } diff --git a/controller/DB.hpp b/controller/DB.hpp index 4b2940cd..f499d387 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CONTROLLER_DB_HPP @@ -32,22 +40,29 @@ #include <unordered_set> #include <vector> #include <atomic> +#include <mutex> #include "../ext/json/json.hpp" -#define ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS 2 - namespace ZeroTier { -class EmbeddedNetworkController; - /** * Base class with common infrastructure for all controller DB implementations */ class DB { public: + class ChangeListener + { + public: + ChangeListener() {} + virtual ~ChangeListener() {} + virtual void onNetworkUpdate(uint64_t networkId,const nlohmann::json &network) {} + virtual void onNetworkMemberUpdate(uint64_t networkId,uint64_t memberId,const nlohmann::json &member) {} + virtual void onNetworkMemberDeauthorize(uint64_t networkId,uint64_t memberId) {} + }; + struct NetworkSummaryInfo { NetworkSummaryInfo() : authorizedMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} @@ -58,27 +73,12 @@ public: int64_t mostRecentDeauthTime; }; - /** - * Ensure that all network fields are present - */ static void initNetwork(nlohmann::json &network); - - /** - * Ensure that all member fields are present - */ static void initMember(nlohmann::json &member); - - /** - * Remove old and temporary network fields - */ static void cleanNetwork(nlohmann::json &network); - - /** - * Remove old and temporary member fields - */ static void cleanMember(nlohmann::json &member); - DB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path); + DB(const Identity &myId,const char *path); virtual ~DB(); virtual bool waitForReady() = 0; @@ -94,19 +94,20 @@ public: bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member); bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info); bool get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members); - bool summary(const uint64_t networkId,NetworkSummaryInfo &info); - void networks(std::vector<uint64_t> &networks); virtual void save(nlohmann::json *orig,nlohmann::json &record) = 0; - virtual void eraseNetwork(const uint64_t networkId) = 0; - virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0; - virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0; + inline void addListener(DB::ChangeListener *const listener) + { + std::lock_guard<std::mutex> l(_changeListeners_l); + _changeListeners.push_back(listener); + } + protected: struct _Network { @@ -120,18 +121,19 @@ protected: std::mutex lock; }; - void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool push); - void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool push); + void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool initialized); + void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool initialized); void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info); - EmbeddedNetworkController *const _controller; const Identity _myId; const Address _myAddress; const std::string _path; std::string _myAddressStr; + std::vector<DB::ChangeListener *> _changeListeners; std::unordered_map< uint64_t,std::shared_ptr<_Network> > _networks; std::unordered_multimap< uint64_t,uint64_t > _networkByMember; + mutable std::mutex _changeListeners_l; mutable std::mutex _networks_l; }; diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 6a4134c6..bf568527 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include <stdint.h> @@ -38,6 +46,11 @@ #include "../version.h" #include "EmbeddedNetworkController.hpp" +#include "LFDB.hpp" +#include "FileDB.hpp" +#ifdef ZT_CONTROLLER_USE_LIBPQ +#include "PostgreSQL.hpp" +#endif #include "../node/Node.hpp" #include "../node/CertificateOfMembership.hpp" @@ -336,14 +349,14 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_IPV6_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; InetAddress ip(OSUtils::jsonString(r["ip"],"::0").c_str()); - ZT_FAST_MEMCPY(rule.v.ipv6.ip,reinterpret_cast<struct sockaddr_in6 *>(&ip)->sin6_addr.s6_addr,16); + memcpy(rule.v.ipv6.ip,reinterpret_cast<struct sockaddr_in6 *>(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast<struct sockaddr_in6 *>(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IPV6_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; InetAddress ip(OSUtils::jsonString(r["ip"],"::0").c_str()); - ZT_FAST_MEMCPY(rule.v.ipv6.ip,reinterpret_cast<struct sockaddr_in6 *>(&ip)->sin6_addr.s6_addr,16); + memcpy(rule.v.ipv6.ip,reinterpret_cast<struct sockaddr_in6 *>(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast<struct sockaddr_in6 *>(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; @@ -456,11 +469,13 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } // anonymous namespace -EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : +EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath, int listenPort, MQConfig *mqc) : _startTime(OSUtils::now()), + _listenPort(listenPort), _node(node), _path(dbPath), - _sender((NetworkController::Sender *)0) + _sender((NetworkController::Sender *)0), + _mqc(mqc) { } @@ -478,12 +493,51 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender) _signingId = signingId; _sender = sender; _signingIdAddressString = signingId.address().toString(tmp); -#ifdef ZT_CONTROLLER_USE_RETHINKDB - if ((_path.length() > 10)&&(_path.substr(0,10) == "rethinkdb:")) - _db.reset(new RethinkDB(this,_signingId,_path.c_str())); - else // else use FileDB after endif + +#ifdef ZT_CONTROLLER_USE_LIBPQ + if ((_path.length() > 9)&&(_path.substr(0,9) == "postgres:")) { + _db.reset(new PostgreSQL(_signingId,_path.substr(9).c_str(), _listenPort, _mqc)); + } else { #endif - _db.reset(new FileDB(this,_signingId,_path.c_str())); + + std::string lfJSON; + OSUtils::readFile((_path + ZT_PATH_SEPARATOR_S ".." ZT_PATH_SEPARATOR_S "local.conf").c_str(),lfJSON); + if (lfJSON.length() > 0) { + nlohmann::json lfConfig(OSUtils::jsonParse(lfJSON)); + nlohmann::json &settings = lfConfig["settings"]; + if (settings.is_object()) { + nlohmann::json &controllerDb = settings["controllerDb"]; + if (controllerDb.is_object()) { + std::string type = controllerDb["type"]; + if (type == "lf") { + std::string lfOwner = controllerDb["owner"]; + std::string lfHost = controllerDb["host"]; + int lfPort = controllerDb["port"]; + bool storeOnlineState = controllerDb["storeOnlineState"]; + if ((lfOwner.length())&&(lfHost.length())&&(lfPort > 0)&&(lfPort < 65536)) { + std::size_t pubHdrLoc = lfOwner.find("Public: "); + if ((pubHdrLoc > 0)&&((pubHdrLoc + 8) < lfOwner.length())) { + std::string lfOwnerPublic = lfOwner.substr(pubHdrLoc + 8); + std::size_t pubHdrEnd = lfOwnerPublic.find_first_of("\n\r\t "); + if (pubHdrEnd != std::string::npos) { + lfOwnerPublic = lfOwnerPublic.substr(0,pubHdrEnd); + _db.reset(new LFDB(_signingId,_path.c_str(),lfOwner.c_str(),lfOwnerPublic.c_str(),lfHost.c_str(),lfPort,storeOnlineState)); + } + } + } + } + } + } + } + if (!_db) + _db.reset(new FileDB(_signingId,_path.c_str())); + + _db->addListener(this); + +#ifdef ZT_CONTROLLER_USE_LIBPQ + } +#endif + _db->waitForReady(); } @@ -1043,6 +1097,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( json network,member; _db->get(nwid,network,address,member); + _db->eraseMember(nwid, address); { std::lock_guard<std::mutex> l(_memberStatus_l); @@ -1135,7 +1190,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } -void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId) +void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId,const nlohmann::json &network) { // Send an update to all members of the network that are online const int64_t now = OSUtils::now(); @@ -1146,7 +1201,7 @@ void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId) } } -void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId) +void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId,const nlohmann::json &member) { // Push update to member if online try { @@ -1511,13 +1566,13 @@ void EmbeddedNetworkController::_request( const std::string ips = ipAssignments[i]; InetAddress ip(ips.c_str()); - // IP assignments are only pushed if there is a corresponding local route. We also now get the netmask bits from - // this route, ignoring the netmask bits field of the assigned IP itself. Using that was worthless and a source - // of user error / poor UX. int routedNetmaskBits = -1; for(unsigned int rk=0;rk<nc->routeCount;++rk) { - if ( (!nc->routes[rk].via.ss_family) && (reinterpret_cast<const InetAddress *>(&(nc->routes[rk].target))->containsAddress(ip)) ) - routedNetmaskBits = reinterpret_cast<const InetAddress *>(&(nc->routes[rk].target))->netmaskBits(); + if (reinterpret_cast<const InetAddress *>(&(nc->routes[rk].target))->containsAddress(ip)) { + const int nb = (int)(reinterpret_cast<const InetAddress *>(&(nc->routes[rk].target))->netmaskBits()); + if (nb > routedNetmaskBits) + routedNetmaskBits = nb; + } } if (routedNetmaskBits >= 0) { @@ -1544,8 +1599,8 @@ void EmbeddedNetworkController::_request( InetAddress ipRangeEnd(OSUtils::jsonString(pool["ipRangeEnd"],"").c_str()); if ( (ipRangeStart.ss_family == AF_INET6) && (ipRangeEnd.ss_family == AF_INET6) ) { uint64_t s[2],e[2],x[2],xx[2]; - ZT_FAST_MEMCPY(s,ipRangeStart.rawIpData(),16); - ZT_FAST_MEMCPY(e,ipRangeEnd.rawIpData(),16); + memcpy(s,ipRangeStart.rawIpData(),16); + memcpy(e,ipRangeEnd.rawIpData(),16); s[0] = Utils::ntoh(s[0]); s[1] = Utils::ntoh(s[1]); e[0] = Utils::ntoh(e[0]); @@ -1609,18 +1664,20 @@ void EmbeddedNetworkController::_request( if ( (ipRangeStartIA.ss_family == AF_INET) && (ipRangeEndIA.ss_family == AF_INET) ) { uint32_t ipRangeStart = Utils::ntoh((uint32_t)(reinterpret_cast<struct sockaddr_in *>(&ipRangeStartIA)->sin_addr.s_addr)); uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast<struct sockaddr_in *>(&ipRangeEndIA)->sin_addr.s_addr)); + if ((ipRangeEnd < ipRangeStart)||(ipRangeStart == 0)) continue; uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; - + // Start with the LSB of the member's address uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); for(uint32_t k=ipRangeStart,trialCount=0;((k<=ipRangeEnd)&&(trialCount < 1000));++k,++trialCount) { uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; ++ipTrialCounter; - if ((ip & 0x000000ff) == 0x000000ff) + if ((ip & 0x000000ff) == 0x000000ff) { continue; // don't allow addresses that end in .255 + } // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = -1; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 417005a4..7bc37be2 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SQLITENETWORKCONTROLLER_HPP @@ -43,23 +51,21 @@ #include "../ext/json/json.hpp" #include "DB.hpp" -#include "FileDB.hpp" -#ifdef ZT_CONTROLLER_USE_RETHINKDB -#include "RethinkDB.hpp" -#endif namespace ZeroTier { class Node; -class EmbeddedNetworkController : public NetworkController +struct MQConfig; + +class EmbeddedNetworkController : public NetworkController,public DB::ChangeListener { public: /** * @param node Parent node * @param dbPath Database path (file path or database credentials) */ - EmbeddedNetworkController(Node *node,const char *dbPath); + EmbeddedNetworkController(Node *node,const char *dbPath, int listenPort, MQConfig *mqc = NULL); virtual ~EmbeddedNetworkController(); virtual void init(const Identity &signingId,Sender *sender); @@ -95,10 +101,9 @@ public: void handleRemoteTrace(const ZT_RemoteTrace &rt); - // Called on update via POST or by JSONDB on external update of network or network member records - void onNetworkUpdate(const uint64_t networkId); - void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId); - void onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId); + virtual void onNetworkUpdate(const uint64_t networkId,const nlohmann::json &network); + virtual void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId,const nlohmann::json &member); + virtual void onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId); private: void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData); @@ -141,17 +146,23 @@ private: }; const int64_t _startTime; + int _listenPort; Node *const _node; std::string _path; Identity _signingId; std::string _signingIdAddressString; NetworkController::Sender *_sender; + std::unique_ptr<DB> _db; BlockingQueue< _RQEntry * > _queue; + std::vector<std::thread> _threads; std::mutex _threads_l; + std::unordered_map< _MemberStatusKey,_MemberStatus,_MemberStatusHash > _memberStatus; std::mutex _memberStatus_l; + + MQConfig *_mqc; }; } // namespace ZeroTier diff --git a/controller/FileDB.cpp b/controller/FileDB.cpp index 8cbd60ce..7b997c49 100644 --- a/controller/FileDB.cpp +++ b/controller/FileDB.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "FileDB.hpp" @@ -21,10 +29,12 @@ namespace ZeroTier { -FileDB::FileDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) : - DB(nc,myId,path), +FileDB::FileDB(const Identity &myId,const char *path) : + DB(myId,path), _networksPath(_path + ZT_PATH_SEPARATOR_S + "network"), - _tracePath(_path + ZT_PATH_SEPARATOR_S + "trace") + _tracePath(_path + ZT_PATH_SEPARATOR_S + "trace"), + _onlineChanged(false), + _running(true) { OSUtils::mkdir(_path.c_str()); OSUtils::lockDownFile(_path.c_str(),true); @@ -61,9 +71,65 @@ FileDB::FileDB(EmbeddedNetworkController *const nc,const Identity &myId,const ch } catch ( ... ) {} } } + + _onlineUpdateThread = std::thread([this]() { + unsigned int cnt = 0; + while (this->_running) { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + if ((++cnt % 20) == 0) { // 5 seconds + std::lock_guard<std::mutex> l(this->_online_l); + if (!this->_running) return; + if (this->_onlineChanged) { + char p[4096],atmp[64]; + for(auto nw=this->_online.begin();nw!=this->_online.end();++nw) { + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx-online.json",_networksPath.c_str(),(unsigned long long)nw->first); + FILE *f = fopen(p,"wb"); + if (f) { + fprintf(f,"{"); + const char *memberPrefix = ""; + for(auto m=nw->second.begin();m!=nw->second.end();++m) { + fprintf(f,"%s\"%.10llx\":{" ZT_EOL_S,memberPrefix,(unsigned long long)m->first); + memberPrefix = ","; + InetAddress lastAddr; + const char *timestampPrefix = " "; + int cnt = 0; + for(auto ts=m->second.rbegin();ts!=m->second.rend();) { + if (cnt < 25) { + if (lastAddr != ts->second) { + lastAddr = ts->second; + fprintf(f,"%s\"%lld\":\"%s\"" ZT_EOL_S,timestampPrefix,(long long)ts->first,ts->second.toString(atmp)); + timestampPrefix = ","; + ++cnt; + ++ts; + } else { + ts = std::map<int64_t,InetAddress>::reverse_iterator(m->second.erase(std::next(ts).base())); + } + } else { + ts = std::map<int64_t,InetAddress>::reverse_iterator(m->second.erase(std::next(ts).base())); + } + } + fprintf(f,"}"); + } + fprintf(f,"}" ZT_EOL_S); + fclose(f); + } + } + this->_onlineChanged = false; + } + } + } + }); } -FileDB::~FileDB() {} +FileDB::~FileDB() +{ + try { + _online_l.lock(); + _running = false; + _online_l.unlock(); + _onlineUpdateThread.join(); + } catch ( ... ) {} +} bool FileDB::waitForReady() { return true; } bool FileDB::isReady() { return true; } @@ -86,14 +152,10 @@ void FileDB::save(nlohmann::json *orig,nlohmann::json &record) if (nwid) { nlohmann::json old; get(nwid,old); - if ((!old.is_object())||(old != record)) { - OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json.new",_networksPath.c_str(),nwid); - OSUtils::ztsnprintf(p2,sizeof(p2),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),nwid); + OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),nwid); if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1); - OSUtils::rename(p1,p2); - _networkChanged(old,record,true); } } @@ -103,10 +165,9 @@ void FileDB::save(nlohmann::json *orig,nlohmann::json &record) if ((id)&&(nwid)) { nlohmann::json network,old; get(nwid,network,id,old); - if ((!old.is_object())||(old != record)) { OSUtils::ztsnprintf(pb,sizeof(pb),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member",_networksPath.c_str(),(unsigned long long)nwid); - OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.10llx.json.new",pb,(unsigned long long)id); + OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.10llx.json",pb,(unsigned long long)id); if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) { OSUtils::ztsnprintf(p2,sizeof(p2),"%s" ZT_PATH_SEPARATOR_S "%.16llx",_networksPath.c_str(),(unsigned long long)nwid); OSUtils::mkdir(p2); @@ -114,9 +175,6 @@ void FileDB::save(nlohmann::json *orig,nlohmann::json &record) if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1); } - OSUtils::ztsnprintf(p2,sizeof(p2),"%s" ZT_PATH_SEPARATOR_S "%.10llx.json",pb,(unsigned long long)id); - OSUtils::rename(p1,p2); - _memberChanged(old,record,true); } } @@ -137,16 +195,38 @@ void FileDB::eraseNetwork(const uint64_t networkId) char p[16384]; OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),networkId); OSUtils::rm(p); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx-online.json",_networksPath.c_str(),networkId); + OSUtils::rm(p); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member",_networksPath.c_str(),(unsigned long long)networkId); + OSUtils::rmDashRf(p); _networkChanged(network,nullJson,true); + std::lock_guard<std::mutex> l(this->_online_l); + this->_online.erase(networkId); + this->_onlineChanged = true; } void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId) { + nlohmann::json network,member,nullJson; + get(networkId,network); + get(memberId,member); + char p[4096]; + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member" ZT_PATH_SEPARATOR_S "%.10llx.json",_networksPath.c_str(),networkId,memberId); + OSUtils::rm(p); + _memberChanged(member,nullJson,true); + std::lock_guard<std::mutex> l(this->_online_l); + this->_online[networkId].erase(memberId); + this->_onlineChanged = true; } void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) { - // Nothing to do here right now in the filesystem store mode since we can just get this from the peer list + char mid[32],atmp[64]; + OSUtils::ztsnprintf(mid,sizeof(mid),"%.10llx",(unsigned long long)memberId); + physicalAddress.toString(atmp); + std::lock_guard<std::mutex> l(this->_online_l); + this->_online[networkId][memberId][OSUtils::now()] = physicalAddress; + this->_onlineChanged = true; } } // namespace ZeroTier diff --git a/controller/FileDB.hpp b/controller/FileDB.hpp index 1a3c12e9..5d55d0a4 100644 --- a/controller/FileDB.hpp +++ b/controller/FileDB.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CONTROLLER_FILEDB_HPP @@ -27,7 +35,7 @@ namespace ZeroTier class FileDB : public DB { public: - FileDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path); + FileDB(const Identity &myId,const char *path); virtual ~FileDB(); virtual bool waitForReady(); @@ -40,6 +48,11 @@ public: protected: std::string _networksPath; std::string _tracePath; + std::thread _onlineUpdateThread; + std::map< uint64_t,std::map<uint64_t,std::map<int64_t,InetAddress> > > _online; + std::mutex _online_l; + bool _onlineChanged; + bool _running; }; } // namespace ZeroTier diff --git a/controller/LFDB.cpp b/controller/LFDB.cpp new file mode 100644 index 00000000..f0c8ebfb --- /dev/null +++ b/controller/LFDB.cpp @@ -0,0 +1,400 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#include "LFDB.hpp" + +#include <thread> +#include <chrono> +#include <iostream> +#include <sstream> + +#include "../osdep/OSUtils.hpp" +#include "../ext/cpp-httplib/httplib.h" + +namespace ZeroTier +{ + +LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,const char *lfOwnerPublic,const char *lfNodeHost,int lfNodePort,bool storeOnlineState) : + DB(myId,path), + _myId(myId), + _lfOwnerPrivate((lfOwnerPrivate) ? lfOwnerPrivate : ""), + _lfOwnerPublic((lfOwnerPublic) ? lfOwnerPublic : ""), + _lfNodeHost((lfNodeHost) ? lfNodeHost : "127.0.0.1"), + _lfNodePort(((lfNodePort > 0)&&(lfNodePort < 65536)) ? lfNodePort : 9980), + _running(true), + _ready(false), + _storeOnlineState(storeOnlineState) +{ + _syncThread = std::thread([this]() { + char controllerAddress[24]; + const uint64_t controllerAddressInt = _myId.address().toInt(); + _myId.address().toString(controllerAddress); + std::string networksSelectorName("com.zerotier.controller.lfdb:"); networksSelectorName.append(controllerAddress); networksSelectorName.append("/network"); + std::string membersSelectorName("com.zerotier.controller.lfdb:"); membersSelectorName.append(controllerAddress); membersSelectorName.append("/member"); + + // LF record masking key is the first 32 bytes of SHA512(controller private key) in hex, + // hiding record values from anything but the controller or someone who has its key. + uint8_t sha512pk[64]; + _myId.sha512PrivateKey(sha512pk); + char maskingKey [128]; + Utils::hex(sha512pk,32,maskingKey); + + httplib::Client htcli(_lfNodeHost.c_str(),_lfNodePort,600); + int64_t timeRangeStart = 0; + while (_running) { + { + std::lock_guard<std::mutex> sl(_state_l); + for(auto ns=_state.begin();ns!=_state.end();++ns) { + if (ns->second.dirty) { + nlohmann::json network; + if (get(ns->first,network)) { + nlohmann::json newrec,selector0; + selector0["Name"] = networksSelectorName; + selector0["Ordinal"] = ns->first; + newrec["Selectors"].push_back(selector0); + newrec["Value"] = network.dump(); + newrec["OwnerPrivate"] = _lfOwnerPrivate; + newrec["MaskingKey"] = maskingKey; + newrec["PulseIfUnchanged"] = true; + auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json"); + if (resp) { + if (resp->status == 200) { + ns->second.dirty = false; + printf("SET network %.16llx %s\n",ns->first,resp->body.c_str()); + } else { + fprintf(stderr,"ERROR: LFDB: %d from node (create/update network): %s" ZT_EOL_S,resp->status,resp->body.c_str()); + } + } else { + fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S); + } + } + } + + for(auto ms=ns->second.members.begin();ms!=ns->second.members.end();++ms) { + if ((_storeOnlineState)&&(ms->second.lastOnlineDirty)&&(ms->second.lastOnlineAddress)) { + nlohmann::json newrec,selector0,selector1,selectors,ip; + char tmp[1024],tmp2[128]; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"com.zerotier.controller.lfdb:%s/network/%.16llx/online",controllerAddress,(unsigned long long)ns->first); + ms->second.lastOnlineAddress.toIpString(tmp2); + selector0["Name"] = tmp; + selector0["Ordinal"] = ms->first; + selector1["Name"] = tmp2; + selector1["Ordinal"] = 0; + selectors.push_back(selector0); + selectors.push_back(selector1); + newrec["Selectors"] = selectors; + const uint8_t *const rawip = (const uint8_t *)ms->second.lastOnlineAddress.rawIpData(); + switch(ms->second.lastOnlineAddress.ss_family) { + case AF_INET: + for(int j=0;j<4;++j) + ip.push_back((unsigned int)rawip[j]); + break; + case AF_INET6: + for(int j=0;j<16;++j) + ip.push_back((unsigned int)rawip[j]); + break; + default: + ip = tmp2; // should never happen since only IP transport is currently supported + break; + } + newrec["Value"] = ip; + newrec["OwnerPrivate"] = _lfOwnerPrivate; + newrec["MaskingKey"] = maskingKey; + newrec["Timestamp"] = ms->second.lastOnlineTime; + newrec["PulseIfUnchanged"] = true; + auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json"); + if (resp) { + if (resp->status == 200) { + ms->second.lastOnlineDirty = false; + printf("SET member online %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str()); + } else { + fprintf(stderr,"ERROR: LFDB: %d from node (create/update member online status): %s" ZT_EOL_S,resp->status,resp->body.c_str()); + } + } else { + fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S); + } + } + + if (ms->second.dirty) { + nlohmann::json network,member; + if (get(ns->first,network,ms->first,member)) { + nlohmann::json newrec,selector0,selector1,selectors; + selector0["Name"] = networksSelectorName; + selector0["Ordinal"] = ns->first; + selector1["Name"] = membersSelectorName; + selector1["Ordinal"] = ms->first; + selectors.push_back(selector0); + selectors.push_back(selector1); + newrec["Selectors"] = selectors; + newrec["Value"] = member.dump(); + newrec["OwnerPrivate"] = _lfOwnerPrivate; + newrec["MaskingKey"] = maskingKey; + newrec["PulseIfUnchanged"] = true; + auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json"); + if (resp) { + if (resp->status == 200) { + ms->second.dirty = false; + printf("SET member %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str()); + } else { + fprintf(stderr,"ERROR: LFDB: %d from node (create/update member): %s" ZT_EOL_S,resp->status,resp->body.c_str()); + } + } else { + fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S); + } + } + } + } + } + } + + { + std::ostringstream query; + query + << '{' + << "\"Ranges\":[{" + << "\"Name\":\"" << networksSelectorName << "\"," + << "\"Range\":[0,18446744073709551615]" + << "}]," + << "\"TimeRange\":[" << timeRangeStart << ",18446744073709551615]," + << "\"MaskingKey\":\"" << maskingKey << "\"," + << "\"Owners\":[\"" << _lfOwnerPublic << "\"]" + << '}'; + auto resp = htcli.Post("/query",query.str(),"application/json"); + if (resp) { + if (resp->status == 200) { + nlohmann::json results(OSUtils::jsonParse(resp->body)); + if ((results.is_array())&&(results.size() > 0)) { + for(std::size_t ri=0;ri<results.size();++ri) { + nlohmann::json &rset = results[ri]; + if ((rset.is_array())&&(rset.size() > 0)) { + nlohmann::json &result = rset[0]; + if (result.is_object()) { + nlohmann::json &record = result["Record"]; + if (record.is_object()) { + const std::string recordValue = result["Value"]; + printf("GET network %s\n",recordValue.c_str()); + nlohmann::json network(OSUtils::jsonParse(recordValue)); + if (network.is_object()) { + const std::string idstr = network["id"]; + const uint64_t id = Utils::hexStrToU64(idstr.c_str()); + if ((id >> 24) == controllerAddressInt) { // sanity check + + std::lock_guard<std::mutex> sl(_state_l); + _NetworkState &ns = _state[id]; + if (!ns.dirty) { + nlohmann::json oldNetwork; + if (get(id,oldNetwork)) { + const uint64_t revision = network["revision"]; + const uint64_t prevRevision = oldNetwork["revision"]; + if (prevRevision < revision) { + _networkChanged(oldNetwork,network,timeRangeStart > 0); + } + } else { + nlohmann::json nullJson; + _networkChanged(nullJson,network,timeRangeStart > 0); + } + } + + } + } + } + } + } + } + } + } else { + fprintf(stderr,"ERROR: LFDB: %d from node: %s" ZT_EOL_S,resp->status,resp->body.c_str()); + } + } else { + fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S); + } + } + + { + std::ostringstream query; + query + << '{' + << "\"Ranges\":[{" + << "\"Name\":\"" << networksSelectorName << "\"," + << "\"Range\":[0,18446744073709551615]" + << "},{" + << "\"Name\":\"" << membersSelectorName << "\"," + << "\"Range\":[0,18446744073709551615]" + << "}]," + << "\"TimeRange\":[" << timeRangeStart << ",18446744073709551615]," + << "\"MaskingKey\":\"" << maskingKey << "\"," + << "\"Owners\":[\"" << _lfOwnerPublic << "\"]" + << '}'; + auto resp = htcli.Post("/query",query.str(),"application/json"); + if (resp) { + if (resp->status == 200) { + nlohmann::json results(OSUtils::jsonParse(resp->body)); + if ((results.is_array())&&(results.size() > 0)) { + for(std::size_t ri=0;ri<results.size();++ri) { + nlohmann::json &rset = results[ri]; + if ((rset.is_array())&&(rset.size() > 0)) { + nlohmann::json &result = rset[0]; + if (result.is_object()) { + nlohmann::json &record = result["Record"]; + if (record.is_object()) { + const std::string recordValue = result["Value"]; + printf("GET member %s\n",recordValue.c_str()); + nlohmann::json member(OSUtils::jsonParse(recordValue)); + if (member.is_object()) { + const std::string nwidstr = member["nwid"]; + const std::string idstr = member["id"]; + const uint64_t nwid = Utils::hexStrToU64(nwidstr.c_str()); + const uint64_t id = Utils::hexStrToU64(idstr.c_str()); + if ((id)&&((nwid >> 24) == controllerAddressInt)) { // sanity check + + std::lock_guard<std::mutex> sl(_state_l); + auto ns = _state.find(nwid); + if ((ns == _state.end())||(!ns->second.members[id].dirty)) { + nlohmann::json network,oldMember; + if (get(nwid,network,id,oldMember)) { + const uint64_t revision = member["revision"]; + const uint64_t prevRevision = oldMember["revision"]; + if (prevRevision < revision) + _memberChanged(oldMember,member,timeRangeStart > 0); + } + } else { + nlohmann::json nullJson; + _memberChanged(nullJson,member,timeRangeStart > 0); + } + + } + } + } + } + } + } + } + } else { + fprintf(stderr,"ERROR: LFDB: %d from node: %s" ZT_EOL_S,resp->status,resp->body.c_str()); + } + } else { + fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S); + } + } + + timeRangeStart = time(nullptr) - 120; // start next query 2m before now to avoid losing updates + _ready = true; + + for(int k=0;k<20;++k) { // 2s delay between queries for remotely modified networks or members + if (!_running) + return; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + }); +} + +LFDB::~LFDB() +{ + _running = false; + _syncThread.join(); +} + +bool LFDB::waitForReady() +{ + while (!_ready) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return true; +} + +bool LFDB::isReady() +{ + return (_ready); +} + +void LFDB::save(nlohmann::json *orig,nlohmann::json &record) +{ + if (orig) { + if (*orig != record) { + record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1; + } + } else { + record["revision"] = 1; + } + + const std::string objtype = record["objtype"]; + if (objtype == "network") { + const uint64_t nwid = OSUtils::jsonIntHex(record["id"],0ULL); + if (nwid) { + nlohmann::json old; + get(nwid,old); + if ((!old.is_object())||(old != record)) { + _networkChanged(old,record,true); + { + std::lock_guard<std::mutex> l(_state_l); + _state[nwid].dirty = true; + } + } + } + } else if (objtype == "member") { + const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"],0ULL); + const uint64_t id = OSUtils::jsonIntHex(record["id"],0ULL); + if ((id)&&(nwid)) { + nlohmann::json network,old; + get(nwid,network,id,old); + if ((!old.is_object())||(old != record)) { + _memberChanged(old,record,true); + { + std::lock_guard<std::mutex> l(_state_l); + _state[nwid].members[id].dirty = true; + } + } + } + } +} + +void LFDB::eraseNetwork(const uint64_t networkId) +{ + // TODO +} + +void LFDB::eraseMember(const uint64_t networkId,const uint64_t memberId) +{ + // TODO +} + +void LFDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) +{ + std::lock_guard<std::mutex> l(_state_l); + auto nw = _state.find(networkId); + if (nw != _state.end()) { + auto m = nw->second.members.find(memberId); + if (m != nw->second.members.end()) { + m->second.lastOnlineTime = OSUtils::now(); + if (physicalAddress) + m->second.lastOnlineAddress = physicalAddress; + m->second.lastOnlineDirty = true; + } + } +} + +} // namespace ZeroTier diff --git a/controller/LFDB.hpp b/controller/LFDB.hpp new file mode 100644 index 00000000..73187462 --- /dev/null +++ b/controller/LFDB.hpp @@ -0,0 +1,102 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_CONTROLLER_LFDB_HPP +#define ZT_CONTROLLER_LFDB_HPP + +#include "DB.hpp" + +#include <mutex> +#include <string> +#include <unordered_map> +#include <atomic> + +namespace ZeroTier { + +/** + * DB implementation for controller that stores data in LF + */ +class LFDB : public DB +{ +public: + /** + * @param myId Identity of controller node (with secret) + * @param path Base path for ZeroTier node itself + * @param lfOwnerPrivate LF owner private in PEM format + * @param lfOwnerPublic LF owner public in @base62 format + * @param lfNodeHost LF node host + * @param lfNodePort LF node http (not https) port + * @param storeOnlineState If true, store online/offline state and IP info in LF (a lot of data, only for private networks!) + */ + LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,const char *lfOwnerPublic,const char *lfNodeHost,int lfNodePort,bool storeOnlineState); + virtual ~LFDB(); + + virtual bool waitForReady(); + virtual bool isReady(); + virtual void save(nlohmann::json *orig,nlohmann::json &record); + virtual void eraseNetwork(const uint64_t networkId); + virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress); + +protected: + const Identity _myId; + + std::string _lfOwnerPrivate,_lfOwnerPublic; + std::string _lfNodeHost; + int _lfNodePort; + + struct _MemberState + { + _MemberState() : + lastOnlineAddress(), + lastOnlineTime(0), + dirty(false), + lastOnlineDirty(false) {} + InetAddress lastOnlineAddress; + int64_t lastOnlineTime; + bool dirty; + bool lastOnlineDirty; + }; + struct _NetworkState + { + _NetworkState() : + members(), + dirty(false) {} + std::unordered_map<uint64_t,_MemberState> members; + bool dirty; + }; + std::unordered_map<uint64_t,_NetworkState> _state; + std::mutex _state_l; + + std::atomic_bool _running; + std::atomic_bool _ready; + std::thread _syncThread; + bool _storeOnlineState; +}; + +} // namespace ZeroTier + +#endif diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp new file mode 100644 index 00000000..c6b9ada4 --- /dev/null +++ b/controller/PostgreSQL.cpp @@ -0,0 +1,1571 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#include "PostgreSQL.hpp" +#include "EmbeddedNetworkController.hpp" +#include "RabbitMQ.hpp" +#include "../version.h" + +#include <libpq-fe.h> +#include <sstream> +#include <amqp.h> +#include <amqp_tcp_socket.h> + +using json = nlohmann::json; +namespace { + +static const int DB_MINIMUM_VERSION = 5; + +static const char *_timestr() +{ + time_t t = time(0); + char *ts = ctime(&t); + char *p = ts; + if (!p) + return ""; + while (*p) { + if (*p == '\n') { + *p = (char)0; + break; + } + ++p; + } + return ts; +} + +std::string join(const std::vector<std::string> &elements, const char * const separator) +{ + switch(elements.size()) { + case 0: + return ""; + case 1: + return elements[0]; + default: + std::ostringstream os; + std::copy(elements.begin(), elements.end()-1, std::ostream_iterator<std::string>(os, separator)); + os << *elements.rbegin(); + return os.str(); + } +} + +} + +using namespace ZeroTier; + +PostgreSQL::PostgreSQL(const Identity &myId, const char *path, int listenPort, MQConfig *mqc) + : DB(myId, path) + , _ready(0) + , _connected(1) + , _run(1) + , _waitNoticePrinted(false) + , _listenPort(listenPort) + , _mqc(mqc) +{ + _connString = std::string(path) + " application_name=controller_" +_myAddressStr; + + // Database Schema Version Check + PGconn *conn = getPgConn(); + if (PQstatus(conn) != CONNECTION_OK) { + fprintf(stderr, "Bad Database Connection: %s", PQerrorMessage(conn)); + exit(1); + } + + PGresult *res = PQexec(conn, "SELECT version FROM ztc_database"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + fprintf(stderr, "Error determining database version"); + exit(1); + } + + if (PQntuples(res) != 1) { + fprintf(stderr, "Invalid number of db version tuples returned."); + exit(1); + } + + int dbVersion = std::stoi(PQgetvalue(res, 0, 0)); + + if (dbVersion < DB_MINIMUM_VERSION) { + fprintf(stderr, "Central database schema version too low. This controller version requires a minimum schema version of %d. Please upgrade your Central instance", DB_MINIMUM_VERSION); + exit(1); + } + + PQclear(res); + res = NULL; + PQfinish(conn); + conn = NULL; + + _readyLock.lock(); + _heartbeatThread = std::thread(&PostgreSQL::heartbeat, this); + _membersDbWatcher = std::thread(&PostgreSQL::membersDbWatcher, this); + _networksDbWatcher = std::thread(&PostgreSQL::networksDbWatcher, this); + for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) { + _commitThread[i] = std::thread(&PostgreSQL::commitThread, this); + } + _onlineNotificationThread = std::thread(&PostgreSQL::onlineNotificationThread, this); +} + +PostgreSQL::~PostgreSQL() +{ + _run = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + _heartbeatThread.join(); + _membersDbWatcher.join(); + _networksDbWatcher.join(); + for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) { + _commitThread[i].join(); + } + _onlineNotificationThread.join(); + +} + + +bool PostgreSQL::waitForReady() +{ + while (_ready < 2) { + if (!_waitNoticePrinted) { + _waitNoticePrinted = true; + fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, ::_timestr(), (unsigned long long)_myAddress.toInt()); + } + _readyLock.lock(); + _readyLock.unlock(); + } + return true; +} + +bool PostgreSQL::isReady() +{ + return ((_ready == 2)&&(_connected)); +} + +void PostgreSQL::save(nlohmann::json *orig, nlohmann::json &record) +{ + try { + if (!record.is_object()) { + return; + } + waitForReady(); + if (orig) { + if (*orig != record) { + record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1; + _commitQueue.post(new nlohmann::json(record)); + } + } else { + record["revision"] = 1; + _commitQueue.post(new nlohmann::json(record)); + } + } catch (std::exception &e) { + fprintf(stderr, "Error on PostgreSQL::save: %s\n", e.what()); + } catch (...) { + fprintf(stderr, "Unknown error on PostgreSQL::save\n"); + } +} + +void PostgreSQL::eraseNetwork(const uint64_t networkId) +{ + char tmp2[24]; + waitForReady(); + Utils::hex(networkId, tmp2); + json *tmp = new json(); + (*tmp)["id"] = tmp2; + (*tmp)["objtype"] = "_delete_network"; + _commitQueue.post(tmp); +} + +void PostgreSQL::eraseMember(const uint64_t networkId, const uint64_t memberId) +{ + char tmp2[24]; + json *tmp = new json(); + Utils::hex(networkId, tmp2); + (*tmp)["nwid"] = tmp2; + Utils::hex(memberId, tmp2); + (*tmp)["id"] = tmp2; + (*tmp)["objtype"] = "_delete_member"; + _commitQueue.post(tmp); +} + +void PostgreSQL::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress) +{ + std::lock_guard<std::mutex> l(_lastOnline_l); + std::pair<int64_t, InetAddress> &i = _lastOnline[std::pair<uint64_t,uint64_t>(networkId, memberId)]; + i.first = OSUtils::now(); + if (physicalAddress) { + i.second = physicalAddress; + } +} + +void PostgreSQL::initializeNetworks(PGconn *conn) +{ + try { + if (PQstatus(conn) != CONNECTION_OK) { + fprintf(stderr, "Bad Database Connection: %s", PQerrorMessage(conn)); + exit(1); + } + + const char *params[1] = { + _myAddressStr.c_str() + }; + + PGresult *res = PQexecParams(conn, "SELECT id, EXTRACT(EPOCH FROM creation_time AT TIME ZONE 'UTC')*1000, capabilities, " + "enable_broadcast, EXTRACT(EPOCH FROM last_modified AT TIME ZONE 'UTC')*1000, mtu, multicast_limit, name, private, remote_trace_level, " + "remote_trace_target, revision, rules, tags, v4_assign_mode, v6_assign_mode FROM ztc_network " + "WHERE deleted = false AND controller_id = $1", + 1, + NULL, + params, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + fprintf(stderr, "Networks Initialization Failed: %s", PQerrorMessage(conn)); + PQclear(res); + exit(1); + } + + int numRows = PQntuples(res); + for (int i = 0; i < numRows; ++i) { + json empty; + json config; + + const char *nwidparam[1] = { + PQgetvalue(res, i, 0) + }; + + config["id"] = PQgetvalue(res, i, 0); + config["nwid"] = PQgetvalue(res, i, 0); + try { + config["creationTime"] = std::stoull(PQgetvalue(res, i, 1)); + } catch (std::exception &e) { + config["creationTime"] = 0ULL; + //fprintf(stderr, "Error converting creation time: %s\n", PQgetvalue(res, i, 1)); + } + config["capabilities"] = json::parse(PQgetvalue(res, i, 2)); + config["enableBroadcast"] = (strcmp(PQgetvalue(res, i, 3),"t")==0); + try { + config["lastModified"] = std::stoull(PQgetvalue(res, i, 4)); + } catch (std::exception &e) { + config["lastModified"] = 0ULL; + //fprintf(stderr, "Error converting last modified: %s\n", PQgetvalue(res, i, 4)); + } + try { + config["mtu"] = std::stoi(PQgetvalue(res, i, 5)); + } catch (std::exception &e) { + config["mtu"] = 2800; + } + try { + config["multicastLimit"] = std::stoi(PQgetvalue(res, i, 6)); + } catch (std::exception &e) { + config["multicastLimit"] = 64; + } + config["name"] = PQgetvalue(res, i, 7); + config["private"] = (strcmp(PQgetvalue(res, i, 8),"t")==0); + try { + config["remoteTraceLevel"] = std::stoi(PQgetvalue(res, i, 9)); + } catch (std::exception &e) { + config["remoteTraceLevel"] = 0; + } + config["remoteTraceTarget"] = PQgetvalue(res, i, 10); + try { + config["revision"] = std::stoull(PQgetvalue(res, i, 11)); + } catch (std::exception &e) { + config["revision"] = 0ULL; + //fprintf(stderr, "Error converting revision: %s\n", PQgetvalue(res, i, 11)); + } + config["rules"] = json::parse(PQgetvalue(res, i, 12)); + config["tags"] = json::parse(PQgetvalue(res, i, 13)); + config["v4AssignMode"] = json::parse(PQgetvalue(res, i, 14)); + config["v6AssignMode"] = json::parse(PQgetvalue(res, i, 15)); + config["objtype"] = "network"; + config["ipAssignmentPools"] = json::array(); + config["routes"] = json::array(); + + PGresult *r2 = PQexecParams(conn, + "SELECT host(ip_range_start), host(ip_range_end) FROM ztc_network_assignment_pool WHERE network_id = $1", + 1, + NULL, + nwidparam, + NULL, + NULL, + 0); + + if (PQresultStatus(r2) != PGRES_TUPLES_OK) { + fprintf(stderr, "ERROR: Error retreiving IP pools for network: %s\n", PQresultErrorMessage(r2)); + PQclear(r2); + PQclear(res); + exit(1); + } + + int n = PQntuples(r2); + for (int j = 0; j < n; ++j) { + json ip; + ip["ipRangeStart"] = PQgetvalue(r2, j, 0); + ip["ipRangeEnd"] = PQgetvalue(r2, j, 1); + + config["ipAssignmentPools"].push_back(ip); + } + + PQclear(r2); + + r2 = PQexecParams(conn, + "SELECT host(address), bits, host(via) FROM ztc_network_route WHERE network_id = $1", + 1, + NULL, + nwidparam, + NULL, + NULL, + 0); + + if (PQresultStatus(r2) != PGRES_TUPLES_OK) { + fprintf(stderr, "ERROR: Error retreiving routes for network: %s\n", PQresultErrorMessage(r2)); + PQclear(r2); + PQclear(res); + exit(1); + } + + n = PQntuples(r2); + for (int j = 0; j < n; ++j) { + std::string addr = PQgetvalue(r2, j, 0); + std::string bits = PQgetvalue(r2, j, 1); + std::string via = PQgetvalue(r2, j, 2); + json route; + route["target"] = addr + "/" + bits; + + if (via == "NULL") { + route["via"] = nullptr; + } else { + route["via"] = via; + } + config["routes"].push_back(route); + } + + PQclear(r2); + + _networkChanged(empty, config, false); + } + + PQclear(res); + + if (++this->_ready == 2) { + if (_waitNoticePrinted) { + fprintf(stderr,"[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); + } + _readyLock.unlock(); + } + } catch (std::exception &e) { + fprintf(stderr, "ERROR: Error initializing networks: %s", e.what()); + exit(-1); + } +} + +void PostgreSQL::initializeMembers(PGconn *conn) +{ + try { + if (PQstatus(conn) != CONNECTION_OK) { + fprintf(stderr, "Bad Database Connection: %s", PQerrorMessage(conn)); + exit(1); + } + + const char *params[1] = { + _myAddressStr.c_str() + }; + + PGresult *res = PQexecParams(conn, + "SELECT m.id, m.network_id, m.active_bridge, m.authorized, m.capabilities, EXTRACT(EPOCH FROM m.creation_time AT TIME ZONE 'UTC')*1000, m.identity, " + " EXTRACT(EPOCH FROM m.last_authorized_time AT TIME ZONE 'UTC')*1000, " + " EXTRACT(EPOCH FROM m.last_deauthorized_time AT TIME ZONE 'UTC')*1000, " + " m.remote_trace_level, m.remote_trace_target, m.tags, m.v_major, m.v_minor, m.v_rev, m.v_proto, " + " m.no_auto_assign_ips, m.revision " + "FROM ztc_member m " + "INNER JOIN ztc_network n " + " ON n.id = m.network_id " + "WHERE n.controller_id = $1 AND m.deleted = false", + 1, + NULL, + params, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + fprintf(stderr, "Member Initialization Failed: %s", PQerrorMessage(conn)); + PQclear(res); + exit(1); + } + + int numRows = PQntuples(res); + for (int i = 0; i < numRows; ++i) { + json empty; + json config; + + std::string memberId(PQgetvalue(res, i, 0)); + std::string networkId(PQgetvalue(res, i, 1)); + std::string ctime = PQgetvalue(res, i, 5); + config["id"] = memberId; + config["nwid"] = networkId; + config["activeBridge"] = (strcmp(PQgetvalue(res, i, 2), "t") == 0); + config["authorized"] = (strcmp(PQgetvalue(res, i, 3), "t") == 0); + try { + config["capabilities"] = json::parse(PQgetvalue(res, i, 4)); + } catch (std::exception &e) { + config["capabilities"] = json::array(); + } + try { + config["creationTime"] = std::stoull(PQgetvalue(res, i, 5)); + } catch (std::exception &e) { + config["creationTime"] = 0ULL; + //fprintf(stderr, "Error upding creation time (member): %s\n", PQgetvalue(res, i, 5)); + } + config["identity"] = PQgetvalue(res, i, 6); + try { + config["lastAuthorizedTime"] = std::stoull(PQgetvalue(res, i, 7)); + } catch(std::exception &e) { + config["lastAuthorizedTime"] = 0ULL; + //fprintf(stderr, "Error updating last auth time (member): %s\n", PQgetvalue(res, i, 7)); + } + try { + config["lastDeauthorizedTime"] = std::stoull(PQgetvalue(res, i, 8)); + } catch( std::exception &e) { + config["lastDeauthorizedTime"] = 0ULL; + //fprintf(stderr, "Error updating last deauth time (member): %s\n", PQgetvalue(res, i, 8)); + } + try { + config["remoteTraceLevel"] = std::stoi(PQgetvalue(res, i, 9)); + } catch (std::exception &e) { + config["remoteTraceLevel"] = 0; + } + config["remoteTraceTarget"] = PQgetvalue(res, i, 10); + try { + config["tags"] = json::parse(PQgetvalue(res, i, 11)); + } catch (std::exception &e) { + config["tags"] = json::array(); + } + try { + config["vMajor"] = std::stoi(PQgetvalue(res, i, 12)); + } catch(std::exception &e) { + config["vMajor"] = -1; + } + try { + config["vMinor"] = std::stoi(PQgetvalue(res, i, 13)); + } catch (std::exception &e) { + config["vMinor"] = -1; + } + try { + config["vRev"] = std::stoi(PQgetvalue(res, i, 14)); + } catch (std::exception &e) { + config["vRev"] = -1; + } + try { + config["vProto"] = std::stoi(PQgetvalue(res, i, 15)); + } catch (std::exception &e) { + config["vProto"] = -1; + } + config["noAutoAssignIps"] = (strcmp(PQgetvalue(res, i, 16), "t") == 0); + try { + config["revision"] = std::stoull(PQgetvalue(res, i, 17)); + } catch (std::exception &e) { + config["revision"] = 0ULL; + //fprintf(stderr, "Error updating revision (member): %s\n", PQgetvalue(res, i, 17)); + } + config["objtype"] = "member"; + config["ipAssignments"] = json::array(); + const char *p2[2] = { + memberId.c_str(), + networkId.c_str() + }; + + PGresult *r2 = PQexecParams(conn, + "SELECT DISTINCT address FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2", + 2, + NULL, + p2, + NULL, + NULL, + 0); + + if (PQresultStatus(r2) != PGRES_TUPLES_OK) { + fprintf(stderr, "Member Initialization Failed: %s", PQerrorMessage(conn)); + PQclear(r2); + PQclear(res); + exit(1); + } + + int n = PQntuples(r2); + for (int j = 0; j < n; ++j) { + config["ipAssignments"].push_back(PQgetvalue(r2, j, 0)); + } + + _memberChanged(empty, config, false); + } + + PQclear(res); + + if (++this->_ready == 2) { + if (_waitNoticePrinted) { + fprintf(stderr,"[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); + } + _readyLock.unlock(); + } + } catch (std::exception &e) { + fprintf(stderr, "ERROR: Error initializing members: %s\n", e.what()); + exit(-1); + } +} + +void PostgreSQL::heartbeat() +{ + char publicId[1024]; + char hostnameTmp[1024]; + _myId.toString(false,publicId); + if (gethostname(hostnameTmp, sizeof(hostnameTmp))!= 0) { + hostnameTmp[0] = (char)0; + } else { + for (int i = 0; i < sizeof(hostnameTmp); ++i) { + if ((hostnameTmp[i] == '.')||(hostnameTmp[i] == 0)) { + hostnameTmp[i] = (char)0; + break; + } + } + } + const char *controllerId = _myAddressStr.c_str(); + const char *publicIdentity = publicId; + const char *hostname = hostnameTmp; + + PGconn *conn = getPgConn(); + if (PQstatus(conn) == CONNECTION_BAD) { + fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); + PQfinish(conn); + exit(1); + } + while (_run == 1) { + if(PQstatus(conn) != CONNECTION_OK) { + fprintf(stderr, "%s heartbeat thread lost connection to Database\n", _myAddressStr.c_str()); + PQfinish(conn); + exit(6); + } + if (conn) { + std::string major = std::to_string(ZEROTIER_ONE_VERSION_MAJOR); + std::string minor = std::to_string(ZEROTIER_ONE_VERSION_MINOR); + std::string rev = std::to_string(ZEROTIER_ONE_VERSION_REVISION); + std::string build = std::to_string(ZEROTIER_ONE_VERSION_BUILD); + std::string now = std::to_string(OSUtils::now()); + std::string host_port = std::to_string(_listenPort); + std::string use_rabbitmq = (_mqc != NULL) ? "true" : "false"; + const char *values[10] = { + controllerId, + hostname, + now.c_str(), + publicIdentity, + major.c_str(), + minor.c_str(), + rev.c_str(), + build.c_str(), + host_port.c_str(), + use_rabbitmq.c_str() + }; + + PGresult *res = PQexecParams(conn, + "INSERT INTO ztc_controller (id, cluster_host, last_alive, public_identity, v_major, v_minor, v_rev, v_build, host_port, use_rabbitmq) " + "VALUES ($1, $2, TO_TIMESTAMP($3::double precision/1000), $4, $5, $6, $7, $8, $9, $10) " + "ON CONFLICT (id) DO UPDATE SET cluster_host = EXCLUDED.cluster_host, last_alive = EXCLUDED.last_alive, " + "public_identity = EXCLUDED.public_identity, v_major = EXCLUDED.v_major, v_minor = EXCLUDED.v_minor, " + "v_rev = EXCLUDED.v_rev, v_build = EXCLUDED.v_rev, host_port = EXCLUDED.host_port, " + "use_rabbitmq = EXCLUDED.use_rabbitmq", + 10, // number of parameters + NULL, // oid field. ignore + values, // values for substitution + NULL, // lengths in bytes of each value + NULL, // binary? + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "Heartbeat Update Failed: %s\n", PQresultErrorMessage(res)); + } + PQclear(res); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + PQfinish(conn); + conn = NULL; +} + +void PostgreSQL::membersDbWatcher() +{ + PGconn *conn = getPgConn(NO_OVERRIDE); + if (PQstatus(conn) == CONNECTION_BAD) { + fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); + PQfinish(conn); + exit(1); + } + + initializeMembers(conn); + + if (this->_mqc != NULL) { + PQfinish(conn); + conn = NULL; + _membersWatcher_RabbitMQ(); + } else { + _membersWatcher_Postgres(conn); + PQfinish(conn); + conn = NULL; + } + + if (_run == 1) { + fprintf(stderr, "ERROR: %s membersDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + exit(9); + } + fprintf(stderr, "Exited membersDbWatcher\n"); +} + +void PostgreSQL::_membersWatcher_Postgres(PGconn *conn) { + char buf[11] = {0}; + std::string cmd = "LISTEN member_" + std::string(_myAddress.toString(buf)); + PGresult *res = PQexec(conn, cmd.c_str()); + if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "LISTEN command failed: %s\n", PQresultErrorMessage(res)); + PQclear(res); + PQfinish(conn); + exit(1); + } + + PQclear(res); res = NULL; + + while(_run == 1) { + if (PQstatus(conn) != CONNECTION_OK) { + fprintf(stderr, "ERROR: Member Watcher lost connection to Postgres."); + exit(-1); + } + PGnotify *notify = NULL; + PQconsumeInput(conn); + while ((notify = PQnotifies(conn)) != NULL) { + //fprintf(stderr, "ASYNC NOTIFY of '%s' id:%s received\n", notify->relname, notify->extra); + + try { + json tmp(json::parse(notify->extra)); + json &ov = tmp["old_val"]; + json &nv = tmp["new_val"]; + json oldConfig, newConfig; + if (ov.is_object()) oldConfig = ov; + if (nv.is_object()) newConfig = nv; + if (oldConfig.is_object() || newConfig.is_object()) { + _memberChanged(oldConfig,newConfig,(this->_ready>=2)); + } + } catch (...) {} // ignore bad records + + free(notify); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +} + +void PostgreSQL::_membersWatcher_RabbitMQ() { + char buf[11] = {0}; + std::string qname = "member_"+ std::string(_myAddress.toString(buf)); + RabbitMQ rmq(_mqc, qname.c_str()); + try { + rmq.init(); + } catch (std::runtime_error &e) { + fprintf(stderr, "RABBITMQ ERROR: %s\n", e.what()); + exit(11); + } + while (_run == 1) { + try { + std::string msg = rmq.consume(); + // fprintf(stderr, "Got Member Update: %s\n", msg.c_str()); + if (msg.empty()) { + continue; + } + json tmp(json::parse(msg)); + json &ov = tmp["old_val"]; + json &nv = tmp["new_val"]; + json oldConfig, newConfig; + if (ov.is_object()) oldConfig = ov; + if (nv.is_object()) newConfig = nv; + if (oldConfig.is_object() || newConfig.is_object()) { + _memberChanged(oldConfig,newConfig,(this->_ready>=2)); + } + } catch (std::runtime_error &e) { + fprintf(stderr, "RABBITMQ ERROR member change: %s\n", e.what()); + break; + } catch(std::exception &e ) { + fprintf(stderr, "RABBITMQ ERROR member change: %s\n", e.what()); + } catch(...) { + fprintf(stderr, "RABBITMQ ERROR member change: unknown error\n"); + } + } +} + +void PostgreSQL::networksDbWatcher() +{ + PGconn *conn = getPgConn(NO_OVERRIDE); + if (PQstatus(conn) == CONNECTION_BAD) { + fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); + PQfinish(conn); + exit(1); + } + + initializeNetworks(conn); + + if (this->_mqc != NULL) { + PQfinish(conn); + conn = NULL; + _networksWatcher_RabbitMQ(); + } else { + _networksWatcher_Postgres(conn); + PQfinish(conn); + conn = NULL; + } + + if (_run == 1) { + fprintf(stderr, "ERROR: %s networksDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + exit(8); + } + fprintf(stderr, "Exited membersDbWatcher\n"); +} + +void PostgreSQL::_networksWatcher_Postgres(PGconn *conn) { + char buf[11] = {0}; + std::string cmd = "LISTEN network_" + std::string(_myAddress.toString(buf)); + PGresult *res = PQexec(conn, cmd.c_str()); + if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "LISTEN command failed: %s\n", PQresultErrorMessage(res)); + PQclear(res); + PQfinish(conn); + exit(1); + } + + PQclear(res); res = NULL; + + while(_run == 1) { + if (PQstatus(conn) != CONNECTION_OK) { + fprintf(stderr, "ERROR: Network Watcher lost connection to Postgres."); + exit(-1); + } + PGnotify *notify = NULL; + PQconsumeInput(conn); + while ((notify = PQnotifies(conn)) != NULL) { + //fprintf(stderr, "ASYNC NOTIFY of '%s' id:%s received\n", notify->relname, notify->extra); + try { + json tmp(json::parse(notify->extra)); + json &ov = tmp["old_val"]; + json &nv = tmp["new_val"]; + json oldConfig, newConfig; + if (ov.is_object()) oldConfig = ov; + if (nv.is_object()) newConfig = nv; + if (oldConfig.is_object()||newConfig.is_object()) { + _networkChanged(oldConfig,newConfig,(this->_ready >= 2)); + } + } catch (...) {} // ignore bad records + free(notify); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +} + +void PostgreSQL::_networksWatcher_RabbitMQ() { + char buf[11] = {0}; + std::string qname = "network_"+ std::string(_myAddress.toString(buf)); + RabbitMQ rmq(_mqc, qname.c_str()); + try { + rmq.init(); + } catch (std::runtime_error &e) { + fprintf(stderr, "RABBITMQ ERROR: %s\n", e.what()); + exit(11); + } + while (_run == 1) { + try { + std::string msg = rmq.consume(); + if (msg.empty()) { + continue; + } + // fprintf(stderr, "Got network update: %s\n", msg.c_str()); + json tmp(json::parse(msg)); + json &ov = tmp["old_val"]; + json &nv = tmp["new_val"]; + json oldConfig, newConfig; + if (ov.is_object()) oldConfig = ov; + if (nv.is_object()) newConfig = nv; + if (oldConfig.is_object()||newConfig.is_object()) { + _networkChanged(oldConfig,newConfig,(this->_ready >= 2)); + } + } catch (std::runtime_error &e) { + fprintf(stderr, "RABBITMQ ERROR: %s\n", e.what()); + break; + } catch (std::exception &e) { + fprintf(stderr, "RABBITMQ ERROR network watcher: %s\n", e.what()); + } catch(...) { + fprintf(stderr, "RABBITMQ ERROR network watcher: unknown error\n"); + } + } +} + +void PostgreSQL::commitThread() +{ + PGconn *conn = getPgConn(); + if (PQstatus(conn) == CONNECTION_BAD) { + fprintf(stderr, "ERROR: Connection to database failed: %s\n", PQerrorMessage(conn)); + PQfinish(conn); + exit(1); + } + + json *config = nullptr; + while(_commitQueue.get(config)&(_run == 1)) { + if (!config) { + continue; + } + if (PQstatus(conn) == CONNECTION_BAD) { + fprintf(stderr, "ERROR: Connection to database failed: %s\n", PQerrorMessage(conn)); + PQfinish(conn); + delete config; + exit(1); + } + try { + const std::string objtype = (*config)["objtype"]; + if (objtype == "member") { + try { + std::string memberId = (*config)["id"]; + std::string networkId = (*config)["nwid"]; + std::string identity = (*config)["identity"]; + std::string target = "NULL"; + + if (!(*config)["remoteTraceTarget"].is_null()) { + target = (*config)["remoteTraceTarget"]; + } + + std::string caps = OSUtils::jsonDump((*config)["capabilities"], -1); + std::string lastAuthTime = std::to_string((long long)(*config)["lastAuthorizedTime"]); + std::string lastDeauthTime = std::to_string((long long)(*config)["lastDeauthorizedTime"]); + std::string rtraceLevel = std::to_string((int)(*config)["remoteTraceLevel"]); + std::string rev = std::to_string((unsigned long long)(*config)["revision"]); + std::string tags = OSUtils::jsonDump((*config)["tags"], -1); + std::string vmajor = std::to_string((int)(*config)["vMajor"]); + std::string vminor = std::to_string((int)(*config)["vMinor"]); + std::string vrev = std::to_string((int)(*config)["vRev"]); + std::string vproto = std::to_string((int)(*config)["vProto"]); + const char *values[19] = { + memberId.c_str(), + networkId.c_str(), + ((*config)["activeBridge"] ? "true" : "false"), + ((*config)["authorized"] ? "true" : "false"), + caps.c_str(), + identity.c_str(), + lastAuthTime.c_str(), + lastDeauthTime.c_str(), + ((*config)["noAutoAssignIps"] ? "true" : "false"), + rtraceLevel.c_str(), + (target == "NULL") ? NULL : target.c_str(), + rev.c_str(), + tags.c_str(), + vmajor.c_str(), + vminor.c_str(), + vrev.c_str(), + vproto.c_str() + }; + + PGresult *res = PQexecParams(conn, + "INSERT INTO ztc_member (id, network_id, active_bridge, authorized, capabilities, " + "identity, last_authorized_time, last_deauthorized_time, no_auto_assign_ips, " + "remote_trace_level, remote_trace_target, revision, tags, v_major, v_minor, v_rev, v_proto) " + "VALUES ($1, $2, $3, $4, $5, $6, " + "TO_TIMESTAMP($7::double precision/1000), TO_TIMESTAMP($8::double precision/1000), " + "$9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (network_id, id) DO UPDATE SET " + "active_bridge = EXCLUDED.active_bridge, authorized = EXCLUDED.authorized, capabilities = EXCLUDED.capabilities, " + "identity = EXCLUDED.identity, last_authorized_time = EXCLUDED.last_authorized_time, " + "last_deauthorized_time = EXCLUDED.last_deauthorized_time, no_auto_assign_ips = EXCLUDED.no_auto_assign_ips, " + "remote_trace_level = EXCLUDED.remote_trace_level, remote_trace_target = EXCLUDED.remote_trace_target, " + "revision = EXCLUDED.revision+1, tags = EXCLUDED.tags, v_major = EXCLUDED.v_major, " + "v_minor = EXCLUDED.v_minor, v_rev = EXCLUDED.v_rev, v_proto = EXCLUDED.v_proto", + 17, + NULL, + values, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error updating member: %s\n", PQresultErrorMessage(res)); + fprintf(stderr, "%s", OSUtils::jsonDump(*config, 2).c_str()); + PQclear(res); + delete config; + config = nullptr; + continue; + } + + PQclear(res); + + res = PQexec(conn, "BEGIN"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error beginning transaction: %s\n", PQresultErrorMessage(res)); + PQclear(res); + delete config; + config = nullptr; + continue; + } + + PQclear(res); + + const char *v2[2] = { + memberId.c_str(), + networkId.c_str() + }; + + res = PQexecParams(conn, + "DELETE FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2", + 2, + NULL, + v2, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error updating IP address assignments: %s\n", PQresultErrorMessage(res)); + PQclear(res); + PQclear(PQexec(conn, "ROLLBACK"));; + delete config; + config = nullptr; + continue; + } + + PQclear(res); + + std::vector<std::string> assignments; + for (auto i = (*config)["ipAssignments"].begin(); i != (*config)["ipAssignments"].end(); ++i) { + std::string addr = *i; + + if (std::find(assignments.begin(), assignments.end(), addr) != assignments.end()) { + continue; + } + + const char *v3[3] = { + memberId.c_str(), + networkId.c_str(), + addr.c_str() + }; + + res = PQexecParams(conn, + "INSERT INTO ztc_member_ip_assignment (member_id, network_id, address) VALUES ($1, $2, $3)", + 3, + NULL, + v3, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error setting IP addresses for member: %s\n", PQresultErrorMessage(res)); + PQclear(res); + PQclear(PQexec(conn, "ROLLBACK")); + break;; + } + } + + res = PQexec(conn, "COMMIT"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error committing ip address data: %s\n", PQresultErrorMessage(res)); + } + + PQclear(res); + + const uint64_t nwidInt = OSUtils::jsonIntHex((*config)["nwid"], 0ULL); + const uint64_t memberidInt = OSUtils::jsonIntHex((*config)["id"], 0ULL); + if (nwidInt && memberidInt) { + nlohmann::json nwOrig; + nlohmann::json memOrig; + + nlohmann::json memNew(*config); + + get(nwidInt, nwOrig, memberidInt, memOrig); + + _memberChanged(memOrig, memNew, (this->_ready>=2)); + } else { + fprintf(stderr, "Can't notify of change. Error parsing nwid or memberid: %lu-%lu\n", nwidInt, memberidInt); + } + + } catch (std::exception &e) { + fprintf(stderr, "ERROR: Error updating member: %s\n", e.what()); + } + } else if (objtype == "network") { + try { + std::string id = (*config)["id"]; + std::string controllerId = _myAddressStr.c_str(); + std::string name = (*config)["name"]; + std::string remoteTraceTarget("NULL"); + if (!(*config)["remoteTraceTarget"].is_null()) { + remoteTraceTarget = (*config)["remoteTraceTarget"]; + } + std::string rulesSource = (*config)["rulesSource"]; + std::string caps = OSUtils::jsonDump((*config)["capabilitles"], -1); + std::string now = std::to_string(OSUtils::now()); + std::string mtu = std::to_string((int)(*config)["mtu"]); + std::string mcastLimit = std::to_string((int)(*config)["multicastLimit"]); + std::string rtraceLevel = std::to_string((int)(*config)["remoteTraceLevel"]); + std::string rules = OSUtils::jsonDump((*config)["rules"], -1); + std::string tags = OSUtils::jsonDump((*config)["tags"], -1); + std::string v4mode = OSUtils::jsonDump((*config)["v4AssignMode"],-1); + std::string v6mode = OSUtils::jsonDump((*config)["v6AssignMode"], -1); + bool enableBroadcast = (*config)["enableBroadcast"]; + bool isPrivate = (*config)["private"]; + + const char *values[16] = { + id.c_str(), + controllerId.c_str(), + caps.c_str(), + enableBroadcast ? "true" : "false", + now.c_str(), + mtu.c_str(), + mcastLimit.c_str(), + name.c_str(), + isPrivate ? "true" : "false", + rtraceLevel.c_str(), + (remoteTraceTarget == "NULL" ? NULL : remoteTraceTarget.c_str()), + rules.c_str(), + rulesSource.c_str(), + tags.c_str(), + v4mode.c_str(), + v6mode.c_str(), + }; + + PGresult *res = PQexecParams(conn, + "UPDATE ztc_network SET controller_id = $2, capabilities = $3, enable_broadcast = $4, " + "last_updated = $5, mtu = $6, multicast_limit = $7, name = $8, private = $9, " + "remote_trace_level = $10, remote_trace_target = $11, rules = $12, rules_source = $13, " + "tags = $14, v4_assign_mode = $15, v6_assign_mode = $16 " + "WHERE id = $1", + 16, + NULL, + values, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error updating network record: %s\n", PQresultErrorMessage(res)); + PQclear(res); + delete config; + config = nullptr; + continue; + } + + PQclear(res); + + res = PQexec(conn, "BEGIN"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error beginnning transaction: %s\n", PQresultErrorMessage(res)); + PQclear(res); + delete config; + config = nullptr; + continue; + } + + PQclear(res); + + const char *params[1] = { + id.c_str() + }; + res = PQexecParams(conn, + "DELETE FROM ztc_network_assignment_pool WHERE network_id = $1", + 1, + NULL, + params, + NULL, + NULL, + 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error updating assignment pool: %s\n", PQresultErrorMessage(res)); + PQclear(res); + PQclear(PQexec(conn, "ROLLBACK")); + delete config; + config = nullptr; + continue; + } + + PQclear(res); + + auto pool = (*config)["ipAssignmentPools"]; + bool err = false; + for (auto i = pool.begin(); i != pool.end(); ++i) { + std::string start = (*i)["ipRangeStart"]; + std::string end = (*i)["ipRangeEnd"]; + const char *p[3] = { + id.c_str(), + start.c_str(), + end.c_str() + }; + + res = PQexecParams(conn, + "INSERT INTO ztc_network_assignment_pool (network_id, ip_range_start, ip_range_end) " + "VALUES ($1, $2, $3)", + 3, + NULL, + p, + NULL, + NULL, + 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error updating assignment pool: %s\n", PQresultErrorMessage(res)); + PQclear(res); + err = true; + break; + } + PQclear(res); + } + if (err) { + PQclear(PQexec(conn, "ROLLBACK")); + delete config; + config = nullptr; + continue; + } + + res = PQexecParams(conn, + "DELETE FROM ztc_network_route WHERE network_id = $1", + 1, + NULL, + params, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error updating routes: %s\n", PQresultErrorMessage(res)); + PQclear(res); + PQclear(PQexec(conn, "ROLLBACK")); + delete config; + config = nullptr; + continue; + } + + + auto routes = (*config)["routes"]; + err = false; + for (auto i = routes.begin(); i != routes.end(); ++i) { + std::string t = (*i)["target"]; + std::vector<std::string> target; + std::istringstream f(t); + std::string s; + while(std::getline(f, s, '/')) { + target.push_back(s); + } + if (target.empty() || target.size() != 2) { + continue; + } + std::string targetAddr = target[0]; + std::string targetBits = target[1]; + std::string via = "NULL"; + if (!(*i)["via"].is_null()) { + via = (*i)["via"]; + } + + const char *p[4] = { + id.c_str(), + targetAddr.c_str(), + targetBits.c_str(), + (via == "NULL" ? NULL : via.c_str()), + }; + + res = PQexecParams(conn, + "INSERT INTO ztc_network_route (network_id, address, bits, via) VALUES ($1, $2, $3, $4)", + 4, + NULL, + p, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error updating routes: %s\n", PQresultErrorMessage(res)); + PQclear(res); + err = true; + break; + } + PQclear(res); + } + if (err) { + PQclear(PQexec(conn, "ROLLBACK")); + delete config; + config = nullptr; + continue; + } + + res = PQexec(conn, "COMMIT"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error committing network update: %s\n", PQresultErrorMessage(res)); + } + PQclear(res); + + const uint64_t nwidInt = OSUtils::jsonIntHex((*config)["nwid"], 0ULL); + if (nwidInt) { + nlohmann::json nwOrig; + nlohmann::json nwNew(*config); + + get(nwidInt, nwOrig); + + _networkChanged(nwOrig, nwNew, true); + } else { + fprintf(stderr, "Can't notify network changed: %lu\n", nwidInt); + } + + } catch (std::exception &e) { + fprintf(stderr, "ERROR: Error updating member: %s\n", e.what()); + } + } else if (objtype == "trace") { + fprintf(stderr, "ERROR: Trace not yet implemented"); + } else if (objtype == "_delete_network") { + try { + std::string networkId = (*config)["nwid"]; + const char *values[1] = { + networkId.c_str() + }; + PGresult * res = PQexecParams(conn, + "UPDATE ztc_network SET deleted = true WHERE id = $1", + 1, + NULL, + values, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error deleting network: %s\n", PQresultErrorMessage(res)); + } + + PQclear(res); + } catch (std::exception &e) { + fprintf(stderr, "ERROR: Error deleting network: %s\n", e.what()); + } + } else if (objtype == "_delete_member") { + try { + std::string memberId = (*config)["id"]; + std::string networkId = (*config)["nwid"]; + + const char *values[2] = { + memberId.c_str(), + networkId.c_str() + }; + + PGresult *res = PQexecParams(conn, + "UPDATE ztc_member SET hidden = true, deleted = true WHERE id = $1 AND network_id = $2", + 2, + NULL, + values, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "ERROR: Error deleting member: %s\n", PQresultErrorMessage(res)); + } + + PQclear(res); + } catch (std::exception &e) { + fprintf(stderr, "ERROR: Error deleting member: %s\n", e.what()); + } + } else { + fprintf(stderr, "ERROR: unknown objtype"); + } + } catch (std::exception &e) { + fprintf(stderr, "ERROR: Error getting objtype: %s\n", e.what()); + } + delete config; + config = nullptr; + + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + PQfinish(conn); + if (_run == 1) { + fprintf(stderr, "ERROR: %s commitThread should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + exit(7); + } +} + +void PostgreSQL::onlineNotificationThread() +{ + PGconn *conn = getPgConn(); + if (PQstatus(conn) == CONNECTION_BAD) { + fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); + PQfinish(conn); + exit(1); + } + _connected = 1; + + int64_t lastUpdatedNetworkStatus = 0; + std::unordered_map< std::pair<uint64_t,uint64_t>,int64_t,_PairHasher > lastOnlineCumulative; + + while (_run == 1) { + if (PQstatus(conn) != CONNECTION_OK) { + fprintf(stderr, "ERROR: Online Notification thread lost connection to Postgres."); + PQfinish(conn); + exit(5); + } + + // map used to send notifications to front end + std::unordered_map<std::string, std::vector<std::string>> updateMap; + + std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > lastOnline; + { + std::lock_guard<std::mutex> l(_lastOnline_l); + lastOnline.swap(_lastOnline); + } + + PGresult *res = NULL; + + std::stringstream memberUpdate; + memberUpdate << "INSERT INTO ztc_member_status (network_id, member_id, address, last_updated) VALUES "; + bool firstRun = true; + bool memberAdded = false; + for (auto i=lastOnline.begin(); i != lastOnline.end(); ++i) { + uint64_t nwid_i = i->first.first; + char nwidTmp[64]; + char memTmp[64]; + char ipTmp[64]; + OSUtils::ztsnprintf(nwidTmp,sizeof(nwidTmp), "%.16llx", nwid_i); + OSUtils::ztsnprintf(memTmp,sizeof(memTmp), "%.10llx", i->first.second); + + auto found = _networks.find(nwid_i); + if (found == _networks.end()) { + continue; // skip members trying to join non-existant networks + } + + std::string networkId(nwidTmp); + std::string memberId(memTmp); + + std::vector<std::string> &members = updateMap[networkId]; + members.push_back(memberId); + + lastOnlineCumulative[i->first] = i->second.first; + + + const char *qvals[2] = { + networkId.c_str(), + memberId.c_str() + }; + + res = PQexecParams(conn, + "SELECT id, network_id FROM ztc_member WHERE network_id = $1 AND id = $2", + 2, + NULL, + qvals, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + fprintf(stderr, "Member count failed: %s", PQerrorMessage(conn)); + PQclear(res); + continue; + } + + int nrows = PQntuples(res); + PQclear(res); + + if (nrows == 1) { + int64_t ts = i->second.first; + std::string ipAddr = i->second.second.toIpString(ipTmp); + std::string timestamp = std::to_string(ts); + + if (firstRun) { + firstRun = false; + } else { + memberUpdate << ", "; + } + + memberUpdate << "('" << networkId << "', '" << memberId << "', "; + if (ipAddr.empty()) { + memberUpdate << "NULL, "; + } else { + memberUpdate << "'" << ipAddr << "', "; + } + memberUpdate << "TO_TIMESTAMP(" << timestamp << "::double precision/1000))"; + memberAdded = true; + } else if (nrows > 1) { + fprintf(stderr, "nrows > 1?!?"); + continue; + } else { + continue; + } + } + memberUpdate << " ON CONFLICT (network_id, member_id) DO UPDATE SET address = EXCLUDED.address, last_updated = EXCLUDED.last_updated;"; + + if (memberAdded) { + res = PQexec(conn, memberUpdate.str().c_str()); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "Multiple insert failed: %s", PQerrorMessage(conn)); + } + PQclear(res); + } + + const int64_t now = OSUtils::now(); + if ((now - lastUpdatedNetworkStatus) > 10000) { + lastUpdatedNetworkStatus = now; + + std::vector<std::pair<uint64_t, std::shared_ptr<_Network>>> networks; + { + std::lock_guard<std::mutex> l(_networks_l); + for (auto i = _networks.begin(); i != _networks.end(); ++i) { + networks.push_back(*i); + } + } + + std::stringstream networkUpdate; + networkUpdate << "INSERT INTO ztc_network_status (network_id, bridge_count, authorized_member_count, online_member_count, total_member_count, last_modified) VALUES "; + bool nwFirstRun = true; + bool networkAdded = false; + for (auto i = networks.begin(); i != networks.end(); ++i) { + char tmp[64]; + Utils::hex(i->first, tmp); + + std::string networkId(tmp); + + std::vector<std::string> &_notUsed = updateMap[networkId]; + (void)_notUsed; + + uint64_t authMemberCount = 0; + uint64_t totalMemberCount = 0; + uint64_t onlineMemberCount = 0; + uint64_t bridgeCount = 0; + uint64_t ts = now; + { + std::lock_guard<std::mutex> l2(i->second->lock); + authMemberCount = i->second->authorizedMembers.size(); + totalMemberCount = i->second->members.size(); + bridgeCount = i->second->activeBridgeMembers.size(); + for (auto m=i->second->members.begin(); m != i->second->members.end(); ++m) { + auto lo = lastOnlineCumulative.find(std::pair<uint64_t,uint64_t>(i->first, m->first)); + if (lo != lastOnlineCumulative.end()) { + if ((now - lo->second) <= (ZT_NETWORK_AUTOCONF_DELAY * 2)) { + ++onlineMemberCount; + } else { + lastOnlineCumulative.erase(lo); + } + } + } + } + + const char *nvals[1] = { + networkId.c_str() + }; + + res = PQexecParams(conn, + "SELECT id FROM ztc_network WHERE id = $1", + 1, + NULL, + nvals, + NULL, + NULL, + 0); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + fprintf(stderr, "Network lookup failed: %s", PQerrorMessage(conn)); + PQclear(res); + continue; + } + + int nrows = PQntuples(res); + PQclear(res); + + if (nrows == 1) { + std::string bc = std::to_string(bridgeCount); + std::string amc = std::to_string(authMemberCount); + std::string omc = std::to_string(onlineMemberCount); + std::string tmc = std::to_string(totalMemberCount); + std::string timestamp = std::to_string(ts); + + if (nwFirstRun) { + nwFirstRun = false; + } else { + networkUpdate << ", "; + } + + networkUpdate << "('" << networkId << "', " << bc << ", " << amc << ", " << omc << ", " << tmc << ", " + << "TO_TIMESTAMP(" << timestamp << "::double precision/1000))"; + + networkAdded = true; + + } else if (nrows > 1) { + fprintf(stderr, "Number of networks > 1?!?!?"); + continue; + } else { + continue; + } + } + networkUpdate << " ON CONFLICT (network_id) DO UPDATE SET bridge_count = EXCLUDED.bridge_count, " + << "authorized_member_count = EXCLUDED.authorized_member_count, online_member_count = EXCLUDED.online_member_count, " + << "total_member_count = EXCLUDED.total_member_count, last_modified = EXCLUDED.last_modified"; + if (networkAdded) { + res = PQexec(conn, networkUpdate.str().c_str()); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "Error during multiple network upsert: %s", PQresultErrorMessage(res)); + } + PQclear(res); + } + } + + // for (auto it = updateMap.begin(); it != updateMap.end(); ++it) { + // std::string networkId = it->first; + // std::vector<std::string> members = it->second; + // std::stringstream queryBuilder; + + // std::string membersStr = ::join(members, ","); + + // queryBuilder << "NOTIFY controller, '" << networkId << ":" << membersStr << "'"; + // std::string query = queryBuilder.str(); + + // PGresult *res = PQexec(conn,query.c_str()); + // if (PQresultStatus(res) != PGRES_COMMAND_OK) { + // fprintf(stderr, "ERROR: Error sending NOTIFY: %s\n", PQresultErrorMessage(res)); + // } + // PQclear(res); + // } + + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + fprintf(stderr, "%s: Fell out of run loop in onlineNotificationThread\n", _myAddressStr.c_str()); + PQfinish(conn); + if (_run == 1) { + fprintf(stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + exit(6); + } +} + +PGconn *PostgreSQL::getPgConn(OverrideMode m) { + if (m == ALLOW_PGBOUNCER_OVERRIDE) { + char *connStr = getenv("PGBOUNCER_CONNSTR"); + if (connStr != NULL) { + fprintf(stderr, "PGBouncer Override\n"); + std::string conn(connStr); + conn += " application_name=controller-"; + conn += _myAddressStr.c_str(); + return PQconnectdb(conn.c_str()); + } + } + + return PQconnectdb(_connString.c_str()); +} +#endif //ZT_CONTROLLER_USE_LIBPQ diff --git a/controller/PostgreSQL.hpp b/controller/PostgreSQL.hpp new file mode 100644 index 00000000..f35f89fc --- /dev/null +++ b/controller/PostgreSQL.hpp @@ -0,0 +1,117 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#ifndef ZT_CONTROLLER_LIBPQ_HPP +#define ZT_CONTROLLER_LIBPQ_HPP + +#include "DB.hpp" + +#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4 + +extern "C" { + typedef struct pg_conn PGconn; +} + +namespace ZeroTier +{ + +struct MQConfig; + +/** + * A controller database driver that talks to PostgreSQL + * + * This is for use with ZeroTier Central. Others are free to build and use it + * but be aware taht we might change it at any time. + */ +class PostgreSQL : public DB +{ +public: + PostgreSQL(const Identity &myId, const char *path, int listenPort, MQConfig *mqc = NULL); + virtual ~PostgreSQL(); + + virtual bool waitForReady(); + virtual bool isReady(); + virtual void save(nlohmann::json *orig, nlohmann::json &record); + virtual void eraseNetwork(const uint64_t networkId); + virtual void eraseMember(const uint64_t networkId, const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress); + +protected: + struct _PairHasher + { + inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); } + }; + +private: + void initializeNetworks(PGconn *conn); + void initializeMembers(PGconn *conn); + void heartbeat(); + void membersDbWatcher(); + void _membersWatcher_Postgres(PGconn *conn); + void _membersWatcher_RabbitMQ(); + void networksDbWatcher(); + void _networksWatcher_Postgres(PGconn *conn); + void _networksWatcher_RabbitMQ(); + + void commitThread(); + void onlineNotificationThread(); + + enum OverrideMode { + ALLOW_PGBOUNCER_OVERRIDE = 0, + NO_OVERRIDE = 1 + }; + + PGconn * getPgConn( OverrideMode m = ALLOW_PGBOUNCER_OVERRIDE ); + + std::string _connString; + + BlockingQueue<nlohmann::json *> _commitQueue; + + std::thread _heartbeatThread; + std::thread _membersDbWatcher; + std::thread _networksDbWatcher; + std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS]; + std::thread _onlineNotificationThread; + + std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > _lastOnline; + + mutable std::mutex _lastOnline_l; + mutable std::mutex _readyLock; + std::atomic<int> _ready, _connected, _run; + mutable volatile bool _waitNoticePrinted; + + int _listenPort; + + MQConfig *_mqc; +}; + +} + +#endif // ZT_CONTROLLER_LIBPQ_HPP + +#endif // ZT_CONTROLLER_USE_LIBPQ diff --git a/controller/README.md b/controller/README.md index 20fb69a0..c93c08f5 100644 --- a/controller/README.md +++ b/controller/README.md @@ -19,9 +19,9 @@ Since ZeroTier nodes are mobile and do not need static IPs, implementing high av ZeroTier network controllers can easily be run in Docker or other container systems. Since containers do not need to actually join networks, extra privilege options like "--device=/dev/net/tun --privileged" are not needed. You'll just need to map the local JSON API port of the running controller and allow it to access the Internet (over UDP/9993 at a minimum) so things can reach and query it. -### RethinkDB Database Implementation +### PostgreSQL Database Implementation -The default controller stores its data in the filesystem in `controller.d` under ZeroTier's home folder. There's an alternative implementation that stores data in RethinkDB that can be built with `make central-controller`. Right now this is only guaranteed to build and run on Linux and is designed for use with [ZeroTier Central](https://my.zerotier.com/). You're welcome to use it but we don't "officially" support it for end-user use and it could change at any time. +The default controller stores its data in the filesystem in `controller.d` under ZeroTier's home folder. There's an alternative implementation that stores data in PostgreSQL that can be built with `make central-controller`. Right now this is only guaranteed to build and run on Centos 7 Linux with PostgreSQL 10 installed via the [PostgreSQL Yum Repository](https://www.postgresql.org/download/linux/redhat/) and is designed for use with [ZeroTier Central](https://my.zerotier.com/). You're welcome to use it but we don't "officially" support it for end-user use and it could change at any time. ### Upgrading from Older (1.1.14 or earlier) Versions diff --git a/controller/RabbitMQ.cpp b/controller/RabbitMQ.cpp new file mode 100644 index 00000000..eec9745d --- /dev/null +++ b/controller/RabbitMQ.cpp @@ -0,0 +1,107 @@ +#include "RabbitMQ.hpp" + +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#include <amqp.h> +#include <amqp_tcp_socket.h> +#include <stdexcept> +#include <cstring> + +namespace ZeroTier +{ + +RabbitMQ::RabbitMQ(MQConfig *cfg, const char *queueName) + : _mqc(cfg) + , _qName(queueName) + , _socket(NULL) + , _status(0) +{ +} + +RabbitMQ::~RabbitMQ() +{ + amqp_channel_close(_conn, _channel, AMQP_REPLY_SUCCESS); + amqp_connection_close(_conn, AMQP_REPLY_SUCCESS); + amqp_destroy_connection(_conn); +} + +void RabbitMQ::init() +{ + struct timeval tval; + memset(&tval, 0, sizeof(struct timeval)); + tval.tv_sec = 5; + + fprintf(stderr, "Initializing RabbitMQ %s\n", _qName); + _conn = amqp_new_connection(); + _socket = amqp_tcp_socket_new(_conn); + if (!_socket) { + throw std::runtime_error("Can't create socket for RabbitMQ"); + } + + _status = amqp_socket_open_noblock(_socket, _mqc->host, _mqc->port, &tval); + if (_status) { + throw std::runtime_error("Can't connect to RabbitMQ"); + } + + amqp_rpc_reply_t r = amqp_login(_conn, "/", 0, 131072, 0, AMQP_SASL_METHOD_PLAIN, + _mqc->username, _mqc->password); + if (r.reply_type != AMQP_RESPONSE_NORMAL) { + throw std::runtime_error("RabbitMQ Login Error"); + } + + static int chan = 0; + { + Mutex::Lock l(_chan_m); + _channel = ++chan; + } + amqp_channel_open(_conn, _channel); + r = amqp_get_rpc_reply(_conn); + if(r.reply_type != AMQP_RESPONSE_NORMAL) { + throw std::runtime_error("Error opening communication channel"); + } + + _q = amqp_queue_declare(_conn, _channel, amqp_cstring_bytes(_qName), 0, 0, 0, 0, amqp_empty_table); + r = amqp_get_rpc_reply(_conn); + if (r.reply_type != AMQP_RESPONSE_NORMAL) { + throw std::runtime_error("Error declaring queue " + std::string(_qName)); + } + + amqp_basic_consume(_conn, _channel, amqp_cstring_bytes(_qName), amqp_empty_bytes, 0, 1, 0, amqp_empty_table); + r = amqp_get_rpc_reply(_conn); + if (r.reply_type != AMQP_RESPONSE_NORMAL) { + throw std::runtime_error("Error consuming queue " + std::string(_qName)); + } + fprintf(stderr, "RabbitMQ Init OK %s\n", _qName); +} + +std::string RabbitMQ::consume() +{ + amqp_rpc_reply_t res; + amqp_envelope_t envelope; + amqp_maybe_release_buffers(_conn); + + struct timeval timeout; + timeout.tv_sec = 1; + timeout.tv_usec = 0; + + res = amqp_consume_message(_conn, &envelope, &timeout, 0); + if (res.reply_type != AMQP_RESPONSE_NORMAL) { + if (res.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION && res.library_error == AMQP_STATUS_TIMEOUT) { + // timeout waiting for message. Return empty string + return ""; + } else { + throw std::runtime_error("Error getting message"); + } + } + + std::string msg( + (const char*)envelope.message.body.bytes, + envelope.message.body.len + ); + amqp_destroy_envelope(&envelope); + return msg; +} + +} + +#endif // ZT_CONTROLLER_USE_LIBPQ diff --git a/controller/RabbitMQ.hpp b/controller/RabbitMQ.hpp new file mode 100644 index 00000000..d341681b --- /dev/null +++ b/controller/RabbitMQ.hpp @@ -0,0 +1,79 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ +#ifndef ZT_CONTROLLER_RABBITMQ_HPP +#define ZT_CONTROLLER_RABBITMQ_HPP + +namespace ZeroTier +{ +struct MQConfig { + const char *host; + int port; + const char *username; + const char *password; +}; +} + +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#include "../node/Mutex.hpp" + +#include <amqp.h> +#include <amqp_tcp_socket.h> +#include <string> + +namespace ZeroTier +{ + +class RabbitMQ { +public: + RabbitMQ(MQConfig *cfg, const char *queueName); + ~RabbitMQ(); + + void init(); + + std::string consume(); + +private: + MQConfig *_mqc; + const char *_qName; + + amqp_socket_t *_socket; + amqp_connection_state_t _conn; + amqp_queue_declare_ok_t *_q; + int _status; + + int _channel; + + Mutex _chan_m; + +}; + +} + +#endif // ZT_CONTROLLER_USE_LIBPQ + +#endif // ZT_CONTROLLER_RABBITMQ_HPP + diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp deleted file mode 100644 index a46d033f..00000000 --- a/controller/RethinkDB.cpp +++ /dev/null @@ -1,497 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -//#define ZT_CONTROLLER_USE_RETHINKDB - -#ifdef ZT_CONTROLLER_USE_RETHINKDB - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <time.h> - -#include "RethinkDB.hpp" -#include "EmbeddedNetworkController.hpp" - -#include "../version.h" - -#include <chrono> -#include <algorithm> -#include <stdexcept> - -#include "../ext/librethinkdbxx/build/include/rethinkdb.h" - -namespace R = RethinkDB; -using json = nlohmann::json; - -namespace ZeroTier { - -static const char *_timestr() -{ - time_t t = time(0); - char *ts = ctime(&t); - char *p = ts; - if (!p) - return ""; - while (*p) { - if (*p == '\n') { - *p = (char)0; - break; - } - ++p; - } - return ts; -} - -RethinkDB::RethinkDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) : - DB(nc,myId,path), - _ready(2), // two tables need to be synchronized before we're ready, so this is ready when it reaches 0 - _run(1), - _waitNoticePrinted(false) -{ - // rethinkdb:host:port:db[:auth] - std::vector<std::string> ps(OSUtils::split(path,":","","")); - if ((ps.size() < 4)||(ps[0] != "rethinkdb")) - throw std::runtime_error("invalid rethinkdb database url"); - _host = ps[1]; - _port = Utils::strToInt(ps[2].c_str()); - _db = ps[3]; - if (ps.size() > 4) - _auth = ps[4]; - - _readyLock.lock(); - - _membersDbWatcher = std::thread([this]() { - try { - while (_run == 1) { - try { - std::unique_ptr<R::Connection> rdb(R::connect(this->_host,this->_port,this->_auth)); - if (rdb) { - _membersDbWatcherConnection = (void *)rdb.get(); - auto cur = R::db(this->_db).table("Member",R::optargs("read_mode","outdated")).get_all(this->_myAddressStr,R::optargs("index","controllerId")).changes(R::optargs("squash",0.05,"include_initial",true,"include_types",true,"include_states",true)).run(*rdb); - while (cur.has_next()) { - if (_run != 1) break; - json tmp(json::parse(cur.next().as_json())); - if ((tmp["type"] == "state")&&(tmp["state"] == "ready")) { - if (--this->_ready == 0) { - if (_waitNoticePrinted) - fprintf(stderr,"[%s] NOTICE: %.10llx controller RethinkDB data download complete." ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - this->_readyLock.unlock(); - } - } else { - try { - json &ov = tmp["old_val"]; - json &nv = tmp["new_val"]; - json oldConfig,newConfig; - if (ov.is_object()) oldConfig = ov["config"]; - if (nv.is_object()) newConfig = nv["config"]; - if (oldConfig.is_object()||newConfig.is_object()) - this->_memberChanged(oldConfig,newConfig,(this->_ready <= 0)); - } catch ( ... ) {} // ignore bad records - } - } - } - } catch (std::exception &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (member change stream): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.what()); - } catch (R::Error &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (member change stream): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.message.c_str()); - } catch ( ... ) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (member change stream): unknown exception" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - } - std::this_thread::sleep_for(std::chrono::milliseconds(250)); - } - } catch ( ... ) {} - }); - - _networksDbWatcher = std::thread([this]() { - try { - while (_run == 1) { - try { - std::unique_ptr<R::Connection> rdb(R::connect(this->_host,this->_port,this->_auth)); - if (rdb) { - _networksDbWatcherConnection = (void *)rdb.get(); - auto cur = R::db(this->_db).table("Network",R::optargs("read_mode","outdated")).get_all(this->_myAddressStr,R::optargs("index","controllerId")).changes(R::optargs("squash",0.05,"include_initial",true,"include_types",true,"include_states",true)).run(*rdb); - while (cur.has_next()) { - if (_run != 1) break; - json tmp(json::parse(cur.next().as_json())); - if ((tmp["type"] == "state")&&(tmp["state"] == "ready")) { - if (--this->_ready == 0) { - if (_waitNoticePrinted) - fprintf(stderr,"[%s] NOTICE: %.10llx controller RethinkDB data download complete." ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - this->_readyLock.unlock(); - } - } else { - try { - json &ov = tmp["old_val"]; - json &nv = tmp["new_val"]; - json oldConfig,newConfig; - if (ov.is_object()) oldConfig = ov["config"]; - if (nv.is_object()) newConfig = nv["config"]; - if (oldConfig.is_object()||newConfig.is_object()) - this->_networkChanged(oldConfig,newConfig,(this->_ready <= 0)); - } catch ( ... ) {} // ignore bad records - } - } - } - } catch (std::exception &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (network change stream): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.what()); - } catch (R::Error &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (network change stream): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.message.c_str()); - } catch ( ... ) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (network change stream): unknown exception" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - } - std::this_thread::sleep_for(std::chrono::milliseconds(250)); - } - } catch ( ... ) {} - }); - - for(int t=0;t<ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS;++t) { - _commitThread[t] = std::thread([this]() { - try { - std::unique_ptr<R::Connection> rdb; - nlohmann::json *config = (nlohmann::json *)0; - while ((this->_commitQueue.get(config))&&(_run == 1)) { - if (!config) - continue; - nlohmann::json record; - const char *table = (const char *)0; - std::string deleteId; - try { - const std::string objtype = (*config)["objtype"]; - if (objtype == "member") { - const std::string nwid = (*config)["nwid"]; - const std::string id = (*config)["id"]; - record["id"] = nwid + "-" + id; - record["controllerId"] = this->_myAddressStr; - record["networkId"] = nwid; - record["nodeId"] = id; - record["config"] = *config; - table = "Member"; - } else if (objtype == "network") { - const std::string id = (*config)["id"]; - record["id"] = id; - record["controllerId"] = this->_myAddressStr; - record["config"] = *config; - table = "Network"; - } else if (objtype == "trace") { - record = *config; - table = "RemoteTrace"; - } else if (objtype == "_delete_network") { - deleteId = (*config)["id"]; - table = "Network"; - } else if (objtype == "_delete_member") { - deleteId = (*config)["nwid"]; - deleteId.push_back('-'); - const std::string tmp = (*config)["id"]; - deleteId.append(tmp); - table = "Member"; - } - } catch (std::exception &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update record creation): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.what()); - table = (const char *)0; - } catch (R::Error &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update record creation): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.message.c_str()); - table = (const char *)0; - } catch ( ... ) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update record creation): unknown exception" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - table = (const char *)0; - } - delete config; - if (!table) - continue; - const std::string jdump(OSUtils::jsonDump(record,-1)); - - while (_run == 1) { - try { - if (!rdb) - rdb = R::connect(this->_host,this->_port,this->_auth); - if (rdb) { - if (deleteId.length() > 0) { - //printf("DELETE: %s" ZT_EOL_S,deleteId.c_str()); - R::db(this->_db).table(table).get(deleteId).delete_().run(*rdb); - } else { - //printf("UPSERT: %s" ZT_EOL_S,record.dump().c_str()); - R::db(this->_db).table(table).insert(R::Datum::from_json(jdump),R::optargs("conflict","update","return_changes",false)).run(*rdb); - } - break; - } else { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update): connect failed (will retry)" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - rdb.reset(); - } - } catch (std::exception &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update): %s [%s]" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.what(),jdump.c_str()); - rdb.reset(); - } catch (R::Error &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update): %s [%s]" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.message.c_str(),jdump.c_str()); - rdb.reset(); - } catch ( ... ) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update): unknown exception [%s]" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),jdump.c_str()); - rdb.reset(); - } - std::this_thread::sleep_for(std::chrono::milliseconds(250)); - } - } - } catch (std::exception &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update outer loop): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.what()); - } catch (R::Error &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update outer loop): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.message.c_str()); - } catch ( ... ) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (insert/update outer loop): unknown exception" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - } - }); - } - - _onlineNotificationThread = std::thread([this]() { - int64_t lastUpdatedNetworkStatus = 0; - std::unordered_map< std::pair<uint64_t,uint64_t>,int64_t,_PairHasher > lastOnlineCumulative; - try { - std::unique_ptr<R::Connection> rdb; - while (_run == 1) { - try { - if (!rdb) { - _connected = 0; - rdb = R::connect(this->_host,this->_port,this->_auth); - } - - if (rdb) { - _connected = 1; - R::Array batch; - R::Object tmpobj; - - std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > lastOnline; - { - std::lock_guard<std::mutex> l(_lastOnline_l); - lastOnline.swap(_lastOnline); - } - - for(auto i=lastOnline.begin();i!=lastOnline.end();++i) { - lastOnlineCumulative[i->first] = i->second.first; - char tmp[64],tmp2[64]; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx-%.10llx",i->first.first,i->first.second); - tmpobj["id"] = tmp; - tmpobj["ts"] = i->second.first; - tmpobj["phy"] = i->second.second.toIpString(tmp2); - batch.emplace_back(tmpobj); - if (batch.size() >= 1024) { - R::db(this->_db).table("MemberStatus",R::optargs("read_mode","outdated")).insert(batch,R::optargs("conflict","update")).run(*rdb); - batch.clear(); - } - } - if (batch.size() > 0) { - R::db(this->_db).table("MemberStatus",R::optargs("read_mode","outdated")).insert(batch,R::optargs("conflict","update")).run(*rdb); - batch.clear(); - } - tmpobj.clear(); - - const int64_t now = OSUtils::now(); - if ((now - lastUpdatedNetworkStatus) > 10000) { - lastUpdatedNetworkStatus = now; - - std::vector< std::pair< uint64_t,std::shared_ptr<_Network> > > networks; - { - std::lock_guard<std::mutex> l(_networks_l); - networks.reserve(_networks.size() + 1); - for(auto i=_networks.begin();i!=_networks.end();++i) - networks.push_back(*i); - } - - for(auto i=networks.begin();i!=networks.end();++i) { - char tmp[64]; - Utils::hex(i->first,tmp); - tmpobj["id"] = tmp; - { - std::lock_guard<std::mutex> l2(i->second->lock); - tmpobj["authorizedMemberCount"] = i->second->authorizedMembers.size(); - tmpobj["totalMemberCount"] = i->second->members.size(); - unsigned long onlineMemberCount = 0; - for(auto m=i->second->members.begin();m!=i->second->members.end();++m) { - auto lo = lastOnlineCumulative.find(std::pair<uint64_t,uint64_t>(i->first,m->first)); - if (lo != lastOnlineCumulative.end()) { - if ((now - lo->second) <= (ZT_NETWORK_AUTOCONF_DELAY * 2)) - ++onlineMemberCount; - else lastOnlineCumulative.erase(lo); - } - } - tmpobj["onlineMemberCount"] = onlineMemberCount; - tmpobj["bridgeCount"] = i->second->activeBridgeMembers.size(); - tmpobj["ts"] = now; - } - batch.emplace_back(tmpobj); - if (batch.size() >= 1024) { - R::db(this->_db).table("NetworkStatus",R::optargs("read_mode","outdated")).insert(batch,R::optargs("conflict","update")).run(*rdb); - batch.clear(); - } - } - if (batch.size() > 0) { - R::db(this->_db).table("NetworkStatus",R::optargs("read_mode","outdated")).insert(batch,R::optargs("conflict","update")).run(*rdb); - batch.clear(); - } - } - } - } catch (std::exception &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (node status update): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.what()); - rdb.reset(); - } catch (R::Error &e) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (node status update): %s" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt(),e.message.c_str()); - rdb.reset(); - } catch ( ... ) { - fprintf(stderr,"[%s] ERROR: %.10llx controller RethinkDB (node status update): unknown exception" ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - rdb.reset(); - } - std::this_thread::sleep_for(std::chrono::milliseconds(250)); - } - } catch ( ... ) {} - }); - - _heartbeatThread = std::thread([this]() { - try { - R::Object controllerRecord; - std::unique_ptr<R::Connection> rdb; - - { - char publicId[1024]; - //char secretId[1024]; - char hostname[1024]; - this->_myId.toString(false,publicId); - //this->_myId.toString(true,secretId); - if (gethostname(hostname,sizeof(hostname)) != 0) { - hostname[0] = (char)0; - } else { - for(int i=0;i<sizeof(hostname);++i) { - if ((hostname[i] == '.')||(hostname[i] == 0)) { - hostname[i] = (char)0; - break; - } - } - } - controllerRecord["id"] = this->_myAddressStr.c_str(); - controllerRecord["publicIdentity"] = publicId; - //controllerRecord["secretIdentity"] = secretId; - if (hostname[0]) - controllerRecord["clusterHost"] = hostname; - controllerRecord["vMajor"] = ZEROTIER_ONE_VERSION_MAJOR; - controllerRecord["vMinor"] = ZEROTIER_ONE_VERSION_MINOR; - controllerRecord["vRev"] = ZEROTIER_ONE_VERSION_REVISION; - controllerRecord["vBuild"] = ZEROTIER_ONE_VERSION_BUILD; - } - - while (_run == 1) { - try { - if (!rdb) - rdb = R::connect(this->_host,this->_port,this->_auth); - if (rdb) { - controllerRecord["lastAlive"] = OSUtils::now(); - //printf("HEARTBEAT: %s" ZT_EOL_S,tmp); - R::db(this->_db).table("Controller",R::optargs("read_mode","outdated")).insert(controllerRecord,R::optargs("conflict","update")).run(*rdb); - } - } catch ( ... ) { - rdb.reset(); - } - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - } - } catch ( ... ) {} - }); -} - -RethinkDB::~RethinkDB() -{ - _run = 0; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - _commitQueue.stop(); - for(int t=0;t<ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS;++t) - _commitThread[t].join(); - if (_membersDbWatcherConnection) - ((R::Connection *)_membersDbWatcherConnection)->close(); - if (_networksDbWatcherConnection) - ((R::Connection *)_networksDbWatcherConnection)->close(); - _membersDbWatcher.join(); - _networksDbWatcher.join(); - _heartbeatThread.join(); - _onlineNotificationThread.join(); -} - -bool RethinkDB::waitForReady() -{ - while (_ready > 0) { - if (!_waitNoticePrinted) { - _waitNoticePrinted = true; - fprintf(stderr,"[%s] NOTICE: %.10llx controller RethinkDB waiting for initial data download..." ZT_EOL_S,_timestr(),(unsigned long long)_myAddress.toInt()); - } - _readyLock.lock(); - _readyLock.unlock(); - } - return true; -} - -bool RethinkDB::isReady() -{ - return ((_ready)&&(_connected)); -} - -void RethinkDB::save(nlohmann::json *orig,nlohmann::json &record) -{ - if (!record.is_object()) // sanity check - return; - waitForReady(); - if (orig) { - if (*orig != record) { - record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1; - _commitQueue.post(new nlohmann::json(record)); - } - } else { - record["revision"] = 1; - _commitQueue.post(new nlohmann::json(record)); - } -} - -void RethinkDB::eraseNetwork(const uint64_t networkId) -{ - char tmp2[24]; - waitForReady(); - Utils::hex(networkId,tmp2); - json *tmp = new json(); - (*tmp)["id"] = tmp2; - (*tmp)["objtype"] = "_delete_network"; // pseudo-type, tells thread to delete network - _commitQueue.post(tmp); -} - -void RethinkDB::eraseMember(const uint64_t networkId,const uint64_t memberId) -{ - char tmp2[24]; - json *tmp = new json(); - waitForReady(); - Utils::hex(networkId,tmp2); - (*tmp)["nwid"] = tmp2; - Utils::hex10(memberId,tmp2); - (*tmp)["id"] = tmp2; - (*tmp)["objtype"] = "_delete_member"; // pseudo-type, tells thread to delete network - _commitQueue.post(tmp); -} - -void RethinkDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) -{ - std::lock_guard<std::mutex> l(_lastOnline_l); - std::pair<int64_t,InetAddress> &i = _lastOnline[std::pair<uint64_t,uint64_t>(networkId,memberId)]; - i.first = OSUtils::now(); - if (physicalAddress) - i.second = physicalAddress; -} - -} // namespace ZeroTier - -#endif // ZT_CONTROLLER_USE_RETHINKDB diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp deleted file mode 100644 index 60f04c5b..00000000 --- a/controller/RethinkDB.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#ifdef ZT_CONTROLLER_USE_RETHINKDB - -#ifndef ZT_CONTROLLER_RETHINKDB_HPP -#define ZT_CONTROLLER_RETHINKDB_HPP - -#include "DB.hpp" - -#define ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS 4 - -namespace ZeroTier -{ - -/** - * A controller database driver that talks to RethinkDB - * - * This is for use with ZeroTier Central. Others are free to build and use it - * but be aware that we might change it at any time. - */ -class RethinkDB : public DB -{ -public: - RethinkDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path); - virtual ~RethinkDB(); - - virtual bool waitForReady(); - virtual bool isReady(); - virtual void save(nlohmann::json *orig,nlohmann::json &record); - virtual void eraseNetwork(const uint64_t networkId); - virtual void eraseMember(const uint64_t networkId,const uint64_t memberId); - virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress); - -protected: - struct _PairHasher - { - inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); } - }; - - std::string _host; - std::string _db; - std::string _auth; - int _port; - - void *_networksDbWatcherConnection; - void *_membersDbWatcherConnection; - std::thread _networksDbWatcher; - std::thread _membersDbWatcher; - - BlockingQueue< nlohmann::json * > _commitQueue; - std::thread _commitThread[ZT_CONTROLLER_RETHINKDB_COMMIT_THREADS]; - - std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > _lastOnline; - mutable std::mutex _lastOnline_l; - std::thread _onlineNotificationThread; - - std::thread _heartbeatThread; - - mutable std::mutex _readyLock; // locked until ready - std::atomic<int> _ready,_connected,_run; - mutable volatile bool _waitNoticePrinted; -}; - -} // namespace ZeroTier - -#endif - -#endif // ZT_CONTROLLER_USE_RETHINKDB diff --git a/cycle_controllers.sh b/cycle_controllers.sh new file mode 100755 index 00000000..34acacf4 --- /dev/null +++ b/cycle_controllers.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CONTROLLERS=`kubectl get pods -o=name | grep controller | sed "s/^.\{4\}//"` + +for c in ${CONTROLLERS[@]} +do + kubectl delete pod ${c} + sleep 30 +done diff --git a/debian/changelog b/debian/changelog index 490fbedc..778a875f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +zerotier-one (1.2.99) unstable; urgency=medium + + * 1.4.0pre release + + -- Adam Ierymenko <adam.ierymenko@zerotier.com> Thu, 27 Jun 2019 01:00:00 -0700 + zerotier-one (1.2.12) unstable; urgency=medium * See https://github.com/zerotier/ZeroTierOne for release notes. diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..0f0750c6 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,19 @@ +# Dockerfile for ZeroTier Central Controllers +FROM centos:7 +MAINTAINER Adam Ierymekno <adam.ierymenko@zerotier.com>, Grant Limberg <grant.limberg@zerotier.com> + +RUN yum update -y +RUN yum install -y https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-centos10-10-2.noarch.rpm +RUN yum install -y bash postgresql10 libpqxx-devel + +RUN yum -y install epel-release && yum -y update && yum clean all +RUN yum -y install clang jemalloc jemalloc-devel + + +ADD zerotier-one /usr/local/bin/zerotier-one +RUN chmod a+x /usr/local/bin/zerotier-one + +ADD docker/main.sh / +RUN chmod a+x /main.sh + +ENTRYPOINT /main.sh diff --git a/docker/main.sh b/docker/main.sh new file mode 100644 index 00000000..b8d3b142 --- /dev/null +++ b/docker/main.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +if [ -z "$ZT_IDENTITY_PATH" ]; then + echo '*** FAILED: ZT_IDENTITY_PATH environment variable is not defined' + exit 1 +fi +if [ -z "$ZT_DB_HOST" ]; then + echo '*** FAILED: ZT_DB_HOST environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_PORT" ]; then + echo '*** FAILED: ZT_DB_PORT environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_NAME" ]; then + echo '*** FAILED: ZT_DB_NAME environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_USER" ]; then + echo '*** FAILED: ZT_DB_USER environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_PASSWORD" ]; then + echo '*** FAILED: ZT_DB_PASSWORD environment variable not defined' + exit 1 +fi + +RMQ="" +if [ "$ZT_USE_RABBITMQ" == "true" ]; then + if [ -z "$RABBITMQ_HOST" ]; then + echo '*** FAILED: RABBITMQ_HOST environment variable not defined' + exit 1 + fi + if [ -z "$RABBITMQ_PORT" ]; then + echo '*** FAILED: RABBITMQ_PORT environment variable not defined' + exit 1 + fi + if [ -z "$RABBITMQ_USERNAME" ]; then + echo '*** FAILED: RABBITMQ_USERNAME environment variable not defined' + exit 1 + fi + if [ -z "$RABBITMQ_PASSWORD" ]; then + echo '*** FAILED: RABBITMQ_PASSWORD environment variable not defined' + exit 1 + fi + RMQ=", \"rabbitmq\": { + \"host\": \"${RABBITMQ_HOST}\", + \"port\": ${RABBITMQ_PORT}, + \"username\": \"${RABBITMQ_USERNAME}\", + \"password\": \"${RABBITMQ_PASSWORD}\" + }" +fi + +mkdir -p /var/lib/zerotier-one + +pushd /var/lib/zerotier-one +ln -s $ZT_IDENTITY_PATH/identity.public identity.public +ln -s $ZT_IDENTITY_PATH/identity.secret identity.secret +popd + +DEFAULT_PORT=9993 + +echo "{ + \"settings\": { + \"portMappingEnabled\": true, + \"softwareUpdate\": \"disable\", + \"interfacePrefixBlacklist\": [ + \"inot\", + \"nat64\" + ], + \"controllerDbPath\": \"postgres:host=${ZT_DB_HOST} port=${ZT_DB_PORT} dbname=${ZT_DB_NAME} user=${ZT_DB_USER} password=${ZT_DB_PASSWORD} sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}\" + ${RMQ} + } +} +" > /var/lib/zerotier-one/local.conf + +export GLIBCXX_FORCE_NEW=1 +export GLIBCPP_FORCE_NEW=1 +export LD_PRELOAD="/usr/lib64/libjemalloc.so" +exec /usr/local/bin/zerotier-one -p${ZT_CONTROLLER_PORT:-$DEFAULT_PORT} /var/lib/zerotier-one diff --git a/ext/arm32-neon-salsa2012-asm/salsa2012.h b/ext/arm32-neon-salsa2012-asm/salsa2012.h index 95b247f2..262c9b9b 100644 --- a/ext/arm32-neon-salsa2012-asm/salsa2012.h +++ b/ext/arm32-neon-salsa2012-asm/salsa2012.h @@ -5,8 +5,10 @@ #include <sys/auxv.h> #include <asm/hwcap.h> #define zt_arm_has_neon() ((getauxval(AT_HWCAP) & HWCAP_NEON) != 0) -#else +#elif defined(__ARM_NEON__) || defined(__ARM_NEON) #define zt_arm_has_neon() (true) +#else +#define zt_arm_has_neon() (false) #endif #ifdef __cplusplus diff --git a/ext/bin/tap-mac/tap.kext/Contents/Info.plist b/ext/bin/tap-mac/tap.kext/Contents/Info.plist deleted file mode 100644 index c20eefa5..00000000 --- a/ext/bin/tap-mac/tap.kext/Contents/Info.plist +++ /dev/null @@ -1,36 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>CFBundleDevelopmentRegion</key> - <string>English</string> - <key>CFBundleExecutable</key> - <string>tap</string> - <key>CFBundleIdentifier</key> - <string>com.zerotier.tap</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>tap</string> - <key>CFBundlePackageType</key> - <string>KEXT</string> - <key>CFBundleShortVersionString</key> - <string>20150118</string> - <key>CFBundleSignature</key> - <string>????</string> - <key>CFBundleVersion</key> - <string>1.0</string> - <key>OSBundleLibraries</key> - <dict> - <key>com.apple.kpi.mach</key> - <string>8.0</string> - <key>com.apple.kpi.bsd</key> - <string>8.0</string> - <key>com.apple.kpi.libkern</key> - <string>8.0</string> - <key>com.apple.kpi.unsupported</key> - <string>8.0</string> - </dict> -</dict> -</plist> - diff --git a/ext/bin/tap-mac/tap.kext/Contents/MacOS/tap b/ext/bin/tap-mac/tap.kext/Contents/MacOS/tap Binary files differdeleted file mode 100755 index 48bf9625..00000000 --- a/ext/bin/tap-mac/tap.kext/Contents/MacOS/tap +++ /dev/null diff --git a/ext/bin/tap-mac/tap.kext/Contents/_CodeSignature/CodeResources b/ext/bin/tap-mac/tap.kext/Contents/_CodeSignature/CodeResources deleted file mode 100644 index 0710b400..00000000 --- a/ext/bin/tap-mac/tap.kext/Contents/_CodeSignature/CodeResources +++ /dev/null @@ -1,105 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>files</key> - <dict/> - <key>files2</key> - <dict/> - <key>rules</key> - <dict> - <key>^Resources/</key> - <true/> - <key>^Resources/.*\.lproj/</key> - <dict> - <key>optional</key> - <true/> - <key>weight</key> - <real>1000</real> - </dict> - <key>^Resources/.*\.lproj/locversion.plist$</key> - <dict> - <key>omit</key> - <true/> - <key>weight</key> - <real>1100</real> - </dict> - <key>^version.plist$</key> - <true/> - </dict> - <key>rules2</key> - <dict> - <key>.*\.dSYM($|/)</key> - <dict> - <key>weight</key> - <real>11</real> - </dict> - <key>^(.*/)?\.DS_Store$</key> - <dict> - <key>omit</key> - <true/> - <key>weight</key> - <real>2000</real> - </dict> - <key>^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/</key> - <dict> - <key>nested</key> - <true/> - <key>weight</key> - <real>10</real> - </dict> - <key>^.*</key> - <true/> - <key>^Info\.plist$</key> - <dict> - <key>omit</key> - <true/> - <key>weight</key> - <real>20</real> - </dict> - <key>^PkgInfo$</key> - <dict> - <key>omit</key> - <true/> - <key>weight</key> - <real>20</real> - </dict> - <key>^Resources/</key> - <dict> - <key>weight</key> - <real>20</real> - </dict> - <key>^Resources/.*\.lproj/</key> - <dict> - <key>optional</key> - <true/> - <key>weight</key> - <real>1000</real> - </dict> - <key>^Resources/.*\.lproj/locversion.plist$</key> - <dict> - <key>omit</key> - <true/> - <key>weight</key> - <real>1100</real> - </dict> - <key>^[^/]+$</key> - <dict> - <key>nested</key> - <true/> - <key>weight</key> - <real>10</real> - </dict> - <key>^embedded\.provisionprofile$</key> - <dict> - <key>weight</key> - <real>20</real> - </dict> - <key>^version\.plist$</key> - <dict> - <key>weight</key> - <real>20</real> - </dict> - </dict> -</dict> -</plist> diff --git a/ext/cpp-httplib/LICENSE b/ext/cpp-httplib/LICENSE new file mode 100644 index 00000000..3e5ed359 --- /dev/null +++ b/ext/cpp-httplib/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2017 yhirose + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/ext/cpp-httplib/README.md b/ext/cpp-httplib/README.md new file mode 100644 index 00000000..2bd23454 --- /dev/null +++ b/ext/cpp-httplib/README.md @@ -0,0 +1,259 @@ +cpp-httplib +=========== + +[](https://travis-ci.org/yhirose/cpp-httplib) +[](https://ci.appveyor.com/project/yhirose/cpp-httplib) + +A C++ header-only cross platform HTTP/HTTPS library. + +It's extremely easy to setup. Just include **httplib.h** file in your code! + +Inspired by [Sinatra](http://www.sinatrarb.com/) and [express](https://github.com/visionmedia/express). + +Server Example +-------------- + +```c++ +#include <httplib.h> + +int main(void) +{ + using namespace httplib; + + Server svr; + + svr.Get("/hi", [](const Request& req, Response& res) { + res.set_content("Hello World!", "text/plain"); + }); + + svr.Get(R"(/numbers/(\d+))", [&](const Request& req, Response& res) { + auto numbers = req.matches[1]; + res.set_content(numbers, "text/plain"); + }); + + svr.listen("localhost", 1234); +} +``` + +`Post`, `Put`, `Delete` and `Options` methods are also supported. + +### Bind a socket to multiple interfaces and any available port + +```cpp +int port = svr.bind_to_any_port("0.0.0.0"); +svr.listen_after_bind(); +``` + +### Method Chain + +```cpp +svr.Get("/get", [](const auto& req, auto& res) { + res.set_content("get", "text/plain"); + }) + .Post("/post", [](const auto& req, auto& res) { + res.set_content(req.body(), "text/plain"); + }) + .listen("localhost", 1234); +``` + +### Static File Server + +```cpp +svr.set_base_dir("./www"); +``` + +### Logging + +```cpp +svr.set_logger([](const auto& req, const auto& res) { + your_logger(req, res); +}); +``` + +### Error Handler + +```cpp +svr.set_error_handler([](const auto& req, auto& res) { + const char* fmt = "<p>Error Status: <span style='color:red;'>%d</span></p>"; + char buf[BUFSIZ]; + snprintf(buf, sizeof(buf), fmt, res.status); + res.set_content(buf, "text/html"); +}); +``` + +### 'multipart/form-data' POST data + +```cpp +svr.Post("/multipart", [&](const auto& req, auto& res) { + auto size = req.files.size(); + auto ret = req.has_file("name1")); + const auto& file = req.get_file_value("name1"); + // file.filename; + // file.content_type; + auto body = req.body.substr(file.offset, file.length)); +}) +``` + +Client Example +-------------- + +### GET + +```c++ +#include <httplib.h> +#include <iostream> + +int main(void) +{ + httplib::Client cli("localhost", 1234); + + auto res = cli.Get("/hi"); + if (res && res->status == 200) { + std::cout << res->body << std::endl; + } +} +``` + +### GET with Content Receiver + +```c++ + std::string body; + auto res = cli.Get("/large-data", [&](const char *data, size_t len) { + body.append(data, len); + }); + assert(res->body.empty()); +``` + +### POST + +```c++ +res = cli.Post("/post", "text", "text/plain"); +res = cli.Post("/person", "name=john1¬e=coder", "application/x-www-form-urlencoded"); +``` + +### POST with parameters + +```c++ +httplib::Params params; +params.emplace("name", "john"); +params.emplace("note", "coder"); + +auto res = cli.Post("/post", params); +``` + or + +```c++ +httplib::Params params{ + { "name", "john" }, + { "note", "coder" } +}; + +auto res = cli.Post("/post", params); +``` + +### PUT + +```c++ +res = cli.Put("/resource/foo", "text", "text/plain"); +``` + +### DELETE + +```c++ +res = cli.Delete("/resource/foo"); +``` + +### OPTIONS + +```c++ +res = cli.Options("*"); +res = cli.Options("/resource/foo"); +``` + +### Connection Timeout + +```c++ +httplib::Client cli("localhost", 8080, 5); // timeouts in 5 seconds +``` +### With Progress Callback + +```cpp +httplib::Client client(url, port); + +// prints: 0 / 000 bytes => 50% complete +std::shared_ptr<httplib::Response> res = + cli.Get("/", [](uint64_t len, uint64_t total) { + printf("%lld / %lld bytes => %d%% complete\n", + len, total, + (int)((len/total)*100)); + return true; // return 'false' if you want to cancel the request. + } +); +``` + + + +This feature was contributed by [underscorediscovery](https://github.com/yhirose/cpp-httplib/pull/23). + +### Basic Authentication + +```cpp +httplib::Client cli("httplib.org"); + +auto res = cli.Get("/basic-auth/hello/world", { + httplib::make_basic_authentication_header("hello", "world") +}); +// res->status should be 200 +// res->body should be "{\n \"authenticated\": true, \n \"user\": \"hello\"\n}\n". +``` + +### Range + +```cpp +httplib::Client cli("httpbin.org"); + +auto res = cli.Get("/range/32", { + httplib::make_range_header(1, 10) // 'Range: bytes=1-10' +}); +// res->status should be 206. +// res->body should be "bcdefghijk". +``` + +OpenSSL Support +--------------- + +SSL support is available with `CPPHTTPLIB_OPENSSL_SUPPORT`. `libssl` and `libcrypto` should be linked. + +```c++ +#define CPPHTTPLIB_OPENSSL_SUPPORT + +SSLServer svr("./cert.pem", "./key.pem"); + +SSLClient cli("localhost", 8080); +cli.set_ca_cert_path("./ca-bundle.crt"); +cli.enable_server_certificate_verification(true); +``` + +Zlib Support +------------ + +'gzip' compression is available with `CPPHTTPLIB_ZLIB_SUPPORT`. + +The server applies gzip compression to the following MIME type contents: + + * all text types + * image/svg+xml + * application/javascript + * application/json + * application/xml + * application/xhtml+xml + +NOTE +---- + +g++ 4.8 cannot build this library since `<regex>` in g++4.8 is [broken](https://stackoverflow.com/questions/12530406/is-gcc-4-8-or-earlier-buggy-about-regular-expressions). + +License +------- + +MIT license (© 2019 Yuji Hirose) diff --git a/ext/cpp-httplib/httplib.h b/ext/cpp-httplib/httplib.h new file mode 100644 index 00000000..5adfc2af --- /dev/null +++ b/ext/cpp-httplib/httplib.h @@ -0,0 +1,2779 @@ +// +// httplib.h +// +// Copyright (c) 2019 Yuji Hirose. All rights reserved. +// MIT License +// + +#ifndef CPPHTTPLIB_HTTPLIB_H +#define CPPHTTPLIB_HTTPLIB_H + +#ifdef _WIN32 +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif //_CRT_SECURE_NO_WARNINGS + +#ifndef _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_NONSTDC_NO_DEPRECATE +#endif //_CRT_NONSTDC_NO_DEPRECATE + +#if defined(_MSC_VER) && _MSC_VER < 1900 +#define snprintf _snprintf_s +#endif // _MSC_VER + +#ifndef S_ISREG +#define S_ISREG(m) (((m)&S_IFREG) == S_IFREG) +#endif // S_ISREG + +#ifndef S_ISDIR +#define S_ISDIR(m) (((m)&S_IFDIR) == S_IFDIR) +#endif // S_ISDIR + +#ifndef NOMINMAX +#define NOMINMAX +#endif // NOMINMAX + +#include <io.h> +#include <winsock2.h> +#include <ws2tcpip.h> + +#pragma comment(lib, "ws2_32.lib") + +#ifndef strcasecmp +#define strcasecmp _stricmp +#endif // strcasecmp + +typedef SOCKET socket_t; +#else +#include <arpa/inet.h> +#include <cstring> +#include <netdb.h> +#include <netinet/in.h> +#include <pthread.h> +#include <signal.h> +#include <sys/select.h> +#include <sys/socket.h> +#include <unistd.h> + +typedef int socket_t; +#define INVALID_SOCKET (-1) +#endif //_WIN32 + +#include <assert.h> +#include <atomic> +#include <fcntl.h> +#include <fstream> +#include <functional> +#include <map> +#include <memory> +#include <mutex> +#include <regex> +#include <string> +#include <sys/stat.h> +#include <thread> + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +#include <openssl/err.h> +#include <openssl/ssl.h> +#include <openssl/x509v3.h> + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +inline const unsigned char *ASN1_STRING_get0_data(const ASN1_STRING *asn1) { + return M_ASN1_STRING_data(asn1); +} +#endif +#endif + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +#include <zlib.h> +#endif + +/* + * Configuration + */ +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND 5 +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_USECOND 0 +#define CPPHTTPLIB_KEEPALIVE_MAX_COUNT 5 +#define CPPHTTPLIB_READ_TIMEOUT_SECOND 5 +#define CPPHTTPLIB_READ_TIMEOUT_USECOND 0 +#define CPPHTTPLIB_REQUEST_URI_MAX_LENGTH 8192 +#define CPPHTTPLIB_PAYLOAD_MAX_LENGTH (std::numeric_limits<size_t>::max)() +#define CPPHTTPLIB_RECV_BUFSIZ size_t(4096u) + +namespace httplib { + +namespace detail { + +struct ci { + bool operator()(const std::string &s1, const std::string &s2) const { + return std::lexicographical_compare( + s1.begin(), s1.end(), s2.begin(), s2.end(), + [](char c1, char c2) { return ::tolower(c1) < ::tolower(c2); }); + } +}; + +} // namespace detail + +enum class HttpVersion { v1_0 = 0, v1_1 }; + +typedef std::multimap<std::string, std::string, detail::ci> Headers; + +template <typename uint64_t, typename... Args> +std::pair<std::string, std::string> make_range_header(uint64_t value, + Args... args); + +typedef std::multimap<std::string, std::string> Params; +typedef std::smatch Match; + +typedef std::function<std::string(uint64_t offset)> ContentProducer; +typedef std::function<void(const char *data, size_t len)> ContentReceiver; +typedef std::function<bool(uint64_t current, uint64_t total)> Progress; + +struct MultipartFile { + std::string filename; + std::string content_type; + size_t offset = 0; + size_t length = 0; +}; +typedef std::multimap<std::string, MultipartFile> MultipartFiles; + +struct Request { + std::string version; + std::string method; + std::string target; + std::string path; + Headers headers; + std::string body; + Params params; + MultipartFiles files; + Match matches; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + const SSL *ssl; +#endif + + bool has_header(const char *key) const; + std::string get_header_value(const char *key, size_t id = 0) const; + size_t get_header_value_count(const char *key) const; + void set_header(const char *key, const char *val); + + bool has_param(const char *key) const; + std::string get_param_value(const char *key, size_t id = 0) const; + size_t get_param_value_count(const char *key) const; + + bool has_file(const char *key) const; + MultipartFile get_file_value(const char *key) const; +}; + +struct Response { + std::string version; + int status; + Headers headers; + std::string body; + + ContentProducer content_producer; + ContentReceiver content_receiver; + Progress progress; + + bool has_header(const char *key) const; + std::string get_header_value(const char *key, size_t id = 0) const; + size_t get_header_value_count(const char *key) const; + void set_header(const char *key, const char *val); + + void set_redirect(const char *uri); + void set_content(const char *s, size_t n, const char *content_type); + void set_content(const std::string &s, const char *content_type); + + Response() : status(-1) {} +}; + +class Stream { +public: + virtual ~Stream() {} + virtual int read(char *ptr, size_t size) = 0; + virtual int write(const char *ptr, size_t size1) = 0; + virtual int write(const char *ptr) = 0; + virtual std::string get_remote_addr() const = 0; + + template <typename... Args> + void write_format(const char *fmt, const Args &... args); +}; + +class SocketStream : public Stream { +public: + SocketStream(socket_t sock); + virtual ~SocketStream(); + + virtual int read(char *ptr, size_t size); + virtual int write(const char *ptr, size_t size); + virtual int write(const char *ptr); + virtual std::string get_remote_addr() const; + +private: + socket_t sock_; +}; + +class BufferStream : public Stream { +public: + BufferStream() {} + virtual ~BufferStream() {} + + virtual int read(char *ptr, size_t size); + virtual int write(const char *ptr, size_t size); + virtual int write(const char *ptr); + virtual std::string get_remote_addr() const; + + const std::string &get_buffer() const; + +private: + std::string buffer; +}; + +class Server { +public: + typedef std::function<void(const Request &, Response &)> Handler; + typedef std::function<void(const Request &, const Response &)> Logger; + + Server(); + + virtual ~Server(); + + virtual bool is_valid() const; + + Server &Get(const char *pattern, Handler handler); + Server &Post(const char *pattern, Handler handler); + + Server &Put(const char *pattern, Handler handler); + Server &Patch(const char *pattern, Handler handler); + Server &Delete(const char *pattern, Handler handler); + Server &Options(const char *pattern, Handler handler); + + bool set_base_dir(const char *path); + + void set_error_handler(Handler handler); + void set_logger(Logger logger); + + void set_keep_alive_max_count(size_t count); + void set_payload_max_length(uint64_t length); + + int bind_to_any_port(const char *host, int socket_flags = 0); + bool listen_after_bind(); + + bool listen(const char *host, int port, int socket_flags = 0); + + bool is_running() const; + void stop(); + +protected: + bool process_request(Stream &strm, bool last_connection, + bool &connection_close, + std::function<void(Request &)> setup_request = nullptr); + + size_t keep_alive_max_count_; + size_t payload_max_length_; + +private: + typedef std::vector<std::pair<std::regex, Handler>> Handlers; + + socket_t create_server_socket(const char *host, int port, + int socket_flags) const; + int bind_internal(const char *host, int port, int socket_flags); + bool listen_internal(); + + bool routing(Request &req, Response &res); + bool handle_file_request(Request &req, Response &res); + bool dispatch_request(Request &req, Response &res, Handlers &handlers); + + bool parse_request_line(const char *s, Request &req); + void write_response(Stream &strm, bool last_connection, const Request &req, + Response &res); + + virtual bool read_and_close_socket(socket_t sock); + + std::atomic<bool> is_running_; + std::atomic<socket_t> svr_sock_; + std::string base_dir_; + Handlers get_handlers_; + Handlers post_handlers_; + Handlers put_handlers_; + Handlers patch_handlers_; + Handlers delete_handlers_; + Handlers options_handlers_; + Handler error_handler_; + Logger logger_; + + // TODO: Use thread pool... + std::mutex running_threads_mutex_; + int running_threads_; +}; + +class Client { +public: + Client(const char *host, int port = 80, time_t timeout_sec = 300); + + virtual ~Client(); + + virtual bool is_valid() const; + + std::shared_ptr<Response> Get(const char *path, Progress progress = nullptr); + std::shared_ptr<Response> Get(const char *path, const Headers &headers, + Progress progress = nullptr); + + std::shared_ptr<Response> Get(const char *path, + ContentReceiver content_receiver, + Progress progress = nullptr); + std::shared_ptr<Response> Get(const char *path, const Headers &headers, + ContentReceiver content_receiver, + Progress progress = nullptr); + + std::shared_ptr<Response> Head(const char *path); + std::shared_ptr<Response> Head(const char *path, const Headers &headers); + + std::shared_ptr<Response> Post(const char *path, const std::string &body, + const char *content_type); + std::shared_ptr<Response> Post(const char *path, const Headers &headers, + const std::string &body, + const char *content_type); + + std::shared_ptr<Response> Post(const char *path, const Params ¶ms); + std::shared_ptr<Response> Post(const char *path, const Headers &headers, + const Params ¶ms); + + std::shared_ptr<Response> Put(const char *path, const std::string &body, + const char *content_type); + std::shared_ptr<Response> Put(const char *path, const Headers &headers, + const std::string &body, + const char *content_type); + + std::shared_ptr<Response> Patch(const char *path, const std::string &body, + const char *content_type); + std::shared_ptr<Response> Patch(const char *path, const Headers &headers, + const std::string &body, + const char *content_type); + + std::shared_ptr<Response> Delete(const char *path, + const std::string &body = std::string(), + const char *content_type = nullptr); + std::shared_ptr<Response> Delete(const char *path, const Headers &headers, + const std::string &body = std::string(), + const char *content_type = nullptr); + + std::shared_ptr<Response> Options(const char *path); + std::shared_ptr<Response> Options(const char *path, const Headers &headers); + + bool send(Request &req, Response &res); + +protected: + bool process_request(Stream &strm, Request &req, Response &res, + bool &connection_close); + + const std::string host_; + const int port_; + time_t timeout_sec_; + const std::string host_and_port_; + +private: + socket_t create_client_socket() const; + bool read_response_line(Stream &strm, Response &res); + void write_request(Stream &strm, Request &req); + + virtual bool read_and_close_socket(socket_t sock, Request &req, + Response &res); + virtual bool is_ssl() const; +}; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +class SSLSocketStream : public Stream { +public: + SSLSocketStream(socket_t sock, SSL *ssl); + virtual ~SSLSocketStream(); + + virtual int read(char *ptr, size_t size); + virtual int write(const char *ptr, size_t size); + virtual int write(const char *ptr); + virtual std::string get_remote_addr() const; + +private: + socket_t sock_; + SSL *ssl_; +}; + +class SSLServer : public Server { +public: + SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path = nullptr, + const char *client_ca_cert_dir_path = nullptr); + + virtual ~SSLServer(); + + virtual bool is_valid() const; + +private: + virtual bool read_and_close_socket(socket_t sock); + + SSL_CTX *ctx_; + std::mutex ctx_mutex_; +}; + +class SSLClient : public Client { +public: + SSLClient(const char *host, int port = 443, time_t timeout_sec = 300, + const char *client_cert_path = nullptr, + const char *client_key_path = nullptr); + + virtual ~SSLClient(); + + virtual bool is_valid() const; + + void set_ca_cert_path(const char *ca_ceert_file_path, + const char *ca_cert_dir_path = nullptr); + void enable_server_certificate_verification(bool enabled); + + long get_openssl_verify_result() const; + +private: + virtual bool read_and_close_socket(socket_t sock, Request &req, + Response &res); + virtual bool is_ssl() const; + + bool verify_host(X509 *server_cert) const; + bool verify_host_with_subject_alt_name(X509 *server_cert) const; + bool verify_host_with_common_name(X509 *server_cert) const; + bool check_host_name(const char *pattern, size_t pattern_len) const; + + SSL_CTX *ctx_; + std::mutex ctx_mutex_; + std::vector<std::string> host_components_; + std::string ca_cert_file_path_; + std::string ca_cert_dir_path_; + bool server_certificate_verification_ = false; + long verify_result_ = 0; +}; +#endif + +/* + * Implementation + */ +namespace detail { + +inline bool is_hex(char c, int &v) { + if (0x20 <= c && isdigit(c)) { + v = c - '0'; + return true; + } else if ('A' <= c && c <= 'F') { + v = c - 'A' + 10; + return true; + } else if ('a' <= c && c <= 'f') { + v = c - 'a' + 10; + return true; + } + return false; +} + +inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, + int &val) { + if (i >= s.size()) { return false; } + + val = 0; + for (; cnt; i++, cnt--) { + if (!s[i]) { return false; } + int v = 0; + if (is_hex(s[i], v)) { + val = val * 16 + v; + } else { + return false; + } + } + return true; +} + +inline std::string from_i_to_hex(uint64_t n) { + const char *charset = "0123456789abcdef"; + std::string ret; + do { + ret = charset[n & 15] + ret; + n >>= 4; + } while (n > 0); + return ret; +} + +inline size_t to_utf8(int code, char *buff) { + if (code < 0x0080) { + buff[0] = (code & 0x7F); + return 1; + } else if (code < 0x0800) { + buff[0] = (0xC0 | ((code >> 6) & 0x1F)); + buff[1] = (0x80 | (code & 0x3F)); + return 2; + } else if (code < 0xD800) { + buff[0] = (0xE0 | ((code >> 12) & 0xF)); + buff[1] = (0x80 | ((code >> 6) & 0x3F)); + buff[2] = (0x80 | (code & 0x3F)); + return 3; + } else if (code < 0xE000) { // D800 - DFFF is invalid... + return 0; + } else if (code < 0x10000) { + buff[0] = (0xE0 | ((code >> 12) & 0xF)); + buff[1] = (0x80 | ((code >> 6) & 0x3F)); + buff[2] = (0x80 | (code & 0x3F)); + return 3; + } else if (code < 0x110000) { + buff[0] = (0xF0 | ((code >> 18) & 0x7)); + buff[1] = (0x80 | ((code >> 12) & 0x3F)); + buff[2] = (0x80 | ((code >> 6) & 0x3F)); + buff[3] = (0x80 | (code & 0x3F)); + return 4; + } + + // NOTREACHED + return 0; +} + +// NOTE: This code came up with the following stackoverflow post: +// https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c +inline std::string base64_encode(const std::string &in) { + static const auto lookup = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::string out; + out.reserve(in.size()); + + int val = 0; + int valb = -6; + + for (uint8_t c : in) { + val = (val << 8) + c; + valb += 8; + while (valb >= 0) { + out.push_back(lookup[(val >> valb) & 0x3F]); + valb -= 6; + } + } + + if (valb > -6) { + out.push_back(lookup[((val << 8) >> (valb + 8)) & 0x3F]); + } + + while (out.size() % 4) { + out.push_back('='); + } + + return out; +} + +inline bool is_file(const std::string &path) { + struct stat st; + return stat(path.c_str(), &st) >= 0 && S_ISREG(st.st_mode); +} + +inline bool is_dir(const std::string &path) { + struct stat st; + return stat(path.c_str(), &st) >= 0 && S_ISDIR(st.st_mode); +} + +inline bool is_valid_path(const std::string &path) { + size_t level = 0; + size_t i = 0; + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + + while (i < path.size()) { + // Read component + auto beg = i; + while (i < path.size() && path[i] != '/') { + i++; + } + + auto len = i - beg; + assert(len > 0); + + if (!path.compare(beg, len, ".")) { + ; + } else if (!path.compare(beg, len, "..")) { + if (level == 0) { return false; } + level--; + } else { + level++; + } + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + } + + return true; +} + +inline void read_file(const std::string &path, std::string &out) { + std::ifstream fs(path, std::ios_base::binary); + fs.seekg(0, std::ios_base::end); + auto size = fs.tellg(); + fs.seekg(0); + out.resize(static_cast<size_t>(size)); + fs.read(&out[0], size); +} + +inline std::string file_extension(const std::string &path) { + std::smatch m; + auto pat = std::regex("\\.([a-zA-Z0-9]+)$"); + if (std::regex_search(path, m, pat)) { return m[1].str(); } + return std::string(); +} + +template <class Fn> void split(const char *b, const char *e, char d, Fn fn) { + int i = 0; + int beg = 0; + + while (e ? (b + i != e) : (b[i] != '\0')) { + if (b[i] == d) { + fn(&b[beg], &b[i]); + beg = i + 1; + } + i++; + } + + if (i) { fn(&b[beg], &b[i]); } +} + +// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer` +// to store data. The call can set memory on stack for performance. +class stream_line_reader { +public: + stream_line_reader(Stream &strm, char *fixed_buffer, size_t fixed_buffer_size) + : strm_(strm), fixed_buffer_(fixed_buffer), + fixed_buffer_size_(fixed_buffer_size) {} + + const char *ptr() const { + if (glowable_buffer_.empty()) { + return fixed_buffer_; + } else { + return glowable_buffer_.data(); + } + } + + size_t size() const { + if (glowable_buffer_.empty()) { + return fixed_buffer_used_size_; + } else { + return glowable_buffer_.size(); + } + } + + bool getline() { + fixed_buffer_used_size_ = 0; + glowable_buffer_.clear(); + + for (size_t i = 0;; i++) { + char byte; + auto n = strm_.read(&byte, 1); + + if (n < 0) { + return false; + } else if (n == 0) { + if (i == 0) { + return false; + } else { + break; + } + } + + append(byte); + + if (byte == '\n') { break; } + } + + return true; + } + +private: + void append(char c) { + if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) { + fixed_buffer_[fixed_buffer_used_size_++] = c; + fixed_buffer_[fixed_buffer_used_size_] = '\0'; + } else { + if (glowable_buffer_.empty()) { + assert(fixed_buffer_[fixed_buffer_used_size_] == '\0'); + glowable_buffer_.assign(fixed_buffer_, fixed_buffer_used_size_); + } + glowable_buffer_ += c; + } + } + + Stream &strm_; + char *fixed_buffer_; + const size_t fixed_buffer_size_; + size_t fixed_buffer_used_size_; + std::string glowable_buffer_; +}; + +inline int close_socket(socket_t sock) { +#ifdef _WIN32 + return closesocket(sock); +#else + return close(sock); +#endif +} + +inline int select_read(socket_t sock, time_t sec, time_t usec) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(sock, &fds); + + timeval tv; + tv.tv_sec = static_cast<long>(sec); + tv.tv_usec = static_cast<long>(usec); + + return select(static_cast<int>(sock + 1), &fds, nullptr, nullptr, &tv); +} + +inline bool wait_until_socket_is_ready(socket_t sock, time_t sec, time_t usec) { + fd_set fdsr; + FD_ZERO(&fdsr); + FD_SET(sock, &fdsr); + + auto fdsw = fdsr; + auto fdse = fdsr; + + timeval tv; + tv.tv_sec = static_cast<long>(sec); + tv.tv_usec = static_cast<long>(usec); + + if (select(static_cast<int>(sock + 1), &fdsr, &fdsw, &fdse, &tv) < 0) { + return false; + } else if (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw)) { + int error = 0; + socklen_t len = sizeof(error); + if (getsockopt(sock, SOL_SOCKET, SO_ERROR, (char *)&error, &len) < 0 || + error) { + return false; + } + } else { + return false; + } + + return true; +} + +template <typename T> +inline bool read_and_close_socket(socket_t sock, size_t keep_alive_max_count, + T callback) { + bool ret = false; + + if (keep_alive_max_count > 0) { + auto count = keep_alive_max_count; + while (count > 0 && + detail::select_read(sock, CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND, + CPPHTTPLIB_KEEPALIVE_TIMEOUT_USECOND) > 0) { + SocketStream strm(sock); + auto last_connection = count == 1; + auto connection_close = false; + + ret = callback(strm, last_connection, connection_close); + if (!ret || connection_close) { break; } + + count--; + } + } else { + SocketStream strm(sock); + auto dummy_connection_close = false; + ret = callback(strm, true, dummy_connection_close); + } + + close_socket(sock); + return ret; +} + +inline int shutdown_socket(socket_t sock) { +#ifdef _WIN32 + return shutdown(sock, SD_BOTH); +#else + return shutdown(sock, SHUT_RDWR); +#endif +} + +template <typename Fn> +socket_t create_socket(const char *host, int port, Fn fn, + int socket_flags = 0) { +#ifdef _WIN32 +#define SO_SYNCHRONOUS_NONALERT 0x20 +#define SO_OPENTYPE 0x7008 + + int opt = SO_SYNCHRONOUS_NONALERT; + setsockopt(INVALID_SOCKET, SOL_SOCKET, SO_OPENTYPE, (char *)&opt, + sizeof(opt)); +#endif + + // Get address info + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = socket_flags; + hints.ai_protocol = 0; + + auto service = std::to_string(port); + + if (getaddrinfo(host, service.c_str(), &hints, &result)) { + return INVALID_SOCKET; + } + + for (auto rp = result; rp; rp = rp->ai_next) { + // Create a socket +#ifdef _WIN32 + auto sock = WSASocketW(rp->ai_family, rp->ai_socktype, rp->ai_protocol, + nullptr, 0, WSA_FLAG_NO_HANDLE_INHERIT); +#else + auto sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); +#endif + if (sock == INVALID_SOCKET) { continue; } + +#ifndef _WIN32 + if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { continue; } +#endif + + // Make 'reuse address' option available + int yes = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&yes, sizeof(yes)); +#ifdef SO_REUSEPORT + setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, (char *)&yes, sizeof(yes)); +#endif + + // bind or connect + if (fn(sock, *rp)) { + freeaddrinfo(result); + return sock; + } + + close_socket(sock); + } + + freeaddrinfo(result); + return INVALID_SOCKET; +} + +inline void set_nonblocking(socket_t sock, bool nonblocking) { +#ifdef _WIN32 + auto flags = nonblocking ? 1UL : 0UL; + ioctlsocket(sock, FIONBIO, &flags); +#else + auto flags = fcntl(sock, F_GETFL, 0); + fcntl(sock, F_SETFL, + nonblocking ? (flags | O_NONBLOCK) : (flags & (~O_NONBLOCK))); +#endif +} + +inline bool is_connection_error() { +#ifdef _WIN32 + return WSAGetLastError() != WSAEWOULDBLOCK; +#else + return errno != EINPROGRESS; +#endif +} + +inline std::string get_remote_addr(socket_t sock) { + struct sockaddr_storage addr; + socklen_t len = sizeof(addr); + + if (!getpeername(sock, (struct sockaddr *)&addr, &len)) { + char ipstr[NI_MAXHOST]; + + if (!getnameinfo((struct sockaddr *)&addr, len, ipstr, sizeof(ipstr), + nullptr, 0, NI_NUMERICHOST)) { + return ipstr; + } + } + + return std::string(); +} + +inline const char *find_content_type(const std::string &path) { + auto ext = file_extension(path); + if (ext == "txt") { + return "text/plain"; + } else if (ext == "html") { + return "text/html"; + } else if (ext == "css") { + return "text/css"; + } else if (ext == "jpeg" || ext == "jpg") { + return "image/jpg"; + } else if (ext == "png") { + return "image/png"; + } else if (ext == "gif") { + return "image/gif"; + } else if (ext == "svg") { + return "image/svg+xml"; + } else if (ext == "ico") { + return "image/x-icon"; + } else if (ext == "json") { + return "application/json"; + } else if (ext == "pdf") { + return "application/pdf"; + } else if (ext == "js") { + return "application/javascript"; + } else if (ext == "xml") { + return "application/xml"; + } else if (ext == "xhtml") { + return "application/xhtml+xml"; + } + return nullptr; +} + +inline const char *status_message(int status) { + switch (status) { + case 200: return "OK"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 303: return "See Other"; + case 304: return "Not Modified"; + case 400: return "Bad Request"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 413: return "Payload Too Large"; + case 414: return "Request-URI Too Long"; + case 415: return "Unsupported Media Type"; + default: + case 500: return "Internal Server Error"; + } +} + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +inline bool can_compress(const std::string &content_type) { + return !content_type.find("text/") || content_type == "image/svg+xml" || + content_type == "application/javascript" || + content_type == "application/json" || + content_type == "application/xml" || + content_type == "application/xhtml+xml"; +} + +inline bool compress(std::string &content) { + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + auto ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, + Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { return false; } + + strm.avail_in = content.size(); + strm.next_in = (Bytef *)content.data(); + + std::string compressed; + + const auto bufsiz = 16384; + char buff[bufsiz]; + do { + strm.avail_out = bufsiz; + strm.next_out = (Bytef *)buff; + ret = deflate(&strm, Z_FINISH); + assert(ret != Z_STREAM_ERROR); + compressed.append(buff, bufsiz - strm.avail_out); + } while (strm.avail_out == 0); + + assert(ret == Z_STREAM_END); + assert(strm.avail_in == 0); + + content.swap(compressed); + + deflateEnd(&strm); + return true; +} + +class decompressor { +public: + decompressor() { + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + // 15 is the value of wbits, which should be at the maximum possible value + // to ensure that any gzip stream can be decoded. The offset of 16 specifies + // that the stream to decompress will be formatted with a gzip wrapper. + is_valid_ = inflateInit2(&strm, 16 + 15) == Z_OK; + } + + ~decompressor() { inflateEnd(&strm); } + + bool is_valid() const { return is_valid_; } + + template <typename T> + bool decompress(const char *data, size_t data_len, T callback) { + int ret = Z_OK; + std::string decompressed; + + // strm.avail_in = content.size(); + // strm.next_in = (Bytef *)content.data(); + strm.avail_in = data_len; + strm.next_in = (Bytef *)data; + + const auto bufsiz = 16384; + char buff[bufsiz]; + do { + strm.avail_out = bufsiz; + strm.next_out = (Bytef *)buff; + + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: inflateEnd(&strm); return false; + } + + decompressed.append(buff, bufsiz - strm.avail_out); + } while (strm.avail_out == 0); + + if (ret == Z_STREAM_END) { + callback(decompressed.data(), decompressed.size()); + return true; + } + + return false; + } + +private: + bool is_valid_; + z_stream strm; +}; +#endif + +inline bool has_header(const Headers &headers, const char *key) { + return headers.find(key) != headers.end(); +} + +inline const char *get_header_value(const Headers &headers, const char *key, + size_t id = 0, const char *def = nullptr) { + auto it = headers.find(key); + std::advance(it, id); + if (it != headers.end()) { return it->second.c_str(); } + return def; +} + +inline uint64_t get_header_value_uint64(const Headers &headers, const char *key, + int def = 0) { + auto it = headers.find(key); + if (it != headers.end()) { + return std::strtoull(it->second.data(), nullptr, 10); + } + return def; +} + +inline bool read_headers(Stream &strm, Headers &headers) { + static std::regex re(R"((.+?):\s*(.+?)\s*\r\n)"); + + const auto bufsiz = 2048; + char buf[bufsiz]; + + stream_line_reader reader(strm, buf, bufsiz); + + for (;;) { + if (!reader.getline()) { return false; } + if (!strcmp(reader.ptr(), "\r\n")) { break; } + std::cmatch m; + if (std::regex_match(reader.ptr(), m, re)) { + auto key = std::string(m[1]); + auto val = std::string(m[2]); + headers.emplace(key, val); + } + } + + return true; +} + +template <typename T> +inline bool read_content_with_length(Stream &strm, size_t len, + Progress progress, T callback) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + + size_t r = 0; + while (r < len) { + auto n = strm.read(buf, std::min((len - r), CPPHTTPLIB_RECV_BUFSIZ)); + if (n <= 0) { return false; } + + callback(buf, n); + + r += n; + + if (progress) { + if (!progress(r, len)) { return false; } + } + } + + return true; +} + +inline void skip_content_with_length(Stream &strm, size_t len) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + size_t r = 0; + while (r < len) { + auto n = strm.read(buf, std::min((len - r), CPPHTTPLIB_RECV_BUFSIZ)); + if (n <= 0) { return; } + r += n; + } +} + +template <typename T> +inline bool read_content_without_length(Stream &strm, T callback) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + for (;;) { + auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ); + if (n < 0) { + return false; + } else if (n == 0) { + return true; + } + callback(buf, n); + } + + return true; +} + +template <typename T> +inline bool read_content_chunked(Stream &strm, T callback) { + const auto bufsiz = 16; + char buf[bufsiz]; + + stream_line_reader reader(strm, buf, bufsiz); + + if (!reader.getline()) { return false; } + + auto chunk_len = std::stoi(reader.ptr(), 0, 16); + + while (chunk_len > 0) { + if (!read_content_with_length(strm, chunk_len, nullptr, callback)) { + return false; + } + + if (!reader.getline()) { return false; } + + if (strcmp(reader.ptr(), "\r\n")) { break; } + + if (!reader.getline()) { return false; } + + chunk_len = std::stoi(reader.ptr(), 0, 16); + } + + if (chunk_len == 0) { + // Reader terminator after chunks + if (!reader.getline() || strcmp(reader.ptr(), "\r\n")) return false; + } + + return true; +} + +inline bool is_chunked_transfer_encoding(const Headers &headers) { + return !strcasecmp(get_header_value(headers, "Transfer-Encoding", 0, ""), + "chunked"); +} + +template <typename T, typename U> +bool read_content(Stream &strm, T &x, uint64_t payload_max_length, int &status, + Progress progress, U callback) { + + ContentReceiver out = [&](const char *buf, size_t n) { callback(buf, n); }; + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + detail::decompressor decompressor; + + if (!decompressor.is_valid()) { + status = 500; + return false; + } + + if (x.get_header_value("Content-Encoding") == "gzip") { + out = [&](const char *buf, size_t n) { + decompressor.decompress( + buf, n, [&](const char *buf, size_t n) { callback(buf, n); }); + }; + } +#else + if (x.get_header_value("Content-Encoding") == "gzip") { + status = 415; + return false; + } +#endif + + auto ret = true; + auto exceed_payload_max_length = false; + + if (is_chunked_transfer_encoding(x.headers)) { + ret = read_content_chunked(strm, out); + } else if (!has_header(x.headers, "Content-Length")) { + ret = read_content_without_length(strm, out); + } else { + auto len = get_header_value_uint64(x.headers, "Content-Length", 0); + if (len > 0) { + if ((len > payload_max_length) || + // For 32-bit platform + (sizeof(size_t) < sizeof(uint64_t) && + len > std::numeric_limits<size_t>::max())) { + exceed_payload_max_length = true; + skip_content_with_length(strm, len); + ret = false; + } else { + ret = read_content_with_length(strm, len, progress, out); + } + } + } + + if (!ret) { status = exceed_payload_max_length ? 413 : 400; } + + return ret; +} + +template <typename T> inline void write_headers(Stream &strm, const T &info) { + for (const auto &x : info.headers) { + strm.write_format("%s: %s\r\n", x.first.c_str(), x.second.c_str()); + } + strm.write("\r\n"); +} + +template <typename T> +inline void write_content_chunked(Stream &strm, const T &x) { + auto chunked_response = !x.has_header("Content-Length"); + uint64_t offset = 0; + auto data_available = true; + while (data_available) { + auto chunk = x.content_producer(offset); + offset += chunk.size(); + data_available = !chunk.empty(); + + // Emit chunked response header and footer for each chunk + if (chunked_response) { + chunk = from_i_to_hex(chunk.size()) + "\r\n" + chunk + "\r\n"; + } + + if (strm.write(chunk.c_str(), chunk.size()) < 0) { + break; // Stop on error + } + } +} + +inline std::string encode_url(const std::string &s) { + std::string result; + + for (auto i = 0; s[i]; i++) { + switch (s[i]) { + case ' ': result += "%20"; break; + case '+': result += "%2B"; break; + case '\r': result += "%0D"; break; + case '\n': result += "%0A"; break; + case '\'': result += "%27"; break; + case ',': result += "%2C"; break; + case ':': result += "%3A"; break; + case ';': result += "%3B"; break; + default: + auto c = static_cast<uint8_t>(s[i]); + if (c >= 0x80) { + result += '%'; + char hex[4]; + size_t len = snprintf(hex, sizeof(hex) - 1, "%02X", c); + assert(len == 2); + result.append(hex, len); + } else { + result += s[i]; + } + break; + } + } + + return result; +} + +inline std::string decode_url(const std::string &s) { + std::string result; + + for (size_t i = 0; i < s.size(); i++) { + if (s[i] == '%' && i + 1 < s.size()) { + if (s[i + 1] == 'u') { + int val = 0; + if (from_hex_to_i(s, i + 2, 4, val)) { + // 4 digits Unicode codes + char buff[4]; + size_t len = to_utf8(val, buff); + if (len > 0) { result.append(buff, len); } + i += 5; // 'u0000' + } else { + result += s[i]; + } + } else { + int val = 0; + if (from_hex_to_i(s, i + 1, 2, val)) { + // 2 digits hex codes + result += val; + i += 2; // '00' + } else { + result += s[i]; + } + } + } else if (s[i] == '+') { + result += ' '; + } else { + result += s[i]; + } + } + + return result; +} + +inline void parse_query_text(const std::string &s, Params ¶ms) { + split(&s[0], &s[s.size()], '&', [&](const char *b, const char *e) { + std::string key; + std::string val; + split(b, e, '=', [&](const char *b, const char *e) { + if (key.empty()) { + key.assign(b, e); + } else { + val.assign(b, e); + } + }); + params.emplace(key, decode_url(val)); + }); +} + +inline bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary) { + auto pos = content_type.find("boundary="); + if (pos == std::string::npos) { return false; } + + boundary = content_type.substr(pos + 9); + return true; +} + +inline bool parse_multipart_formdata(const std::string &boundary, + const std::string &body, + MultipartFiles &files) { + static std::string dash = "--"; + static std::string crlf = "\r\n"; + + static std::regex re_content_type("Content-Type: (.*?)", + std::regex_constants::icase); + + static std::regex re_content_disposition( + "Content-Disposition: form-data; name=\"(.*?)\"(?:; filename=\"(.*?)\")?", + std::regex_constants::icase); + + auto dash_boundary = dash + boundary; + + auto pos = body.find(dash_boundary); + if (pos != 0) { return false; } + + pos += dash_boundary.size(); + + auto next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + pos = next_pos + crlf.size(); + + while (pos < body.size()) { + next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + std::string name; + MultipartFile file; + + auto header = body.substr(pos, (next_pos - pos)); + + while (pos != next_pos) { + std::smatch m; + if (std::regex_match(header, m, re_content_type)) { + file.content_type = m[1]; + } else if (std::regex_match(header, m, re_content_disposition)) { + name = m[1]; + file.filename = m[2]; + } + + pos = next_pos + crlf.size(); + + next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + header = body.substr(pos, (next_pos - pos)); + } + + pos = next_pos + crlf.size(); + + next_pos = body.find(crlf + dash_boundary, pos); + + if (next_pos == std::string::npos) { return false; } + + file.offset = pos; + file.length = next_pos - pos; + + pos = next_pos + crlf.size() + dash_boundary.size(); + + next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + files.emplace(name, file); + + pos = next_pos + crlf.size(); + } + + return true; +} + +inline std::string to_lower(const char *beg, const char *end) { + std::string out; + auto it = beg; + while (it != end) { + out += ::tolower(*it); + it++; + } + return out; +} + +inline void make_range_header_core(std::string &) {} + +template <typename uint64_t> +inline void make_range_header_core(std::string &field, uint64_t value) { + if (!field.empty()) { field += ", "; } + field += std::to_string(value) + "-"; +} + +template <typename uint64_t, typename... Args> +inline void make_range_header_core(std::string &field, uint64_t value1, + uint64_t value2, Args... args) { + if (!field.empty()) { field += ", "; } + field += std::to_string(value1) + "-" + std::to_string(value2); + make_range_header_core(field, args...); +} + +#ifdef _WIN32 +class WSInit { +public: + WSInit() { + WSADATA wsaData; + WSAStartup(0x0002, &wsaData); + } + + ~WSInit() { WSACleanup(); } +}; + +static WSInit wsinit_; +#endif + +} // namespace detail + +// Header utilities +template <typename uint64_t, typename... Args> +inline std::pair<std::string, std::string> make_range_header(uint64_t value, + Args... args) { + std::string field; + detail::make_range_header_core(field, value, args...); + field.insert(0, "bytes="); + return std::make_pair("Range", field); +} + + +inline std::pair<std::string, std::string> +make_basic_authentication_header(const std::string& username, const std::string& password) { + auto field = "Basic " + detail::base64_encode(username + ":" + password); + return std::make_pair("Authorization", field); +} +// Request implementation +inline bool Request::has_header(const char *key) const { + return detail::has_header(headers, key); +} + +inline std::string Request::get_header_value(const char *key, size_t id) const { + return detail::get_header_value(headers, key, id, ""); +} + +inline size_t Request::get_header_value_count(const char *key) const { + auto r = headers.equal_range(key); + return std::distance(r.first, r.second); +} + +inline void Request::set_header(const char *key, const char *val) { + headers.emplace(key, val); +} + +inline bool Request::has_param(const char *key) const { + return params.find(key) != params.end(); +} + +inline std::string Request::get_param_value(const char *key, size_t id) const { + auto it = params.find(key); + std::advance(it, id); + if (it != params.end()) { return it->second; } + return std::string(); +} + +inline size_t Request::get_param_value_count(const char *key) const { + auto r = params.equal_range(key); + return std::distance(r.first, r.second); +} + +inline bool Request::has_file(const char *key) const { + return files.find(key) != files.end(); +} + +inline MultipartFile Request::get_file_value(const char *key) const { + auto it = files.find(key); + if (it != files.end()) { return it->second; } + return MultipartFile(); +} + +// Response implementation +inline bool Response::has_header(const char *key) const { + return headers.find(key) != headers.end(); +} + +inline std::string Response::get_header_value(const char *key, + size_t id) const { + return detail::get_header_value(headers, key, id, ""); +} + +inline size_t Response::get_header_value_count(const char *key) const { + auto r = headers.equal_range(key); + return std::distance(r.first, r.second); +} + +inline void Response::set_header(const char *key, const char *val) { + headers.emplace(key, val); +} + +inline void Response::set_redirect(const char *url) { + set_header("Location", url); + status = 302; +} + +inline void Response::set_content(const char *s, size_t n, + const char *content_type) { + body.assign(s, n); + set_header("Content-Type", content_type); +} + +inline void Response::set_content(const std::string &s, + const char *content_type) { + body = s; + set_header("Content-Type", content_type); +} + +// Rstream implementation +template <typename... Args> +inline void Stream::write_format(const char *fmt, const Args &... args) { + const auto bufsiz = 2048; + char buf[bufsiz]; + +#if defined(_MSC_VER) && _MSC_VER < 1900 + auto n = _snprintf_s(buf, bufsiz, bufsiz - 1, fmt, args...); +#else + auto n = snprintf(buf, bufsiz - 1, fmt, args...); +#endif + if (n > 0) { + if (n >= bufsiz - 1) { + std::vector<char> glowable_buf(bufsiz); + + while (n >= static_cast<int>(glowable_buf.size() - 1)) { + glowable_buf.resize(glowable_buf.size() * 2); +#if defined(_MSC_VER) && _MSC_VER < 1900 + n = _snprintf_s(&glowable_buf[0], glowable_buf.size(), + glowable_buf.size() - 1, fmt, args...); +#else + n = snprintf(&glowable_buf[0], glowable_buf.size() - 1, fmt, args...); +#endif + } + write(&glowable_buf[0], n); + } else { + write(buf, n); + } + } +} + +// Socket stream implementation +inline SocketStream::SocketStream(socket_t sock) : sock_(sock) {} + +inline SocketStream::~SocketStream() {} + +inline int SocketStream::read(char *ptr, size_t size) { + if (detail::select_read(sock_, CPPHTTPLIB_READ_TIMEOUT_SECOND, + CPPHTTPLIB_READ_TIMEOUT_USECOND) > 0) { + return recv(sock_, ptr, static_cast<int>(size), 0); + } + return -1; +} + +inline int SocketStream::write(const char *ptr, size_t size) { + return send(sock_, ptr, static_cast<int>(size), 0); +} + +inline int SocketStream::write(const char *ptr) { + return write(ptr, strlen(ptr)); +} + +inline std::string SocketStream::get_remote_addr() const { + return detail::get_remote_addr(sock_); +} + +// Buffer stream implementation +inline int BufferStream::read(char *ptr, size_t size) { +#if defined(_MSC_VER) && _MSC_VER < 1900 + return static_cast<int>(buffer._Copy_s(ptr, size, size)); +#else + return static_cast<int>(buffer.copy(ptr, size)); +#endif +} + +inline int BufferStream::write(const char *ptr, size_t size) { + buffer.append(ptr, size); + return static_cast<int>(size); +} + +inline int BufferStream::write(const char *ptr) { + size_t size = strlen(ptr); + buffer.append(ptr, size); + return static_cast<int>(size); +} + +inline std::string BufferStream::get_remote_addr() const { return ""; } + +inline const std::string &BufferStream::get_buffer() const { return buffer; } + +// HTTP server implementation +inline Server::Server() + : keep_alive_max_count_(CPPHTTPLIB_KEEPALIVE_MAX_COUNT), + payload_max_length_(CPPHTTPLIB_PAYLOAD_MAX_LENGTH), is_running_(false), + svr_sock_(INVALID_SOCKET), running_threads_(0) { +#ifndef _WIN32 + signal(SIGPIPE, SIG_IGN); +#endif +} + +inline Server::~Server() {} + +inline Server &Server::Get(const char *pattern, Handler handler) { + get_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Post(const char *pattern, Handler handler) { + post_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Put(const char *pattern, Handler handler) { + put_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Patch(const char *pattern, Handler handler) { + patch_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Delete(const char *pattern, Handler handler) { + delete_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Options(const char *pattern, Handler handler) { + options_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline bool Server::set_base_dir(const char *path) { + if (detail::is_dir(path)) { + base_dir_ = path; + return true; + } + return false; +} + +inline void Server::set_error_handler(Handler handler) { + error_handler_ = handler; +} + +inline void Server::set_logger(Logger logger) { logger_ = logger; } + +inline void Server::set_keep_alive_max_count(size_t count) { + keep_alive_max_count_ = count; +} + +inline void Server::set_payload_max_length(uint64_t length) { + payload_max_length_ = length; +} + +inline int Server::bind_to_any_port(const char *host, int socket_flags) { + return bind_internal(host, 0, socket_flags); +} + +inline bool Server::listen_after_bind() { return listen_internal(); } + +inline bool Server::listen(const char *host, int port, int socket_flags) { + if (bind_internal(host, port, socket_flags) < 0) return false; + return listen_internal(); +} + +inline bool Server::is_running() const { return is_running_; } + +inline void Server::stop() { + if (is_running_) { + assert(svr_sock_ != INVALID_SOCKET); + std::atomic<socket_t> sock(svr_sock_.exchange(INVALID_SOCKET)); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } +} + +inline bool Server::parse_request_line(const char *s, Request &req) { + static std::regex re("(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) " + "(([^?]+)(?:\\?(.+?))?) (HTTP/1\\.[01])\r\n"); + + std::cmatch m; + if (std::regex_match(s, m, re)) { + req.version = std::string(m[5]); + req.method = std::string(m[1]); + req.target = std::string(m[2]); + req.path = detail::decode_url(m[3]); + + // Parse query text + auto len = std::distance(m[4].first, m[4].second); + if (len > 0) { detail::parse_query_text(m[4], req.params); } + + return true; + } + + return false; +} + +inline void Server::write_response(Stream &strm, bool last_connection, + const Request &req, Response &res) { + assert(res.status != -1); + + if (400 <= res.status && error_handler_) { error_handler_(req, res); } + + // Response line + strm.write_format("HTTP/1.1 %d %s\r\n", res.status, + detail::status_message(res.status)); + + // Headers + if (last_connection || req.get_header_value("Connection") == "close") { + res.set_header("Connection", "close"); + } + + if (!last_connection && req.get_header_value("Connection") == "Keep-Alive") { + res.set_header("Connection", "Keep-Alive"); + } + + if (res.body.empty()) { + if (!res.has_header("Content-Length")) { + if (res.content_producer) { + // Streamed response + res.set_header("Transfer-Encoding", "chunked"); + } else { + res.set_header("Content-Length", "0"); + } + } + } else { +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + // TODO: 'Accpet-Encoding' has gzip, not gzip;q=0 + const auto &encodings = req.get_header_value("Accept-Encoding"); + if (encodings.find("gzip") != std::string::npos && + detail::can_compress(res.get_header_value("Content-Type"))) { + if (detail::compress(res.body)) { + res.set_header("Content-Encoding", "gzip"); + } + } +#endif + + if (!res.has_header("Content-Type")) { + res.set_header("Content-Type", "text/plain"); + } + + auto length = std::to_string(res.body.size()); + res.set_header("Content-Length", length.c_str()); + } + + detail::write_headers(strm, res); + + // Body + if (req.method != "HEAD") { + if (!res.body.empty()) { + strm.write(res.body.c_str(), res.body.size()); + } else if (res.content_producer) { + detail::write_content_chunked(strm, res); + } + } + + // Log + if (logger_) { logger_(req, res); } +} + +inline bool Server::handle_file_request(Request &req, Response &res) { + if (!base_dir_.empty() && detail::is_valid_path(req.path)) { + std::string path = base_dir_ + req.path; + + if (!path.empty() && path.back() == '/') { path += "index.html"; } + + if (detail::is_file(path)) { + detail::read_file(path, res.body); + auto type = detail::find_content_type(path); + if (type) { res.set_header("Content-Type", type); } + res.status = 200; + return true; + } + } + + return false; +} + +inline socket_t Server::create_server_socket(const char *host, int port, + int socket_flags) const { + return detail::create_socket( + host, port, + [](socket_t sock, struct addrinfo &ai) -> bool { + if (::bind(sock, ai.ai_addr, static_cast<int>(ai.ai_addrlen))) { + return false; + } + if (::listen(sock, 5)) { // Listen through 5 channels + return false; + } + return true; + }, + socket_flags); +} + +inline int Server::bind_internal(const char *host, int port, int socket_flags) { + if (!is_valid()) { return -1; } + + svr_sock_ = create_server_socket(host, port, socket_flags); + if (svr_sock_ == INVALID_SOCKET) { return -1; } + + if (port == 0) { + struct sockaddr_storage address; + socklen_t len = sizeof(address); + if (getsockname(svr_sock_, reinterpret_cast<struct sockaddr *>(&address), + &len) == -1) { + return -1; + } + if (address.ss_family == AF_INET) { + return ntohs(reinterpret_cast<struct sockaddr_in *>(&address)->sin_port); + } else if (address.ss_family == AF_INET6) { + return ntohs( + reinterpret_cast<struct sockaddr_in6 *>(&address)->sin6_port); + } else { + return -1; + } + } else { + return port; + } +} + +inline bool Server::listen_internal() { + auto ret = true; + + is_running_ = true; + + for (;;) { + if (svr_sock_ == INVALID_SOCKET) { + // The server socket was closed by 'stop' method. + break; + } + + auto val = detail::select_read(svr_sock_, 0, 100000); + + if (val == 0) { // Timeout + continue; + } + + socket_t sock = accept(svr_sock_, nullptr, nullptr); + + if (sock == INVALID_SOCKET) { + if (svr_sock_ != INVALID_SOCKET) { + detail::close_socket(svr_sock_); + ret = false; + } else { + ; // The server socket was closed by user. + } + break; + } + + // TODO: Use thread pool... + std::thread([=]() { + { + std::lock_guard<std::mutex> guard(running_threads_mutex_); + running_threads_++; + } + + read_and_close_socket(sock); + + { + std::lock_guard<std::mutex> guard(running_threads_mutex_); + running_threads_--; + } + }).detach(); + } + + // TODO: Use thread pool... + for (;;) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + std::lock_guard<std::mutex> guard(running_threads_mutex_); + if (!running_threads_) { break; } + } + + is_running_ = false; + + return ret; +} + +inline bool Server::routing(Request &req, Response &res) { + if (req.method == "GET" && handle_file_request(req, res)) { return true; } + + if (req.method == "GET" || req.method == "HEAD") { + return dispatch_request(req, res, get_handlers_); + } else if (req.method == "POST") { + return dispatch_request(req, res, post_handlers_); + } else if (req.method == "PUT") { + return dispatch_request(req, res, put_handlers_); + } else if (req.method == "PATCH") { + return dispatch_request(req, res, patch_handlers_); + } else if (req.method == "DELETE") { + return dispatch_request(req, res, delete_handlers_); + } else if (req.method == "OPTIONS") { + return dispatch_request(req, res, options_handlers_); + } + return false; +} + +inline bool Server::dispatch_request(Request &req, Response &res, + Handlers &handlers) { + for (const auto &x : handlers) { + const auto &pattern = x.first; + const auto &handler = x.second; + + if (std::regex_match(req.path, req.matches, pattern)) { + handler(req, res); + return true; + } + } + return false; +} + +inline bool +Server::process_request(Stream &strm, bool last_connection, + bool &connection_close, + std::function<void(Request &)> setup_request) { + const auto bufsiz = 2048; + char buf[bufsiz]; + + detail::stream_line_reader reader(strm, buf, bufsiz); + + // Connection has been closed on client + if (!reader.getline()) { return false; } + + Request req; + Response res; + + res.version = "HTTP/1.1"; + + // Check if the request URI doesn't exceed the limit + if (reader.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) { + res.status = 414; + write_response(strm, last_connection, req, res); + return true; + } + + // Request line and headers + if (!parse_request_line(reader.ptr(), req) || + !detail::read_headers(strm, req.headers)) { + res.status = 400; + write_response(strm, last_connection, req, res); + return true; + } + + if (req.get_header_value("Connection") == "close") { + connection_close = true; + } + + req.set_header("REMOTE_ADDR", strm.get_remote_addr().c_str()); + + // Body + if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH") { + if (!detail::read_content( + strm, req, payload_max_length_, res.status, Progress(), + [&](const char *buf, size_t n) { req.body.append(buf, n); })) { + write_response(strm, last_connection, req, res); + return true; + } + + const auto &content_type = req.get_header_value("Content-Type"); + + if (!content_type.find("application/x-www-form-urlencoded")) { + detail::parse_query_text(req.body, req.params); + } else if (!content_type.find("multipart/form-data")) { + std::string boundary; + if (!detail::parse_multipart_boundary(content_type, boundary) || + !detail::parse_multipart_formdata(boundary, req.body, req.files)) { + res.status = 400; + write_response(strm, last_connection, req, res); + return true; + } + } + } + + // TODO: Add additional request info + if (setup_request) { setup_request(req); } + + if (routing(req, res)) { + if (res.status == -1) { res.status = 200; } + } else { + res.status = 404; + } + + write_response(strm, last_connection, req, res); + return true; +} + +inline bool Server::is_valid() const { return true; } + +inline bool Server::read_and_close_socket(socket_t sock) { + return detail::read_and_close_socket( + sock, keep_alive_max_count_, + [this](Stream &strm, bool last_connection, bool &connection_close) { + return process_request(strm, last_connection, connection_close); + }); +} + +// HTTP client implementation +inline Client::Client(const char *host, int port, time_t timeout_sec) + : host_(host), port_(port), timeout_sec_(timeout_sec), + host_and_port_(host_ + ":" + std::to_string(port_)) {} + +inline Client::~Client() {} + +inline bool Client::is_valid() const { return true; } + +inline socket_t Client::create_client_socket() const { + return detail::create_socket( + host_.c_str(), port_, [=](socket_t sock, struct addrinfo &ai) -> bool { + detail::set_nonblocking(sock, true); + + auto ret = connect(sock, ai.ai_addr, static_cast<int>(ai.ai_addrlen)); + if (ret < 0) { + if (detail::is_connection_error() || + !detail::wait_until_socket_is_ready(sock, timeout_sec_, 0)) { + detail::close_socket(sock); + return false; + } + } + + detail::set_nonblocking(sock, false); + return true; + }); +} + +inline bool Client::read_response_line(Stream &strm, Response &res) { + const auto bufsiz = 2048; + char buf[bufsiz]; + + detail::stream_line_reader reader(strm, buf, bufsiz); + + if (!reader.getline()) { return false; } + + const static std::regex re("(HTTP/1\\.[01]) (\\d+?) .*\r\n"); + + std::cmatch m; + if (std::regex_match(reader.ptr(), m, re)) { + res.version = std::string(m[1]); + res.status = std::stoi(std::string(m[2])); + } + + return true; +} + +inline bool Client::send(Request &req, Response &res) { + if (req.path.empty()) { return false; } + + auto sock = create_client_socket(); + if (sock == INVALID_SOCKET) { return false; } + + return read_and_close_socket(sock, req, res); +} + +inline void Client::write_request(Stream &strm, Request &req) { + BufferStream bstrm; + + // Request line + auto path = detail::encode_url(req.path); + + bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str()); + + // Headers + if (!req.has_header("Host")) { + if (is_ssl()) { + if (port_ == 443) { + req.set_header("Host", host_.c_str()); + } else { + req.set_header("Host", host_and_port_.c_str()); + } + } else { + if (port_ == 80) { + req.set_header("Host", host_.c_str()); + } else { + req.set_header("Host", host_and_port_.c_str()); + } + } + } + + if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); } + + if (!req.has_header("User-Agent")) { + req.set_header("User-Agent", "cpp-httplib/0.2"); + } + + // TODO: Support KeepAlive connection + // if (!req.has_header("Connection")) { + req.set_header("Connection", "close"); + // } + + if (req.body.empty()) { + if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH") { + req.set_header("Content-Length", "0"); + } + } else { + if (!req.has_header("Content-Type")) { + req.set_header("Content-Type", "text/plain"); + } + + if (!req.has_header("Content-Length")) { + auto length = std::to_string(req.body.size()); + req.set_header("Content-Length", length.c_str()); + } + } + + detail::write_headers(bstrm, req); + + // Body + if (!req.body.empty()) { bstrm.write(req.body.c_str(), req.body.size()); } + + // Flush buffer + auto &data = bstrm.get_buffer(); + strm.write(data.data(), data.size()); +} + +inline bool Client::process_request(Stream &strm, Request &req, Response &res, + bool &connection_close) { + // Send request + write_request(strm, req); + + // Receive response and headers + if (!read_response_line(strm, res) || + !detail::read_headers(strm, res.headers)) { + return false; + } + + if (res.get_header_value("Connection") == "close" || + res.version == "HTTP/1.0") { + connection_close = true; + } + + // Body + if (req.method != "HEAD") { + ContentReceiver out = [&](const char *buf, size_t n) { + res.body.append(buf, n); + }; + + if (res.content_receiver) { + out = [&](const char *buf, size_t n) { res.content_receiver(buf, n); }; + } + + int dummy_status; + if (!detail::read_content(strm, res, std::numeric_limits<uint64_t>::max(), + dummy_status, res.progress, out)) { + return false; + } + } + + return true; +} + +inline bool Client::read_and_close_socket(socket_t sock, Request &req, + Response &res) { + return detail::read_and_close_socket( + sock, 0, + [&](Stream &strm, bool /*last_connection*/, bool &connection_close) { + return process_request(strm, req, res, connection_close); + }); +} + +inline bool Client::is_ssl() const { return false; } + +inline std::shared_ptr<Response> Client::Get(const char *path, + Progress progress) { + return Get(path, Headers(), progress); +} + +inline std::shared_ptr<Response> +Client::Get(const char *path, const Headers &headers, Progress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + + auto res = std::make_shared<Response>(); + res->progress = progress; + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr<Response> Client::Get(const char *path, + ContentReceiver content_receiver, + Progress progress) { + return Get(path, Headers(), content_receiver, progress); +} + +inline std::shared_ptr<Response> Client::Get(const char *path, + const Headers &headers, + ContentReceiver content_receiver, + Progress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + + auto res = std::make_shared<Response>(); + res->content_receiver = content_receiver; + res->progress = progress; + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr<Response> Client::Head(const char *path) { + return Head(path, Headers()); +} + +inline std::shared_ptr<Response> Client::Head(const char *path, + const Headers &headers) { + Request req; + req.method = "HEAD"; + req.headers = headers; + req.path = path; + + auto res = std::make_shared<Response>(); + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr<Response> Client::Post(const char *path, + const std::string &body, + const char *content_type) { + return Post(path, Headers(), body, content_type); +} + +inline std::shared_ptr<Response> Client::Post(const char *path, + const Headers &headers, + const std::string &body, + const char *content_type) { + Request req; + req.method = "POST"; + req.headers = headers; + req.path = path; + + req.headers.emplace("Content-Type", content_type); + req.body = body; + + auto res = std::make_shared<Response>(); + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr<Response> Client::Post(const char *path, + const Params ¶ms) { + return Post(path, Headers(), params); +} + +inline std::shared_ptr<Response> +Client::Post(const char *path, const Headers &headers, const Params ¶ms) { + std::string query; + for (auto it = params.begin(); it != params.end(); ++it) { + if (it != params.begin()) { query += "&"; } + query += it->first; + query += "="; + query += detail::encode_url(it->second); + } + + return Post(path, headers, query, "application/x-www-form-urlencoded"); +} + +inline std::shared_ptr<Response> Client::Put(const char *path, + const std::string &body, + const char *content_type) { + return Put(path, Headers(), body, content_type); +} + +inline std::shared_ptr<Response> Client::Put(const char *path, + const Headers &headers, + const std::string &body, + const char *content_type) { + Request req; + req.method = "PUT"; + req.headers = headers; + req.path = path; + + req.headers.emplace("Content-Type", content_type); + req.body = body; + + auto res = std::make_shared<Response>(); + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr<Response> Client::Patch(const char *path, + const std::string &body, + const char *content_type) { + return Patch(path, Headers(), body, content_type); +} + +inline std::shared_ptr<Response> Client::Patch(const char *path, + const Headers &headers, + const std::string &body, + const char *content_type) { + Request req; + req.method = "PATCH"; + req.headers = headers; + req.path = path; + + req.headers.emplace("Content-Type", content_type); + req.body = body; + + auto res = std::make_shared<Response>(); + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr<Response> Client::Delete(const char *path, + const std::string &body, + const char *content_type) { + return Delete(path, Headers(), body, content_type); +} + +inline std::shared_ptr<Response> Client::Delete(const char *path, + const Headers &headers, + const std::string &body, + const char *content_type) { + Request req; + req.method = "DELETE"; + req.headers = headers; + req.path = path; + + if (content_type) { req.headers.emplace("Content-Type", content_type); } + req.body = body; + + auto res = std::make_shared<Response>(); + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr<Response> Client::Options(const char *path) { + return Options(path, Headers()); +} + +inline std::shared_ptr<Response> Client::Options(const char *path, + const Headers &headers) { + Request req; + req.method = "OPTIONS"; + req.path = path; + req.headers = headers; + + auto res = std::make_shared<Response>(); + + return send(req, *res) ? res : nullptr; +} + +/* + * SSL Implementation + */ +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +namespace detail { + +template <typename U, typename V, typename T> +inline bool +read_and_close_socket_ssl(socket_t sock, size_t keep_alive_max_count, + // TODO: OpenSSL 1.0.2 occasionally crashes... + // The upcoming 1.1.0 is going to be thread safe. + SSL_CTX *ctx, std::mutex &ctx_mutex, + U SSL_connect_or_accept, V setup, T callback) { + SSL *ssl = nullptr; + { + std::lock_guard<std::mutex> guard(ctx_mutex); + ssl = SSL_new(ctx); + } + + if (!ssl) { + close_socket(sock); + return false; + } + + auto bio = BIO_new_socket(sock, BIO_NOCLOSE); + SSL_set_bio(ssl, bio, bio); + + if (!setup(ssl)) { + SSL_shutdown(ssl); + { + std::lock_guard<std::mutex> guard(ctx_mutex); + SSL_free(ssl); + } + + close_socket(sock); + return false; + } + + bool ret = false; + + if (SSL_connect_or_accept(ssl) == 1) { + if (keep_alive_max_count > 0) { + auto count = keep_alive_max_count; + while (count > 0 && + detail::select_read(sock, CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND, + CPPHTTPLIB_KEEPALIVE_TIMEOUT_USECOND) > 0) { + SSLSocketStream strm(sock, ssl); + auto last_connection = count == 1; + auto connection_close = false; + + ret = callback(ssl, strm, last_connection, connection_close); + if (!ret || connection_close) { break; } + + count--; + } + } else { + SSLSocketStream strm(sock, ssl); + auto dummy_connection_close = false; + ret = callback(ssl, strm, true, dummy_connection_close); + } + } + + SSL_shutdown(ssl); + { + std::lock_guard<std::mutex> guard(ctx_mutex); + SSL_free(ssl); + } + + close_socket(sock); + + return ret; +} + +class SSLInit { +public: + SSLInit() { + SSL_load_error_strings(); + SSL_library_init(); + } + + ~SSLInit() { ERR_free_strings(); } +}; + +static SSLInit sslinit_; + +} // namespace detail + +// SSL socket stream implementation +inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl) + : sock_(sock), ssl_(ssl) {} + +inline SSLSocketStream::~SSLSocketStream() {} + +inline int SSLSocketStream::read(char *ptr, size_t size) { + if (SSL_pending(ssl_) > 0 || + detail::select_read(sock_, CPPHTTPLIB_READ_TIMEOUT_SECOND, + CPPHTTPLIB_READ_TIMEOUT_USECOND) > 0) { + return SSL_read(ssl_, ptr, size); + } + return -1; +} + +inline int SSLSocketStream::write(const char *ptr, size_t size) { + return SSL_write(ssl_, ptr, size); +} + +inline int SSLSocketStream::write(const char *ptr) { + return write(ptr, strlen(ptr)); +} + +inline std::string SSLSocketStream::get_remote_addr() const { + return detail::get_remote_addr(sock_); +} + +// SSL HTTP server implementation +inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path, + const char *client_ca_cert_dir_path) { + ctx_ = SSL_CTX_new(SSLv23_server_method()); + + if (ctx_) { + SSL_CTX_set_options(ctx_, + SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | + SSL_OP_NO_COMPRESSION | + SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); + + // auto ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); + // SSL_CTX_set_tmp_ecdh(ctx_, ecdh); + // EC_KEY_free(ecdh); + + if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 || + SSL_CTX_use_PrivateKey_file(ctx_, private_key_path, SSL_FILETYPE_PEM) != + 1) { + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } else if (client_ca_cert_file_path || client_ca_cert_dir_path) { + // if (client_ca_cert_file_path) { + // auto list = SSL_load_client_CA_file(client_ca_cert_file_path); + // SSL_CTX_set_client_CA_list(ctx_, list); + // } + + SSL_CTX_load_verify_locations(ctx_, client_ca_cert_file_path, + client_ca_cert_dir_path); + + SSL_CTX_set_verify( + ctx_, + SSL_VERIFY_PEER | + SSL_VERIFY_FAIL_IF_NO_PEER_CERT, // SSL_VERIFY_CLIENT_ONCE, + nullptr); + } + } +} + +inline SSLServer::~SSLServer() { + if (ctx_) { SSL_CTX_free(ctx_); } +} + +inline bool SSLServer::is_valid() const { return ctx_; } + +inline bool SSLServer::read_and_close_socket(socket_t sock) { + return detail::read_and_close_socket_ssl( + sock, keep_alive_max_count_, ctx_, ctx_mutex_, SSL_accept, + [](SSL * /*ssl*/) { return true; }, + [this](SSL *ssl, Stream &strm, bool last_connection, + bool &connection_close) { + return process_request(strm, last_connection, connection_close, + [&](Request &req) { req.ssl = ssl; }); + }); +} + +// SSL HTTP client implementation +inline SSLClient::SSLClient(const char *host, int port, time_t timeout_sec, + const char *client_cert_path, + const char *client_key_path) + : Client(host, port, timeout_sec) { + ctx_ = SSL_CTX_new(SSLv23_client_method()); + + detail::split(&host_[0], &host_[host_.size()], '.', + [&](const char *b, const char *e) { + host_components_.emplace_back(std::string(b, e)); + }); + if (client_cert_path && client_key_path) { + if (SSL_CTX_use_certificate_file(ctx_, client_cert_path, + SSL_FILETYPE_PEM) != 1 || + SSL_CTX_use_PrivateKey_file(ctx_, client_key_path, SSL_FILETYPE_PEM) != + 1) { + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } + } +} + +inline SSLClient::~SSLClient() { + if (ctx_) { SSL_CTX_free(ctx_); } +} + +inline bool SSLClient::is_valid() const { return ctx_; } + +inline void SSLClient::set_ca_cert_path(const char *ca_cert_file_path, + const char *ca_cert_dir_path) { + if (ca_cert_file_path) { ca_cert_file_path_ = ca_cert_file_path; } + if (ca_cert_dir_path) { ca_cert_dir_path_ = ca_cert_dir_path; } +} + +inline void SSLClient::enable_server_certificate_verification(bool enabled) { + server_certificate_verification_ = enabled; +} + +inline long SSLClient::get_openssl_verify_result() const { + return verify_result_; +} + +inline bool SSLClient::read_and_close_socket(socket_t sock, Request &req, + Response &res) { + + return is_valid() && + detail::read_and_close_socket_ssl( + sock, 0, ctx_, ctx_mutex_, + [&](SSL *ssl) { + if (ca_cert_file_path_.empty()) { + SSL_CTX_set_verify(ctx_, SSL_VERIFY_NONE, nullptr); + } else { + if (!SSL_CTX_load_verify_locations( + ctx_, ca_cert_file_path_.c_str(), nullptr)) { + return false; + } + SSL_CTX_set_verify(ctx_, SSL_VERIFY_PEER, nullptr); + } + + if (SSL_connect(ssl) != 1) { return false; } + + if (server_certificate_verification_) { + verify_result_ = SSL_get_verify_result(ssl); + + if (verify_result_ != X509_V_OK) { return false; } + + auto server_cert = SSL_get_peer_certificate(ssl); + + if (server_cert == nullptr) { return false; } + + if (!verify_host(server_cert)) { + X509_free(server_cert); + return false; + } + X509_free(server_cert); + } + + return true; + }, + [&](SSL *ssl) { + SSL_set_tlsext_host_name(ssl, host_.c_str()); + return true; + }, + [&](SSL * /*ssl*/, Stream &strm, bool /*last_connection*/, + bool &connection_close) { + return process_request(strm, req, res, connection_close); + }); +} + +inline bool SSLClient::is_ssl() const { return true; } + +inline bool SSLClient::verify_host(X509 *server_cert) const { + /* Quote from RFC2818 section 3.1 "Server Identity" + + If a subjectAltName extension of type dNSName is present, that MUST + be used as the identity. Otherwise, the (most specific) Common Name + field in the Subject field of the certificate MUST be used. Although + the use of the Common Name is existing practice, it is deprecated and + Certification Authorities are encouraged to use the dNSName instead. + + Matching is performed using the matching rules specified by + [RFC2459]. If more than one identity of a given type is present in + the certificate (e.g., more than one dNSName name, a match in any one + of the set is considered acceptable.) Names may contain the wildcard + character * which is considered to match any single domain name + component or component fragment. E.g., *.a.com matches foo.a.com but + not bar.foo.a.com. f*.com matches foo.com but not bar.com. + + In some cases, the URI is specified as an IP address rather than a + hostname. In this case, the iPAddress subjectAltName must be present + in the certificate and must exactly match the IP in the URI. + + */ + return verify_host_with_subject_alt_name(server_cert) || + verify_host_with_common_name(server_cert); +} + +inline bool +SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { + auto ret = false; + + auto type = GEN_DNS; + + struct in6_addr addr6; + struct in_addr addr; + size_t addr_len = 0; + + if (inet_pton(AF_INET6, host_.c_str(), &addr6)) { + type = GEN_IPADD; + addr_len = sizeof(struct in6_addr); + } else if (inet_pton(AF_INET, host_.c_str(), &addr)) { + type = GEN_IPADD; + addr_len = sizeof(struct in_addr); + } + + auto alt_names = static_cast<const struct stack_st_GENERAL_NAME *>( + X509_get_ext_d2i(server_cert, NID_subject_alt_name, nullptr, nullptr)); + + if (alt_names) { + auto dsn_matched = false; + auto ip_mached = false; + + auto count = sk_GENERAL_NAME_num(alt_names); + + for (auto i = 0; i < count && !dsn_matched; i++) { + auto val = sk_GENERAL_NAME_value(alt_names, i); + if (val->type == type) { + auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5); + auto name_len = (size_t)ASN1_STRING_length(val->d.ia5); + + if (strlen(name) == name_len) { + switch (type) { + case GEN_DNS: dsn_matched = check_host_name(name, name_len); break; + + case GEN_IPADD: + if (!memcmp(&addr6, name, addr_len) || + !memcmp(&addr, name, addr_len)) { + ip_mached = true; + } + break; + } + } + } + } + + if (dsn_matched || ip_mached) { ret = true; } + } + + GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names); + + return ret; +} + +inline bool SSLClient::verify_host_with_common_name(X509 *server_cert) const { + const auto subject_name = X509_get_subject_name(server_cert); + + if (subject_name != nullptr) { + char name[BUFSIZ]; + auto name_len = X509_NAME_get_text_by_NID(subject_name, NID_commonName, + name, sizeof(name)); + + if (name_len != -1) { return check_host_name(name, name_len); } + } + + return false; +} + +inline bool SSLClient::check_host_name(const char *pattern, + size_t pattern_len) const { + if (host_.size() == pattern_len && host_ == pattern) { return true; } + + // Wildcard match + // https://bugs.launchpad.net/ubuntu/+source/firefox-3.0/+bug/376484 + std::vector<std::string> pattern_components; + detail::split(&pattern[0], &pattern[pattern_len], '.', + [&](const char *b, const char *e) { + pattern_components.emplace_back(std::string(b, e)); + }); + + if (host_components_.size() != pattern_components.size()) { return false; } + + auto itr = pattern_components.begin(); + for (const auto &h : host_components_) { + auto &p = *itr; + if (p != h && p != "*") { + auto partial_match = (p.size() > 0 && p[p.size() - 1] == '*' && + !p.compare(0, p.size() - 1, h)); + if (!partial_match) { return false; } + } + ++itr; + } + + return true; +} +#endif + +} // namespace httplib + +#endif // CPPHTTPLIB_HTTPLIB_H diff --git a/ext/curl-7.58.0/Win32/include/curl/curl.h b/ext/curl-7.58.0/Win32/include/curl/curl.h new file mode 100644 index 00000000..7680acd1 --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/curl.h @@ -0,0 +1,2751 @@ +#ifndef __CURL_CURL_H +#define __CURL_CURL_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* + * If you have libcurl problems, all docs and details are found here: + * https://curl.haxx.se/libcurl/ + * + * curl-library mailing list subscription and unsubscription web interface: + * https://cool.haxx.se/mailman/listinfo/curl-library/ + */ + +#ifdef CURL_NO_OLDIES +#define CURL_STRICTER +#endif + +#include "curlver.h" /* libcurl version defines */ +#include "system.h" /* determine things run-time */ + +/* + * Define WIN32 when build target is Win32 API + */ + +#if (defined(_WIN32) || defined(__WIN32__)) && \ + !defined(WIN32) && !defined(__SYMBIAN32__) +#define WIN32 +#endif + +#include <stdio.h> +#include <limits.h> + +#if defined(__FreeBSD__) && (__FreeBSD__ >= 2) +/* Needed for __FreeBSD_version symbol definition */ +#include <osreldate.h> +#endif + +/* The include stuff here below is mainly for time_t! */ +#include <sys/types.h> +#include <time.h> + +#if defined(WIN32) && !defined(_WIN32_WCE) && !defined(__CYGWIN__) +#if !(defined(_WINSOCKAPI_) || defined(_WINSOCK_H) || \ + defined(__LWIP_OPT_H__) || defined(LWIP_HDR_OPT_H)) +/* The check above prevents the winsock2 inclusion if winsock.h already was + included, since they can't co-exist without problems */ +#include <winsock2.h> +#include <ws2tcpip.h> +#endif +#endif + +/* HP-UX systems version 9, 10 and 11 lack sys/select.h and so does oldish + libc5-based Linux systems. Only include it on systems that are known to + require it! */ +#if defined(_AIX) || defined(__NOVELL_LIBC__) || defined(__NetBSD__) || \ + defined(__minix) || defined(__SYMBIAN32__) || defined(__INTEGRITY) || \ + defined(ANDROID) || defined(__ANDROID__) || defined(__OpenBSD__) || \ + defined(__CYGWIN__) || \ + (defined(__FreeBSD_version) && (__FreeBSD_version < 800000)) +#include <sys/select.h> +#endif + +#if !defined(WIN32) && !defined(_WIN32_WCE) +#include <sys/socket.h> +#endif + +#if !defined(WIN32) && !defined(__WATCOMC__) && !defined(__VXWORKS__) +#include <sys/time.h> +#endif + +#ifdef __BEOS__ +#include <support/SupportDefs.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(BUILDING_LIBCURL) || defined(CURL_STRICTER) +typedef struct Curl_easy CURL; +typedef struct Curl_share CURLSH; +#else +typedef void CURL; +typedef void CURLSH; +#endif + +/* + * libcurl external API function linkage decorations. + */ + +#ifdef CURL_STATICLIB +# define CURL_EXTERN +#elif defined(WIN32) || defined(_WIN32) || defined(__SYMBIAN32__) +# if defined(BUILDING_LIBCURL) +# define CURL_EXTERN __declspec(dllexport) +# else +# define CURL_EXTERN __declspec(dllimport) +# endif +#elif defined(BUILDING_LIBCURL) && defined(CURL_HIDDEN_SYMBOLS) +# define CURL_EXTERN CURL_EXTERN_SYMBOL +#else +# define CURL_EXTERN +#endif + +#ifndef curl_socket_typedef +/* socket typedef */ +#if defined(WIN32) && !defined(__LWIP_OPT_H__) && !defined(LWIP_HDR_OPT_H) +typedef SOCKET curl_socket_t; +#define CURL_SOCKET_BAD INVALID_SOCKET +#else +typedef int curl_socket_t; +#define CURL_SOCKET_BAD -1 +#endif +#define curl_socket_typedef +#endif /* curl_socket_typedef */ + +/* enum for the different supported SSL backends */ +typedef enum { + CURLSSLBACKEND_NONE = 0, + CURLSSLBACKEND_OPENSSL = 1, + CURLSSLBACKEND_GNUTLS = 2, + CURLSSLBACKEND_NSS = 3, + CURLSSLBACKEND_OBSOLETE4 = 4, /* Was QSOSSL. */ + CURLSSLBACKEND_GSKIT = 5, + CURLSSLBACKEND_POLARSSL = 6, + CURLSSLBACKEND_WOLFSSL = 7, + CURLSSLBACKEND_SCHANNEL = 8, + CURLSSLBACKEND_DARWINSSL = 9, + CURLSSLBACKEND_AXTLS = 10, + CURLSSLBACKEND_MBEDTLS = 11 +} curl_sslbackend; + +/* aliases for library clones and renames */ +#define CURLSSLBACKEND_LIBRESSL CURLSSLBACKEND_OPENSSL +#define CURLSSLBACKEND_BORINGSSL CURLSSLBACKEND_OPENSSL +#define CURLSSLBACKEND_CYASSL CURLSSLBACKEND_WOLFSSL + +struct curl_httppost { + struct curl_httppost *next; /* next entry in the list */ + char *name; /* pointer to allocated name */ + long namelength; /* length of name length */ + char *contents; /* pointer to allocated data contents */ + long contentslength; /* length of contents field, see also + CURL_HTTPPOST_LARGE */ + char *buffer; /* pointer to allocated buffer contents */ + long bufferlength; /* length of buffer field */ + char *contenttype; /* Content-Type */ + struct curl_slist *contentheader; /* list of extra headers for this form */ + struct curl_httppost *more; /* if one field name has more than one + file, this link should link to following + files */ + long flags; /* as defined below */ + +/* specified content is a file name */ +#define CURL_HTTPPOST_FILENAME (1<<0) +/* specified content is a file name */ +#define CURL_HTTPPOST_READFILE (1<<1) +/* name is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRNAME (1<<2) +/* contents is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRCONTENTS (1<<3) +/* upload file from buffer */ +#define CURL_HTTPPOST_BUFFER (1<<4) +/* upload file from pointer contents */ +#define CURL_HTTPPOST_PTRBUFFER (1<<5) +/* upload file contents by using the regular read callback to get the data and + pass the given pointer as custom pointer */ +#define CURL_HTTPPOST_CALLBACK (1<<6) +/* use size in 'contentlen', added in 7.46.0 */ +#define CURL_HTTPPOST_LARGE (1<<7) + + char *showfilename; /* The file name to show. If not set, the + actual file name will be used (if this + is a file part) */ + void *userp; /* custom pointer used for + HTTPPOST_CALLBACK posts */ + curl_off_t contentlen; /* alternative length of contents + field. Used if CURL_HTTPPOST_LARGE is + set. Added in 7.46.0 */ +}; + +/* This is the CURLOPT_PROGRESSFUNCTION callback proto. It is now considered + deprecated but was the only choice up until 7.31.0 */ +typedef int (*curl_progress_callback)(void *clientp, + double dltotal, + double dlnow, + double ultotal, + double ulnow); + +/* This is the CURLOPT_XFERINFOFUNCTION callback proto. It was introduced in + 7.32.0, it avoids floating point and provides more detailed information. */ +typedef int (*curl_xferinfo_callback)(void *clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow); + +#ifndef CURL_MAX_READ_SIZE + /* The maximum receive buffer size configurable via CURLOPT_BUFFERSIZE. */ +#define CURL_MAX_READ_SIZE 524288 +#endif + +#ifndef CURL_MAX_WRITE_SIZE + /* Tests have proven that 20K is a very bad buffer size for uploads on + Windows, while 16K for some odd reason performed a lot better. + We do the ifndef check to allow this value to easier be changed at build + time for those who feel adventurous. The practical minimum is about + 400 bytes since libcurl uses a buffer of this size as a scratch area + (unrelated to network send operations). */ +#define CURL_MAX_WRITE_SIZE 16384 +#endif + +#ifndef CURL_MAX_HTTP_HEADER +/* The only reason to have a max limit for this is to avoid the risk of a bad + server feeding libcurl with a never-ending header that will cause reallocs + infinitely */ +#define CURL_MAX_HTTP_HEADER (100*1024) +#endif + +/* This is a magic return code for the write callback that, when returned, + will signal libcurl to pause receiving on the current transfer. */ +#define CURL_WRITEFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_write_callback)(char *buffer, + size_t size, + size_t nitems, + void *outstream); + + + +/* enumeration of file types */ +typedef enum { + CURLFILETYPE_FILE = 0, + CURLFILETYPE_DIRECTORY, + CURLFILETYPE_SYMLINK, + CURLFILETYPE_DEVICE_BLOCK, + CURLFILETYPE_DEVICE_CHAR, + CURLFILETYPE_NAMEDPIPE, + CURLFILETYPE_SOCKET, + CURLFILETYPE_DOOR, /* is possible only on Sun Solaris now */ + + CURLFILETYPE_UNKNOWN /* should never occur */ +} curlfiletype; + +#define CURLFINFOFLAG_KNOWN_FILENAME (1<<0) +#define CURLFINFOFLAG_KNOWN_FILETYPE (1<<1) +#define CURLFINFOFLAG_KNOWN_TIME (1<<2) +#define CURLFINFOFLAG_KNOWN_PERM (1<<3) +#define CURLFINFOFLAG_KNOWN_UID (1<<4) +#define CURLFINFOFLAG_KNOWN_GID (1<<5) +#define CURLFINFOFLAG_KNOWN_SIZE (1<<6) +#define CURLFINFOFLAG_KNOWN_HLINKCOUNT (1<<7) + +/* Content of this structure depends on information which is known and is + achievable (e.g. by FTP LIST parsing). Please see the url_easy_setopt(3) man + page for callbacks returning this structure -- some fields are mandatory, + some others are optional. The FLAG field has special meaning. */ +struct curl_fileinfo { + char *filename; + curlfiletype filetype; + time_t time; + unsigned int perm; + int uid; + int gid; + curl_off_t size; + long int hardlinks; + + struct { + /* If some of these fields is not NULL, it is a pointer to b_data. */ + char *time; + char *perm; + char *user; + char *group; + char *target; /* pointer to the target filename of a symlink */ + } strings; + + unsigned int flags; + + /* used internally */ + char *b_data; + size_t b_size; + size_t b_used; +}; + +/* return codes for CURLOPT_CHUNK_BGN_FUNCTION */ +#define CURL_CHUNK_BGN_FUNC_OK 0 +#define CURL_CHUNK_BGN_FUNC_FAIL 1 /* tell the lib to end the task */ +#define CURL_CHUNK_BGN_FUNC_SKIP 2 /* skip this chunk over */ + +/* if splitting of data transfer is enabled, this callback is called before + download of an individual chunk started. Note that parameter "remains" works + only for FTP wildcard downloading (for now), otherwise is not used */ +typedef long (*curl_chunk_bgn_callback)(const void *transfer_info, + void *ptr, + int remains); + +/* return codes for CURLOPT_CHUNK_END_FUNCTION */ +#define CURL_CHUNK_END_FUNC_OK 0 +#define CURL_CHUNK_END_FUNC_FAIL 1 /* tell the lib to end the task */ + +/* If splitting of data transfer is enabled this callback is called after + download of an individual chunk finished. + Note! After this callback was set then it have to be called FOR ALL chunks. + Even if downloading of this chunk was skipped in CHUNK_BGN_FUNC. + This is the reason why we don't need "transfer_info" parameter in this + callback and we are not interested in "remains" parameter too. */ +typedef long (*curl_chunk_end_callback)(void *ptr); + +/* return codes for FNMATCHFUNCTION */ +#define CURL_FNMATCHFUNC_MATCH 0 /* string corresponds to the pattern */ +#define CURL_FNMATCHFUNC_NOMATCH 1 /* pattern doesn't match the string */ +#define CURL_FNMATCHFUNC_FAIL 2 /* an error occurred */ + +/* callback type for wildcard downloading pattern matching. If the + string matches the pattern, return CURL_FNMATCHFUNC_MATCH value, etc. */ +typedef int (*curl_fnmatch_callback)(void *ptr, + const char *pattern, + const char *string); + +/* These are the return codes for the seek callbacks */ +#define CURL_SEEKFUNC_OK 0 +#define CURL_SEEKFUNC_FAIL 1 /* fail the entire transfer */ +#define CURL_SEEKFUNC_CANTSEEK 2 /* tell libcurl seeking can't be done, so + libcurl might try other means instead */ +typedef int (*curl_seek_callback)(void *instream, + curl_off_t offset, + int origin); /* 'whence' */ + +/* This is a return code for the read callback that, when returned, will + signal libcurl to immediately abort the current transfer. */ +#define CURL_READFUNC_ABORT 0x10000000 +/* This is a return code for the read callback that, when returned, will + signal libcurl to pause sending data on the current transfer. */ +#define CURL_READFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_read_callback)(char *buffer, + size_t size, + size_t nitems, + void *instream); + +typedef enum { + CURLSOCKTYPE_IPCXN, /* socket created for a specific IP connection */ + CURLSOCKTYPE_ACCEPT, /* socket created by accept() call */ + CURLSOCKTYPE_LAST /* never use */ +} curlsocktype; + +/* The return code from the sockopt_callback can signal information back + to libcurl: */ +#define CURL_SOCKOPT_OK 0 +#define CURL_SOCKOPT_ERROR 1 /* causes libcurl to abort and return + CURLE_ABORTED_BY_CALLBACK */ +#define CURL_SOCKOPT_ALREADY_CONNECTED 2 + +typedef int (*curl_sockopt_callback)(void *clientp, + curl_socket_t curlfd, + curlsocktype purpose); + +struct curl_sockaddr { + int family; + int socktype; + int protocol; + unsigned int addrlen; /* addrlen was a socklen_t type before 7.18.0 but it + turned really ugly and painful on the systems that + lack this type */ + struct sockaddr addr; +}; + +typedef curl_socket_t +(*curl_opensocket_callback)(void *clientp, + curlsocktype purpose, + struct curl_sockaddr *address); + +typedef int +(*curl_closesocket_callback)(void *clientp, curl_socket_t item); + +typedef enum { + CURLIOE_OK, /* I/O operation successful */ + CURLIOE_UNKNOWNCMD, /* command was unknown to callback */ + CURLIOE_FAILRESTART, /* failed to restart the read */ + CURLIOE_LAST /* never use */ +} curlioerr; + +typedef enum { + CURLIOCMD_NOP, /* no operation */ + CURLIOCMD_RESTARTREAD, /* restart the read stream from start */ + CURLIOCMD_LAST /* never use */ +} curliocmd; + +typedef curlioerr (*curl_ioctl_callback)(CURL *handle, + int cmd, + void *clientp); + +#ifndef CURL_DID_MEMORY_FUNC_TYPEDEFS +/* + * The following typedef's are signatures of malloc, free, realloc, strdup and + * calloc respectively. Function pointers of these types can be passed to the + * curl_global_init_mem() function to set user defined memory management + * callback routines. + */ +typedef void *(*curl_malloc_callback)(size_t size); +typedef void (*curl_free_callback)(void *ptr); +typedef void *(*curl_realloc_callback)(void *ptr, size_t size); +typedef char *(*curl_strdup_callback)(const char *str); +typedef void *(*curl_calloc_callback)(size_t nmemb, size_t size); + +#define CURL_DID_MEMORY_FUNC_TYPEDEFS +#endif + +/* the kind of data that is passed to information_callback*/ +typedef enum { + CURLINFO_TEXT = 0, + CURLINFO_HEADER_IN, /* 1 */ + CURLINFO_HEADER_OUT, /* 2 */ + CURLINFO_DATA_IN, /* 3 */ + CURLINFO_DATA_OUT, /* 4 */ + CURLINFO_SSL_DATA_IN, /* 5 */ + CURLINFO_SSL_DATA_OUT, /* 6 */ + CURLINFO_END +} curl_infotype; + +typedef int (*curl_debug_callback) + (CURL *handle, /* the handle/transfer this concerns */ + curl_infotype type, /* what kind of data */ + char *data, /* points to the data */ + size_t size, /* size of the data pointed to */ + void *userptr); /* whatever the user please */ + +/* All possible error codes from all sorts of curl functions. Future versions + may return other values, stay prepared. + + Always add new return codes last. Never *EVER* remove any. The return + codes must remain the same! + */ + +typedef enum { + CURLE_OK = 0, + CURLE_UNSUPPORTED_PROTOCOL, /* 1 */ + CURLE_FAILED_INIT, /* 2 */ + CURLE_URL_MALFORMAT, /* 3 */ + CURLE_NOT_BUILT_IN, /* 4 - [was obsoleted in August 2007 for + 7.17.0, reused in April 2011 for 7.21.5] */ + CURLE_COULDNT_RESOLVE_PROXY, /* 5 */ + CURLE_COULDNT_RESOLVE_HOST, /* 6 */ + CURLE_COULDNT_CONNECT, /* 7 */ + CURLE_WEIRD_SERVER_REPLY, /* 8 */ + CURLE_REMOTE_ACCESS_DENIED, /* 9 a service was denied by the server + due to lack of access - when login fails + this is not returned. */ + CURLE_FTP_ACCEPT_FAILED, /* 10 - [was obsoleted in April 2006 for + 7.15.4, reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASS_REPLY, /* 11 */ + CURLE_FTP_ACCEPT_TIMEOUT, /* 12 - timeout occurred accepting server + [was obsoleted in August 2007 for 7.17.0, + reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASV_REPLY, /* 13 */ + CURLE_FTP_WEIRD_227_FORMAT, /* 14 */ + CURLE_FTP_CANT_GET_HOST, /* 15 */ + CURLE_HTTP2, /* 16 - A problem in the http2 framing layer. + [was obsoleted in August 2007 for 7.17.0, + reused in July 2014 for 7.38.0] */ + CURLE_FTP_COULDNT_SET_TYPE, /* 17 */ + CURLE_PARTIAL_FILE, /* 18 */ + CURLE_FTP_COULDNT_RETR_FILE, /* 19 */ + CURLE_OBSOLETE20, /* 20 - NOT USED */ + CURLE_QUOTE_ERROR, /* 21 - quote command failure */ + CURLE_HTTP_RETURNED_ERROR, /* 22 */ + CURLE_WRITE_ERROR, /* 23 */ + CURLE_OBSOLETE24, /* 24 - NOT USED */ + CURLE_UPLOAD_FAILED, /* 25 - failed upload "command" */ + CURLE_READ_ERROR, /* 26 - couldn't open/read from file */ + CURLE_OUT_OF_MEMORY, /* 27 */ + /* Note: CURLE_OUT_OF_MEMORY may sometimes indicate a conversion error + instead of a memory allocation error if CURL_DOES_CONVERSIONS + is defined + */ + CURLE_OPERATION_TIMEDOUT, /* 28 - the timeout time was reached */ + CURLE_OBSOLETE29, /* 29 - NOT USED */ + CURLE_FTP_PORT_FAILED, /* 30 - FTP PORT operation failed */ + CURLE_FTP_COULDNT_USE_REST, /* 31 - the REST command failed */ + CURLE_OBSOLETE32, /* 32 - NOT USED */ + CURLE_RANGE_ERROR, /* 33 - RANGE "command" didn't work */ + CURLE_HTTP_POST_ERROR, /* 34 */ + CURLE_SSL_CONNECT_ERROR, /* 35 - wrong when connecting with SSL */ + CURLE_BAD_DOWNLOAD_RESUME, /* 36 - couldn't resume download */ + CURLE_FILE_COULDNT_READ_FILE, /* 37 */ + CURLE_LDAP_CANNOT_BIND, /* 38 */ + CURLE_LDAP_SEARCH_FAILED, /* 39 */ + CURLE_OBSOLETE40, /* 40 - NOT USED */ + CURLE_FUNCTION_NOT_FOUND, /* 41 - NOT USED starting with 7.53.0 */ + CURLE_ABORTED_BY_CALLBACK, /* 42 */ + CURLE_BAD_FUNCTION_ARGUMENT, /* 43 */ + CURLE_OBSOLETE44, /* 44 - NOT USED */ + CURLE_INTERFACE_FAILED, /* 45 - CURLOPT_INTERFACE failed */ + CURLE_OBSOLETE46, /* 46 - NOT USED */ + CURLE_TOO_MANY_REDIRECTS, /* 47 - catch endless re-direct loops */ + CURLE_UNKNOWN_OPTION, /* 48 - User specified an unknown option */ + CURLE_TELNET_OPTION_SYNTAX, /* 49 - Malformed telnet option */ + CURLE_OBSOLETE50, /* 50 - NOT USED */ + CURLE_PEER_FAILED_VERIFICATION, /* 51 - peer's certificate or fingerprint + wasn't verified fine */ + CURLE_GOT_NOTHING, /* 52 - when this is a specific error */ + CURLE_SSL_ENGINE_NOTFOUND, /* 53 - SSL crypto engine not found */ + CURLE_SSL_ENGINE_SETFAILED, /* 54 - can not set SSL crypto engine as + default */ + CURLE_SEND_ERROR, /* 55 - failed sending network data */ + CURLE_RECV_ERROR, /* 56 - failure in receiving network data */ + CURLE_OBSOLETE57, /* 57 - NOT IN USE */ + CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */ + CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */ + CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */ + CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized/bad encoding */ + CURLE_LDAP_INVALID_URL, /* 62 - Invalid LDAP URL */ + CURLE_FILESIZE_EXCEEDED, /* 63 - Maximum file size exceeded */ + CURLE_USE_SSL_FAILED, /* 64 - Requested FTP SSL level failed */ + CURLE_SEND_FAIL_REWIND, /* 65 - Sending the data requires a rewind + that failed */ + CURLE_SSL_ENGINE_INITFAILED, /* 66 - failed to initialise ENGINE */ + CURLE_LOGIN_DENIED, /* 67 - user, password or similar was not + accepted and we failed to login */ + CURLE_TFTP_NOTFOUND, /* 68 - file not found on server */ + CURLE_TFTP_PERM, /* 69 - permission problem on server */ + CURLE_REMOTE_DISK_FULL, /* 70 - out of disk space on server */ + CURLE_TFTP_ILLEGAL, /* 71 - Illegal TFTP operation */ + CURLE_TFTP_UNKNOWNID, /* 72 - Unknown transfer ID */ + CURLE_REMOTE_FILE_EXISTS, /* 73 - File already exists */ + CURLE_TFTP_NOSUCHUSER, /* 74 - No such user */ + CURLE_CONV_FAILED, /* 75 - conversion failed */ + CURLE_CONV_REQD, /* 76 - caller must register conversion + callbacks using curl_easy_setopt options + CURLOPT_CONV_FROM_NETWORK_FUNCTION, + CURLOPT_CONV_TO_NETWORK_FUNCTION, and + CURLOPT_CONV_FROM_UTF8_FUNCTION */ + CURLE_SSL_CACERT_BADFILE, /* 77 - could not load CACERT file, missing + or wrong format */ + CURLE_REMOTE_FILE_NOT_FOUND, /* 78 - remote file not found */ + CURLE_SSH, /* 79 - error from the SSH layer, somewhat + generic so the error message will be of + interest when this has happened */ + + CURLE_SSL_SHUTDOWN_FAILED, /* 80 - Failed to shut down the SSL + connection */ + CURLE_AGAIN, /* 81 - socket is not ready for send/recv, + wait till it's ready and try again (Added + in 7.18.2) */ + CURLE_SSL_CRL_BADFILE, /* 82 - could not load CRL file, missing or + wrong format (Added in 7.19.0) */ + CURLE_SSL_ISSUER_ERROR, /* 83 - Issuer check failed. (Added in + 7.19.0) */ + CURLE_FTP_PRET_FAILED, /* 84 - a PRET command failed */ + CURLE_RTSP_CSEQ_ERROR, /* 85 - mismatch of RTSP CSeq numbers */ + CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Ids */ + CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ + CURLE_CHUNK_FAILED, /* 88 - chunk callback reported error */ + CURLE_NO_CONNECTION_AVAILABLE, /* 89 - No connection available, the + session will be queued */ + CURLE_SSL_PINNEDPUBKEYNOTMATCH, /* 90 - specified pinned public key did not + match */ + CURLE_SSL_INVALIDCERTSTATUS, /* 91 - invalid certificate status */ + CURLE_HTTP2_STREAM, /* 92 - stream error in HTTP/2 framing layer + */ + CURL_LAST /* never use! */ +} CURLcode; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Previously obsolete error code re-used in 7.38.0 */ +#define CURLE_OBSOLETE16 CURLE_HTTP2 + +/* Previously obsolete error codes re-used in 7.24.0 */ +#define CURLE_OBSOLETE10 CURLE_FTP_ACCEPT_FAILED +#define CURLE_OBSOLETE12 CURLE_FTP_ACCEPT_TIMEOUT + +/* compatibility with older names */ +#define CURLOPT_ENCODING CURLOPT_ACCEPT_ENCODING +#define CURLE_FTP_WEIRD_SERVER_REPLY CURLE_WEIRD_SERVER_REPLY + +/* The following were added in 7.21.5, April 2011 */ +#define CURLE_UNKNOWN_TELNET_OPTION CURLE_UNKNOWN_OPTION + +/* The following were added in 7.17.1 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_SSL_PEER_CERTIFICATE CURLE_PEER_FAILED_VERIFICATION + +/* The following were added in 7.17.0 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_OBSOLETE CURLE_OBSOLETE50 /* no one should be using this! */ +#define CURLE_BAD_PASSWORD_ENTERED CURLE_OBSOLETE46 +#define CURLE_BAD_CALLING_ORDER CURLE_OBSOLETE44 +#define CURLE_FTP_USER_PASSWORD_INCORRECT CURLE_OBSOLETE10 +#define CURLE_FTP_CANT_RECONNECT CURLE_OBSOLETE16 +#define CURLE_FTP_COULDNT_GET_SIZE CURLE_OBSOLETE32 +#define CURLE_FTP_COULDNT_SET_ASCII CURLE_OBSOLETE29 +#define CURLE_FTP_WEIRD_USER_REPLY CURLE_OBSOLETE12 +#define CURLE_FTP_WRITE_ERROR CURLE_OBSOLETE20 +#define CURLE_LIBRARY_NOT_FOUND CURLE_OBSOLETE40 +#define CURLE_MALFORMAT_USER CURLE_OBSOLETE24 +#define CURLE_SHARE_IN_USE CURLE_OBSOLETE57 +#define CURLE_URL_MALFORMAT_USER CURLE_NOT_BUILT_IN + +#define CURLE_FTP_ACCESS_DENIED CURLE_REMOTE_ACCESS_DENIED +#define CURLE_FTP_COULDNT_SET_BINARY CURLE_FTP_COULDNT_SET_TYPE +#define CURLE_FTP_QUOTE_ERROR CURLE_QUOTE_ERROR +#define CURLE_TFTP_DISKFULL CURLE_REMOTE_DISK_FULL +#define CURLE_TFTP_EXISTS CURLE_REMOTE_FILE_EXISTS +#define CURLE_HTTP_RANGE_ERROR CURLE_RANGE_ERROR +#define CURLE_FTP_SSL_FAILED CURLE_USE_SSL_FAILED + +/* The following were added earlier */ + +#define CURLE_OPERATION_TIMEOUTED CURLE_OPERATION_TIMEDOUT + +#define CURLE_HTTP_NOT_FOUND CURLE_HTTP_RETURNED_ERROR +#define CURLE_HTTP_PORT_FAILED CURLE_INTERFACE_FAILED +#define CURLE_FTP_COULDNT_STOR_FILE CURLE_UPLOAD_FAILED + +#define CURLE_FTP_PARTIAL_FILE CURLE_PARTIAL_FILE +#define CURLE_FTP_BAD_DOWNLOAD_RESUME CURLE_BAD_DOWNLOAD_RESUME + +/* This was the error code 50 in 7.7.3 and a few earlier versions, this + is no longer used by libcurl but is instead #defined here only to not + make programs break */ +#define CURLE_ALREADY_COMPLETE 99999 + +/* Provide defines for really old option names */ +#define CURLOPT_FILE CURLOPT_WRITEDATA /* name changed in 7.9.7 */ +#define CURLOPT_INFILE CURLOPT_READDATA /* name changed in 7.9.7 */ +#define CURLOPT_WRITEHEADER CURLOPT_HEADERDATA + +/* Since long deprecated options with no code in the lib that does anything + with them. */ +#define CURLOPT_WRITEINFO CURLOPT_OBSOLETE40 +#define CURLOPT_CLOSEPOLICY CURLOPT_OBSOLETE72 + +#endif /*!CURL_NO_OLDIES*/ + +/* This prototype applies to all conversion callbacks */ +typedef CURLcode (*curl_conv_callback)(char *buffer, size_t length); + +typedef CURLcode (*curl_ssl_ctx_callback)(CURL *curl, /* easy handle */ + void *ssl_ctx, /* actually an + OpenSSL SSL_CTX */ + void *userptr); + +typedef enum { + CURLPROXY_HTTP = 0, /* added in 7.10, new in 7.19.4 default is to use + CONNECT HTTP/1.1 */ + CURLPROXY_HTTP_1_0 = 1, /* added in 7.19.4, force to use CONNECT + HTTP/1.0 */ + CURLPROXY_HTTPS = 2, /* added in 7.52.0 */ + CURLPROXY_SOCKS4 = 4, /* support added in 7.15.2, enum existed already + in 7.10 */ + CURLPROXY_SOCKS5 = 5, /* added in 7.10 */ + CURLPROXY_SOCKS4A = 6, /* added in 7.18.0 */ + CURLPROXY_SOCKS5_HOSTNAME = 7 /* Use the SOCKS5 protocol but pass along the + host name rather than the IP address. added + in 7.18.0 */ +} curl_proxytype; /* this enum was added in 7.10 */ + +/* + * Bitmasks for CURLOPT_HTTPAUTH and CURLOPT_PROXYAUTH options: + * + * CURLAUTH_NONE - No HTTP authentication + * CURLAUTH_BASIC - HTTP Basic authentication (default) + * CURLAUTH_DIGEST - HTTP Digest authentication + * CURLAUTH_NEGOTIATE - HTTP Negotiate (SPNEGO) authentication + * CURLAUTH_GSSNEGOTIATE - Alias for CURLAUTH_NEGOTIATE (deprecated) + * CURLAUTH_NTLM - HTTP NTLM authentication + * CURLAUTH_DIGEST_IE - HTTP Digest authentication with IE flavour + * CURLAUTH_NTLM_WB - HTTP NTLM authentication delegated to winbind helper + * CURLAUTH_ONLY - Use together with a single other type to force no + * authentication or just that single type + * CURLAUTH_ANY - All fine types set + * CURLAUTH_ANYSAFE - All fine types except Basic + */ + +#define CURLAUTH_NONE ((unsigned long)0) +#define CURLAUTH_BASIC (((unsigned long)1)<<0) +#define CURLAUTH_DIGEST (((unsigned long)1)<<1) +#define CURLAUTH_NEGOTIATE (((unsigned long)1)<<2) +/* Deprecated since the advent of CURLAUTH_NEGOTIATE */ +#define CURLAUTH_GSSNEGOTIATE CURLAUTH_NEGOTIATE +/* Used for CURLOPT_SOCKS5_AUTH to stay terminologically correct */ +#define CURLAUTH_GSSAPI CURLAUTH_NEGOTIATE +#define CURLAUTH_NTLM (((unsigned long)1)<<3) +#define CURLAUTH_DIGEST_IE (((unsigned long)1)<<4) +#define CURLAUTH_NTLM_WB (((unsigned long)1)<<5) +#define CURLAUTH_ONLY (((unsigned long)1)<<31) +#define CURLAUTH_ANY (~CURLAUTH_DIGEST_IE) +#define CURLAUTH_ANYSAFE (~(CURLAUTH_BASIC|CURLAUTH_DIGEST_IE)) + +#define CURLSSH_AUTH_ANY ~0 /* all types supported by the server */ +#define CURLSSH_AUTH_NONE 0 /* none allowed, silly but complete */ +#define CURLSSH_AUTH_PUBLICKEY (1<<0) /* public/private key files */ +#define CURLSSH_AUTH_PASSWORD (1<<1) /* password */ +#define CURLSSH_AUTH_HOST (1<<2) /* host key files */ +#define CURLSSH_AUTH_KEYBOARD (1<<3) /* keyboard interactive */ +#define CURLSSH_AUTH_AGENT (1<<4) /* agent (ssh-agent, pageant...) */ +#define CURLSSH_AUTH_GSSAPI (1<<5) /* gssapi (kerberos, ...) */ +#define CURLSSH_AUTH_DEFAULT CURLSSH_AUTH_ANY + +#define CURLGSSAPI_DELEGATION_NONE 0 /* no delegation (default) */ +#define CURLGSSAPI_DELEGATION_POLICY_FLAG (1<<0) /* if permitted by policy */ +#define CURLGSSAPI_DELEGATION_FLAG (1<<1) /* delegate always */ + +#define CURL_ERROR_SIZE 256 + +enum curl_khtype { + CURLKHTYPE_UNKNOWN, + CURLKHTYPE_RSA1, + CURLKHTYPE_RSA, + CURLKHTYPE_DSS, + CURLKHTYPE_ECDSA, + CURLKHTYPE_ED25519 +}; + +struct curl_khkey { + const char *key; /* points to a zero-terminated string encoded with base64 + if len is zero, otherwise to the "raw" data */ + size_t len; + enum curl_khtype keytype; +}; + +/* this is the set of return values expected from the curl_sshkeycallback + callback */ +enum curl_khstat { + CURLKHSTAT_FINE_ADD_TO_FILE, + CURLKHSTAT_FINE, + CURLKHSTAT_REJECT, /* reject the connection, return an error */ + CURLKHSTAT_DEFER, /* do not accept it, but we can't answer right now so + this causes a CURLE_DEFER error but otherwise the + connection will be left intact etc */ + CURLKHSTAT_LAST /* not for use, only a marker for last-in-list */ +}; + +/* this is the set of status codes pass in to the callback */ +enum curl_khmatch { + CURLKHMATCH_OK, /* match */ + CURLKHMATCH_MISMATCH, /* host found, key mismatch! */ + CURLKHMATCH_MISSING, /* no matching host/key found */ + CURLKHMATCH_LAST /* not for use, only a marker for last-in-list */ +}; + +typedef int + (*curl_sshkeycallback) (CURL *easy, /* easy handle */ + const struct curl_khkey *knownkey, /* known */ + const struct curl_khkey *foundkey, /* found */ + enum curl_khmatch, /* libcurl's view on the keys */ + void *clientp); /* custom pointer passed from app */ + +/* parameter for the CURLOPT_USE_SSL option */ +typedef enum { + CURLUSESSL_NONE, /* do not attempt to use SSL */ + CURLUSESSL_TRY, /* try using SSL, proceed anyway otherwise */ + CURLUSESSL_CONTROL, /* SSL for the control connection or fail */ + CURLUSESSL_ALL, /* SSL for all communication or fail */ + CURLUSESSL_LAST /* not an option, never use */ +} curl_usessl; + +/* Definition of bits for the CURLOPT_SSL_OPTIONS argument: */ + +/* - ALLOW_BEAST tells libcurl to allow the BEAST SSL vulnerability in the + name of improving interoperability with older servers. Some SSL libraries + have introduced work-arounds for this flaw but those work-arounds sometimes + make the SSL communication fail. To regain functionality with those broken + servers, a user can this way allow the vulnerability back. */ +#define CURLSSLOPT_ALLOW_BEAST (1<<0) + +/* - NO_REVOKE tells libcurl to disable certificate revocation checks for those + SSL backends where such behavior is present. */ +#define CURLSSLOPT_NO_REVOKE (1<<1) + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2009 */ + +#define CURLFTPSSL_NONE CURLUSESSL_NONE +#define CURLFTPSSL_TRY CURLUSESSL_TRY +#define CURLFTPSSL_CONTROL CURLUSESSL_CONTROL +#define CURLFTPSSL_ALL CURLUSESSL_ALL +#define CURLFTPSSL_LAST CURLUSESSL_LAST +#define curl_ftpssl curl_usessl +#endif /*!CURL_NO_OLDIES*/ + +/* parameter for the CURLOPT_FTP_SSL_CCC option */ +typedef enum { + CURLFTPSSL_CCC_NONE, /* do not send CCC */ + CURLFTPSSL_CCC_PASSIVE, /* Let the server initiate the shutdown */ + CURLFTPSSL_CCC_ACTIVE, /* Initiate the shutdown */ + CURLFTPSSL_CCC_LAST /* not an option, never use */ +} curl_ftpccc; + +/* parameter for the CURLOPT_FTPSSLAUTH option */ +typedef enum { + CURLFTPAUTH_DEFAULT, /* let libcurl decide */ + CURLFTPAUTH_SSL, /* use "AUTH SSL" */ + CURLFTPAUTH_TLS, /* use "AUTH TLS" */ + CURLFTPAUTH_LAST /* not an option, never use */ +} curl_ftpauth; + +/* parameter for the CURLOPT_FTP_CREATE_MISSING_DIRS option */ +typedef enum { + CURLFTP_CREATE_DIR_NONE, /* do NOT create missing dirs! */ + CURLFTP_CREATE_DIR, /* (FTP/SFTP) if CWD fails, try MKD and then CWD + again if MKD succeeded, for SFTP this does + similar magic */ + CURLFTP_CREATE_DIR_RETRY, /* (FTP only) if CWD fails, try MKD and then CWD + again even if MKD failed! */ + CURLFTP_CREATE_DIR_LAST /* not an option, never use */ +} curl_ftpcreatedir; + +/* parameter for the CURLOPT_FTP_FILEMETHOD option */ +typedef enum { + CURLFTPMETHOD_DEFAULT, /* let libcurl pick */ + CURLFTPMETHOD_MULTICWD, /* single CWD operation for each path part */ + CURLFTPMETHOD_NOCWD, /* no CWD at all */ + CURLFTPMETHOD_SINGLECWD, /* one CWD to full dir, then work on file */ + CURLFTPMETHOD_LAST /* not an option, never use */ +} curl_ftpmethod; + +/* bitmask defines for CURLOPT_HEADEROPT */ +#define CURLHEADER_UNIFIED 0 +#define CURLHEADER_SEPARATE (1<<0) + +/* CURLPROTO_ defines are for the CURLOPT_*PROTOCOLS options */ +#define CURLPROTO_HTTP (1<<0) +#define CURLPROTO_HTTPS (1<<1) +#define CURLPROTO_FTP (1<<2) +#define CURLPROTO_FTPS (1<<3) +#define CURLPROTO_SCP (1<<4) +#define CURLPROTO_SFTP (1<<5) +#define CURLPROTO_TELNET (1<<6) +#define CURLPROTO_LDAP (1<<7) +#define CURLPROTO_LDAPS (1<<8) +#define CURLPROTO_DICT (1<<9) +#define CURLPROTO_FILE (1<<10) +#define CURLPROTO_TFTP (1<<11) +#define CURLPROTO_IMAP (1<<12) +#define CURLPROTO_IMAPS (1<<13) +#define CURLPROTO_POP3 (1<<14) +#define CURLPROTO_POP3S (1<<15) +#define CURLPROTO_SMTP (1<<16) +#define CURLPROTO_SMTPS (1<<17) +#define CURLPROTO_RTSP (1<<18) +#define CURLPROTO_RTMP (1<<19) +#define CURLPROTO_RTMPT (1<<20) +#define CURLPROTO_RTMPE (1<<21) +#define CURLPROTO_RTMPTE (1<<22) +#define CURLPROTO_RTMPS (1<<23) +#define CURLPROTO_RTMPTS (1<<24) +#define CURLPROTO_GOPHER (1<<25) +#define CURLPROTO_SMB (1<<26) +#define CURLPROTO_SMBS (1<<27) +#define CURLPROTO_ALL (~0) /* enable everything */ + +/* long may be 32 or 64 bits, but we should never depend on anything else + but 32 */ +#define CURLOPTTYPE_LONG 0 +#define CURLOPTTYPE_OBJECTPOINT 10000 +#define CURLOPTTYPE_STRINGPOINT 10000 +#define CURLOPTTYPE_FUNCTIONPOINT 20000 +#define CURLOPTTYPE_OFF_T 30000 + +/* *STRINGPOINT is an alias for OBJECTPOINT to allow tools to extract the + string options from the header file */ + +/* name is uppercase CURLOPT_<name>, + type is one of the defined CURLOPTTYPE_<type> + number is unique identifier */ +#ifdef CINIT +#undef CINIT +#endif + +#ifdef CURL_ISOCPP +#define CINIT(na,t,nu) CURLOPT_ ## na = CURLOPTTYPE_ ## t + nu +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define STRINGPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLOPT_/**/name = type + number +#endif + +/* + * This macro-mania below setups the CURLOPT_[what] enum, to be used with + * curl_easy_setopt(). The first argument in the CINIT() macro is the [what] + * word. + */ + +typedef enum { + /* This is the FILE * or void * the regular output should be written to. */ + CINIT(WRITEDATA, OBJECTPOINT, 1), + + /* The full URL to get/put */ + CINIT(URL, STRINGPOINT, 2), + + /* Port number to connect to, if other than default. */ + CINIT(PORT, LONG, 3), + + /* Name of proxy to use. */ + CINIT(PROXY, STRINGPOINT, 4), + + /* "user:password;options" to use when fetching. */ + CINIT(USERPWD, STRINGPOINT, 5), + + /* "user:password" to use with proxy. */ + CINIT(PROXYUSERPWD, STRINGPOINT, 6), + + /* Range to get, specified as an ASCII string. */ + CINIT(RANGE, STRINGPOINT, 7), + + /* not used */ + + /* Specified file stream to upload from (use as input): */ + CINIT(READDATA, OBJECTPOINT, 9), + + /* Buffer to receive error messages in, must be at least CURL_ERROR_SIZE + * bytes big. */ + CINIT(ERRORBUFFER, OBJECTPOINT, 10), + + /* Function that will be called to store the output (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11), + + /* Function that will be called to read the input (instead of fread). The + * parameters will use fread() syntax, make sure to follow them. */ + CINIT(READFUNCTION, FUNCTIONPOINT, 12), + + /* Time-out the read operation after this amount of seconds */ + CINIT(TIMEOUT, LONG, 13), + + /* If the CURLOPT_INFILE is used, this can be used to inform libcurl about + * how large the file being sent really is. That allows better error + * checking and better verifies that the upload was successful. -1 means + * unknown size. + * + * For large file support, there is also a _LARGE version of the key + * which takes an off_t type, allowing platforms with larger off_t + * sizes to handle larger files. See below for INFILESIZE_LARGE. + */ + CINIT(INFILESIZE, LONG, 14), + + /* POST static input fields. */ + CINIT(POSTFIELDS, OBJECTPOINT, 15), + + /* Set the referrer page (needed by some CGIs) */ + CINIT(REFERER, STRINGPOINT, 16), + + /* Set the FTP PORT string (interface name, named or numerical IP address) + Use i.e '-' to use default address. */ + CINIT(FTPPORT, STRINGPOINT, 17), + + /* Set the User-Agent string (examined by some CGIs) */ + CINIT(USERAGENT, STRINGPOINT, 18), + + /* If the download receives less than "low speed limit" bytes/second + * during "low speed time" seconds, the operations is aborted. + * You could i.e if you have a pretty high speed connection, abort if + * it is less than 2000 bytes/sec during 20 seconds. + */ + + /* Set the "low speed limit" */ + CINIT(LOW_SPEED_LIMIT, LONG, 19), + + /* Set the "low speed time" */ + CINIT(LOW_SPEED_TIME, LONG, 20), + + /* Set the continuation offset. + * + * Note there is also a _LARGE version of this key which uses + * off_t types, allowing for large file offsets on platforms which + * use larger-than-32-bit off_t's. Look below for RESUME_FROM_LARGE. + */ + CINIT(RESUME_FROM, LONG, 21), + + /* Set cookie in request: */ + CINIT(COOKIE, STRINGPOINT, 22), + + /* This points to a linked list of headers, struct curl_slist kind. This + list is also used for RTSP (in spite of its name) */ + CINIT(HTTPHEADER, OBJECTPOINT, 23), + + /* This points to a linked list of post entries, struct curl_httppost */ + CINIT(HTTPPOST, OBJECTPOINT, 24), + + /* name of the file keeping your private SSL-certificate */ + CINIT(SSLCERT, STRINGPOINT, 25), + + /* password for the SSL or SSH private key */ + CINIT(KEYPASSWD, STRINGPOINT, 26), + + /* send TYPE parameter? */ + CINIT(CRLF, LONG, 27), + + /* send linked-list of QUOTE commands */ + CINIT(QUOTE, OBJECTPOINT, 28), + + /* send FILE * or void * to store headers to, if you use a callback it + is simply passed to the callback unmodified */ + CINIT(HEADERDATA, OBJECTPOINT, 29), + + /* point to a file to read the initial cookies from, also enables + "cookie awareness" */ + CINIT(COOKIEFILE, STRINGPOINT, 31), + + /* What version to specifically try to use. + See CURL_SSLVERSION defines below. */ + CINIT(SSLVERSION, LONG, 32), + + /* What kind of HTTP time condition to use, see defines */ + CINIT(TIMECONDITION, LONG, 33), + + /* Time to use with the above condition. Specified in number of seconds + since 1 Jan 1970 */ + CINIT(TIMEVALUE, LONG, 34), + + /* 35 = OBSOLETE */ + + /* Custom request, for customizing the get command like + HTTP: DELETE, TRACE and others + FTP: to use a different list command + */ + CINIT(CUSTOMREQUEST, STRINGPOINT, 36), + + /* FILE handle to use instead of stderr */ + CINIT(STDERR, OBJECTPOINT, 37), + + /* 38 is not used */ + + /* send linked-list of post-transfer QUOTE commands */ + CINIT(POSTQUOTE, OBJECTPOINT, 39), + + CINIT(OBSOLETE40, OBJECTPOINT, 40), /* OBSOLETE, do not use! */ + + CINIT(VERBOSE, LONG, 41), /* talk a lot */ + CINIT(HEADER, LONG, 42), /* throw the header out too */ + CINIT(NOPROGRESS, LONG, 43), /* shut off the progress meter */ + CINIT(NOBODY, LONG, 44), /* use HEAD to get http document */ + CINIT(FAILONERROR, LONG, 45), /* no output on http error codes >= 400 */ + CINIT(UPLOAD, LONG, 46), /* this is an upload */ + CINIT(POST, LONG, 47), /* HTTP POST method */ + CINIT(DIRLISTONLY, LONG, 48), /* bare names when listing directories */ + + CINIT(APPEND, LONG, 50), /* Append instead of overwrite on upload! */ + + /* Specify whether to read the user+password from the .netrc or the URL. + * This must be one of the CURL_NETRC_* enums below. */ + CINIT(NETRC, LONG, 51), + + CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */ + + CINIT(TRANSFERTEXT, LONG, 53), /* transfer data in text/ASCII format */ + CINIT(PUT, LONG, 54), /* HTTP PUT */ + + /* 55 = OBSOLETE */ + + /* DEPRECATED + * Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_progress_callback + * prototype defines. */ + CINIT(PROGRESSFUNCTION, FUNCTIONPOINT, 56), + + /* Data passed to the CURLOPT_PROGRESSFUNCTION and CURLOPT_XFERINFOFUNCTION + callbacks */ + CINIT(PROGRESSDATA, OBJECTPOINT, 57), +#define CURLOPT_XFERINFODATA CURLOPT_PROGRESSDATA + + /* We want the referrer field set automatically when following locations */ + CINIT(AUTOREFERER, LONG, 58), + + /* Port of the proxy, can be set in the proxy string as well with: + "[host]:[port]" */ + CINIT(PROXYPORT, LONG, 59), + + /* size of the POST input data, if strlen() is not good to use */ + CINIT(POSTFIELDSIZE, LONG, 60), + + /* tunnel non-http operations through a HTTP proxy */ + CINIT(HTTPPROXYTUNNEL, LONG, 61), + + /* Set the interface string to use as outgoing network interface */ + CINIT(INTERFACE, STRINGPOINT, 62), + + /* Set the krb4/5 security level, this also enables krb4/5 awareness. This + * is a string, 'clear', 'safe', 'confidential' or 'private'. If the string + * is set but doesn't match one of these, 'private' will be used. */ + CINIT(KRBLEVEL, STRINGPOINT, 63), + + /* Set if we should verify the peer in ssl handshake, set 1 to verify. */ + CINIT(SSL_VERIFYPEER, LONG, 64), + + /* The CApath or CAfile used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAINFO, STRINGPOINT, 65), + + /* 66 = OBSOLETE */ + /* 67 = OBSOLETE */ + + /* Maximum number of http redirects to follow */ + CINIT(MAXREDIRS, LONG, 68), + + /* Pass a long set to 1 to get the date of the requested document (if + possible)! Pass a zero to shut it off. */ + CINIT(FILETIME, LONG, 69), + + /* This points to a linked list of telnet options */ + CINIT(TELNETOPTIONS, OBJECTPOINT, 70), + + /* Max amount of cached alive connections */ + CINIT(MAXCONNECTS, LONG, 71), + + CINIT(OBSOLETE72, LONG, 72), /* OBSOLETE, do not use! */ + + /* 73 = OBSOLETE */ + + /* Set to explicitly use a new connection for the upcoming transfer. + Do not use this unless you're absolutely sure of this, as it makes the + operation slower and is less friendly for the network. */ + CINIT(FRESH_CONNECT, LONG, 74), + + /* Set to explicitly forbid the upcoming transfer's connection to be re-used + when done. Do not use this unless you're absolutely sure of this, as it + makes the operation slower and is less friendly for the network. */ + CINIT(FORBID_REUSE, LONG, 75), + + /* Set to a file name that contains random data for libcurl to use to + seed the random engine when doing SSL connects. */ + CINIT(RANDOM_FILE, STRINGPOINT, 76), + + /* Set to the Entropy Gathering Daemon socket pathname */ + CINIT(EGDSOCKET, STRINGPOINT, 77), + + /* Time-out connect operations after this amount of seconds, if connects are + OK within this time, then fine... This only aborts the connect phase. */ + CINIT(CONNECTTIMEOUT, LONG, 78), + + /* Function that will be called to store headers (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(HEADERFUNCTION, FUNCTIONPOINT, 79), + + /* Set this to force the HTTP request to get back to GET. Only really usable + if POST, PUT or a custom request have been used first. + */ + CINIT(HTTPGET, LONG, 80), + + /* Set if we should verify the Common name from the peer certificate in ssl + * handshake, set 1 to check existence, 2 to ensure that it matches the + * provided hostname. */ + CINIT(SSL_VERIFYHOST, LONG, 81), + + /* Specify which file name to write all known cookies in after completed + operation. Set file name to "-" (dash) to make it go to stdout. */ + CINIT(COOKIEJAR, STRINGPOINT, 82), + + /* Specify which SSL ciphers to use */ + CINIT(SSL_CIPHER_LIST, STRINGPOINT, 83), + + /* Specify which HTTP version to use! This must be set to one of the + CURL_HTTP_VERSION* enums set below. */ + CINIT(HTTP_VERSION, LONG, 84), + + /* Specifically switch on or off the FTP engine's use of the EPSV command. By + default, that one will always be attempted before the more traditional + PASV command. */ + CINIT(FTP_USE_EPSV, LONG, 85), + + /* type of the file keeping your SSL-certificate ("DER", "PEM", "ENG") */ + CINIT(SSLCERTTYPE, STRINGPOINT, 86), + + /* name of the file keeping your private SSL-key */ + CINIT(SSLKEY, STRINGPOINT, 87), + + /* type of the file keeping your private SSL-key ("DER", "PEM", "ENG") */ + CINIT(SSLKEYTYPE, STRINGPOINT, 88), + + /* crypto engine for the SSL-sub system */ + CINIT(SSLENGINE, STRINGPOINT, 89), + + /* set the crypto engine for the SSL-sub system as default + the param has no meaning... + */ + CINIT(SSLENGINE_DEFAULT, LONG, 90), + + /* Non-zero value means to use the global dns cache */ + CINIT(DNS_USE_GLOBAL_CACHE, LONG, 91), /* DEPRECATED, do not use! */ + + /* DNS cache timeout */ + CINIT(DNS_CACHE_TIMEOUT, LONG, 92), + + /* send linked-list of pre-transfer QUOTE commands */ + CINIT(PREQUOTE, OBJECTPOINT, 93), + + /* set the debug function */ + CINIT(DEBUGFUNCTION, FUNCTIONPOINT, 94), + + /* set the data for the debug function */ + CINIT(DEBUGDATA, OBJECTPOINT, 95), + + /* mark this as start of a cookie session */ + CINIT(COOKIESESSION, LONG, 96), + + /* The CApath directory used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAPATH, STRINGPOINT, 97), + + /* Instruct libcurl to use a smaller receive buffer */ + CINIT(BUFFERSIZE, LONG, 98), + + /* Instruct libcurl to not use any signal/alarm handlers, even when using + timeouts. This option is useful for multi-threaded applications. + See libcurl-the-guide for more background information. */ + CINIT(NOSIGNAL, LONG, 99), + + /* Provide a CURLShare for mutexing non-ts data */ + CINIT(SHARE, OBJECTPOINT, 100), + + /* indicates type of proxy. accepted values are CURLPROXY_HTTP (default), + CURLPROXY_HTTPS, CURLPROXY_SOCKS4, CURLPROXY_SOCKS4A and + CURLPROXY_SOCKS5. */ + CINIT(PROXYTYPE, LONG, 101), + + /* Set the Accept-Encoding string. Use this to tell a server you would like + the response to be compressed. Before 7.21.6, this was known as + CURLOPT_ENCODING */ + CINIT(ACCEPT_ENCODING, STRINGPOINT, 102), + + /* Set pointer to private data */ + CINIT(PRIVATE, OBJECTPOINT, 103), + + /* Set aliases for HTTP 200 in the HTTP Response header */ + CINIT(HTTP200ALIASES, OBJECTPOINT, 104), + + /* Continue to send authentication (user+password) when following locations, + even when hostname changed. This can potentially send off the name + and password to whatever host the server decides. */ + CINIT(UNRESTRICTED_AUTH, LONG, 105), + + /* Specifically switch on or off the FTP engine's use of the EPRT command ( + it also disables the LPRT attempt). By default, those ones will always be + attempted before the good old traditional PORT command. */ + CINIT(FTP_USE_EPRT, LONG, 106), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_USERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(HTTPAUTH, LONG, 107), + + /* Set the ssl context callback function, currently only for OpenSSL ssl_ctx + in second argument. The function must be matching the + curl_ssl_ctx_callback proto. */ + CINIT(SSL_CTX_FUNCTION, FUNCTIONPOINT, 108), + + /* Set the userdata for the ssl context callback function's third + argument */ + CINIT(SSL_CTX_DATA, OBJECTPOINT, 109), + + /* FTP Option that causes missing dirs to be created on the remote server. + In 7.19.4 we introduced the convenience enums for this option using the + CURLFTP_CREATE_DIR prefix. + */ + CINIT(FTP_CREATE_MISSING_DIRS, LONG, 110), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_PROXYUSERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(PROXYAUTH, LONG, 111), + + /* FTP option that changes the timeout, in seconds, associated with + getting a response. This is different from transfer timeout time and + essentially places a demand on the FTP server to acknowledge commands + in a timely manner. */ + CINIT(FTP_RESPONSE_TIMEOUT, LONG, 112), +#define CURLOPT_SERVER_RESPONSE_TIMEOUT CURLOPT_FTP_RESPONSE_TIMEOUT + + /* Set this option to one of the CURL_IPRESOLVE_* defines (see below) to + tell libcurl to resolve names to those IP versions only. This only has + affect on systems with support for more than one, i.e IPv4 _and_ IPv6. */ + CINIT(IPRESOLVE, LONG, 113), + + /* Set this option to limit the size of a file that will be downloaded from + an HTTP or FTP server. + + Note there is also _LARGE version which adds large file support for + platforms which have larger off_t sizes. See MAXFILESIZE_LARGE below. */ + CINIT(MAXFILESIZE, LONG, 114), + + /* See the comment for INFILESIZE above, but in short, specifies + * the size of the file being uploaded. -1 means unknown. + */ + CINIT(INFILESIZE_LARGE, OFF_T, 115), + + /* Sets the continuation offset. There is also a LONG version of this; + * look above for RESUME_FROM. + */ + CINIT(RESUME_FROM_LARGE, OFF_T, 116), + + /* Sets the maximum size of data that will be downloaded from + * an HTTP or FTP server. See MAXFILESIZE above for the LONG version. + */ + CINIT(MAXFILESIZE_LARGE, OFF_T, 117), + + /* Set this option to the file name of your .netrc file you want libcurl + to parse (using the CURLOPT_NETRC option). If not set, libcurl will do + a poor attempt to find the user's home directory and check for a .netrc + file in there. */ + CINIT(NETRC_FILE, STRINGPOINT, 118), + + /* Enable SSL/TLS for FTP, pick one of: + CURLUSESSL_TRY - try using SSL, proceed anyway otherwise + CURLUSESSL_CONTROL - SSL for the control connection or fail + CURLUSESSL_ALL - SSL for all communication or fail + */ + CINIT(USE_SSL, LONG, 119), + + /* The _LARGE version of the standard POSTFIELDSIZE option */ + CINIT(POSTFIELDSIZE_LARGE, OFF_T, 120), + + /* Enable/disable the TCP Nagle algorithm */ + CINIT(TCP_NODELAY, LONG, 121), + + /* 122 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 123 OBSOLETE. Gone in 7.16.0 */ + /* 124 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 125 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 126 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 127 OBSOLETE. Gone in 7.16.0 */ + /* 128 OBSOLETE. Gone in 7.16.0 */ + + /* When FTP over SSL/TLS is selected (with CURLOPT_USE_SSL), this option + can be used to change libcurl's default action which is to first try + "AUTH SSL" and then "AUTH TLS" in this order, and proceed when a OK + response has been received. + + Available parameters are: + CURLFTPAUTH_DEFAULT - let libcurl decide + CURLFTPAUTH_SSL - try "AUTH SSL" first, then TLS + CURLFTPAUTH_TLS - try "AUTH TLS" first, then SSL + */ + CINIT(FTPSSLAUTH, LONG, 129), + + CINIT(IOCTLFUNCTION, FUNCTIONPOINT, 130), + CINIT(IOCTLDATA, OBJECTPOINT, 131), + + /* 132 OBSOLETE. Gone in 7.16.0 */ + /* 133 OBSOLETE. Gone in 7.16.0 */ + + /* zero terminated string for pass on to the FTP server when asked for + "account" info */ + CINIT(FTP_ACCOUNT, STRINGPOINT, 134), + + /* feed cookie into cookie engine */ + CINIT(COOKIELIST, STRINGPOINT, 135), + + /* ignore Content-Length */ + CINIT(IGNORE_CONTENT_LENGTH, LONG, 136), + + /* Set to non-zero to skip the IP address received in a 227 PASV FTP server + response. Typically used for FTP-SSL purposes but is not restricted to + that. libcurl will then instead use the same IP address it used for the + control connection. */ + CINIT(FTP_SKIP_PASV_IP, LONG, 137), + + /* Select "file method" to use when doing FTP, see the curl_ftpmethod + above. */ + CINIT(FTP_FILEMETHOD, LONG, 138), + + /* Local port number to bind the socket to */ + CINIT(LOCALPORT, LONG, 139), + + /* Number of ports to try, including the first one set with LOCALPORT. + Thus, setting it to 1 will make no additional attempts but the first. + */ + CINIT(LOCALPORTRANGE, LONG, 140), + + /* no transfer, set up connection and let application use the socket by + extracting it with CURLINFO_LASTSOCKET */ + CINIT(CONNECT_ONLY, LONG, 141), + + /* Function that will be called to convert from the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_FROM_NETWORK_FUNCTION, FUNCTIONPOINT, 142), + + /* Function that will be called to convert to the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_TO_NETWORK_FUNCTION, FUNCTIONPOINT, 143), + + /* Function that will be called to convert from UTF8 + (instead of using the iconv calls in libcurl) + Note that this is used only for SSL certificate processing */ + CINIT(CONV_FROM_UTF8_FUNCTION, FUNCTIONPOINT, 144), + + /* if the connection proceeds too quickly then need to slow it down */ + /* limit-rate: maximum number of bytes per second to send or receive */ + CINIT(MAX_SEND_SPEED_LARGE, OFF_T, 145), + CINIT(MAX_RECV_SPEED_LARGE, OFF_T, 146), + + /* Pointer to command string to send if USER/PASS fails. */ + CINIT(FTP_ALTERNATIVE_TO_USER, STRINGPOINT, 147), + + /* callback function for setting socket options */ + CINIT(SOCKOPTFUNCTION, FUNCTIONPOINT, 148), + CINIT(SOCKOPTDATA, OBJECTPOINT, 149), + + /* set to 0 to disable session ID re-use for this transfer, default is + enabled (== 1) */ + CINIT(SSL_SESSIONID_CACHE, LONG, 150), + + /* allowed SSH authentication methods */ + CINIT(SSH_AUTH_TYPES, LONG, 151), + + /* Used by scp/sftp to do public/private key authentication */ + CINIT(SSH_PUBLIC_KEYFILE, STRINGPOINT, 152), + CINIT(SSH_PRIVATE_KEYFILE, STRINGPOINT, 153), + + /* Send CCC (Clear Command Channel) after authentication */ + CINIT(FTP_SSL_CCC, LONG, 154), + + /* Same as TIMEOUT and CONNECTTIMEOUT, but with ms resolution */ + CINIT(TIMEOUT_MS, LONG, 155), + CINIT(CONNECTTIMEOUT_MS, LONG, 156), + + /* set to zero to disable the libcurl's decoding and thus pass the raw body + data to the application even when it is encoded/compressed */ + CINIT(HTTP_TRANSFER_DECODING, LONG, 157), + CINIT(HTTP_CONTENT_DECODING, LONG, 158), + + /* Permission used when creating new files and directories on the remote + server for protocols that support it, SFTP/SCP/FILE */ + CINIT(NEW_FILE_PERMS, LONG, 159), + CINIT(NEW_DIRECTORY_PERMS, LONG, 160), + + /* Set the behaviour of POST when redirecting. Values must be set to one + of CURL_REDIR* defines below. This used to be called CURLOPT_POST301 */ + CINIT(POSTREDIR, LONG, 161), + + /* used by scp/sftp to verify the host's public key */ + CINIT(SSH_HOST_PUBLIC_KEY_MD5, STRINGPOINT, 162), + + /* Callback function for opening socket (instead of socket(2)). Optionally, + callback is able change the address or refuse to connect returning + CURL_SOCKET_BAD. The callback should have type + curl_opensocket_callback */ + CINIT(OPENSOCKETFUNCTION, FUNCTIONPOINT, 163), + CINIT(OPENSOCKETDATA, OBJECTPOINT, 164), + + /* POST volatile input fields. */ + CINIT(COPYPOSTFIELDS, OBJECTPOINT, 165), + + /* set transfer mode (;type=<a|i>) when doing FTP via an HTTP proxy */ + CINIT(PROXY_TRANSFER_MODE, LONG, 166), + + /* Callback function for seeking in the input stream */ + CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167), + CINIT(SEEKDATA, OBJECTPOINT, 168), + + /* CRL file */ + CINIT(CRLFILE, STRINGPOINT, 169), + + /* Issuer certificate */ + CINIT(ISSUERCERT, STRINGPOINT, 170), + + /* (IPv6) Address scope */ + CINIT(ADDRESS_SCOPE, LONG, 171), + + /* Collect certificate chain info and allow it to get retrievable with + CURLINFO_CERTINFO after the transfer is complete. */ + CINIT(CERTINFO, LONG, 172), + + /* "name" and "pwd" to use when fetching. */ + CINIT(USERNAME, STRINGPOINT, 173), + CINIT(PASSWORD, STRINGPOINT, 174), + + /* "name" and "pwd" to use with Proxy when fetching. */ + CINIT(PROXYUSERNAME, STRINGPOINT, 175), + CINIT(PROXYPASSWORD, STRINGPOINT, 176), + + /* Comma separated list of hostnames defining no-proxy zones. These should + match both hostnames directly, and hostnames within a domain. For + example, local.com will match local.com and www.local.com, but NOT + notlocal.com or www.notlocal.com. For compatibility with other + implementations of this, .local.com will be considered to be the same as + local.com. A single * is the only valid wildcard, and effectively + disables the use of proxy. */ + CINIT(NOPROXY, STRINGPOINT, 177), + + /* block size for TFTP transfers */ + CINIT(TFTP_BLKSIZE, LONG, 178), + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_SERVICE, STRINGPOINT, 179), /* DEPRECATED, do not use! */ + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_NEC, LONG, 180), + + /* set the bitmask for the protocols that are allowed to be used for the + transfer, which thus helps the app which takes URLs from users or other + external inputs and want to restrict what protocol(s) to deal + with. Defaults to CURLPROTO_ALL. */ + CINIT(PROTOCOLS, LONG, 181), + + /* set the bitmask for the protocols that libcurl is allowed to follow to, + as a subset of the CURLOPT_PROTOCOLS ones. That means the protocol needs + to be set in both bitmasks to be allowed to get redirected to. Defaults + to all protocols except FILE and SCP. */ + CINIT(REDIR_PROTOCOLS, LONG, 182), + + /* set the SSH knownhost file name to use */ + CINIT(SSH_KNOWNHOSTS, STRINGPOINT, 183), + + /* set the SSH host key callback, must point to a curl_sshkeycallback + function */ + CINIT(SSH_KEYFUNCTION, FUNCTIONPOINT, 184), + + /* set the SSH host key callback custom pointer */ + CINIT(SSH_KEYDATA, OBJECTPOINT, 185), + + /* set the SMTP mail originator */ + CINIT(MAIL_FROM, STRINGPOINT, 186), + + /* set the list of SMTP mail receiver(s) */ + CINIT(MAIL_RCPT, OBJECTPOINT, 187), + + /* FTP: send PRET before PASV */ + CINIT(FTP_USE_PRET, LONG, 188), + + /* RTSP request method (OPTIONS, SETUP, PLAY, etc...) */ + CINIT(RTSP_REQUEST, LONG, 189), + + /* The RTSP session identifier */ + CINIT(RTSP_SESSION_ID, STRINGPOINT, 190), + + /* The RTSP stream URI */ + CINIT(RTSP_STREAM_URI, STRINGPOINT, 191), + + /* The Transport: header to use in RTSP requests */ + CINIT(RTSP_TRANSPORT, STRINGPOINT, 192), + + /* Manually initialize the client RTSP CSeq for this handle */ + CINIT(RTSP_CLIENT_CSEQ, LONG, 193), + + /* Manually initialize the server RTSP CSeq for this handle */ + CINIT(RTSP_SERVER_CSEQ, LONG, 194), + + /* The stream to pass to INTERLEAVEFUNCTION. */ + CINIT(INTERLEAVEDATA, OBJECTPOINT, 195), + + /* Let the application define a custom write method for RTP data */ + CINIT(INTERLEAVEFUNCTION, FUNCTIONPOINT, 196), + + /* Turn on wildcard matching */ + CINIT(WILDCARDMATCH, LONG, 197), + + /* Directory matching callback called before downloading of an + individual file (chunk) started */ + CINIT(CHUNK_BGN_FUNCTION, FUNCTIONPOINT, 198), + + /* Directory matching callback called after the file (chunk) + was downloaded, or skipped */ + CINIT(CHUNK_END_FUNCTION, FUNCTIONPOINT, 199), + + /* Change match (fnmatch-like) callback for wildcard matching */ + CINIT(FNMATCH_FUNCTION, FUNCTIONPOINT, 200), + + /* Let the application define custom chunk data pointer */ + CINIT(CHUNK_DATA, OBJECTPOINT, 201), + + /* FNMATCH_FUNCTION user pointer */ + CINIT(FNMATCH_DATA, OBJECTPOINT, 202), + + /* send linked-list of name:port:address sets */ + CINIT(RESOLVE, OBJECTPOINT, 203), + + /* Set a username for authenticated TLS */ + CINIT(TLSAUTH_USERNAME, STRINGPOINT, 204), + + /* Set a password for authenticated TLS */ + CINIT(TLSAUTH_PASSWORD, STRINGPOINT, 205), + + /* Set authentication type for authenticated TLS */ + CINIT(TLSAUTH_TYPE, STRINGPOINT, 206), + + /* Set to 1 to enable the "TE:" header in HTTP requests to ask for + compressed transfer-encoded responses. Set to 0 to disable the use of TE: + in outgoing requests. The current default is 0, but it might change in a + future libcurl release. + + libcurl will ask for the compressed methods it knows of, and if that + isn't any, it will not ask for transfer-encoding at all even if this + option is set to 1. + + */ + CINIT(TRANSFER_ENCODING, LONG, 207), + + /* Callback function for closing socket (instead of close(2)). The callback + should have type curl_closesocket_callback */ + CINIT(CLOSESOCKETFUNCTION, FUNCTIONPOINT, 208), + CINIT(CLOSESOCKETDATA, OBJECTPOINT, 209), + + /* allow GSSAPI credential delegation */ + CINIT(GSSAPI_DELEGATION, LONG, 210), + + /* Set the name servers to use for DNS resolution */ + CINIT(DNS_SERVERS, STRINGPOINT, 211), + + /* Time-out accept operations (currently for FTP only) after this amount + of milliseconds. */ + CINIT(ACCEPTTIMEOUT_MS, LONG, 212), + + /* Set TCP keepalive */ + CINIT(TCP_KEEPALIVE, LONG, 213), + + /* non-universal keepalive knobs (Linux, AIX, HP-UX, more) */ + CINIT(TCP_KEEPIDLE, LONG, 214), + CINIT(TCP_KEEPINTVL, LONG, 215), + + /* Enable/disable specific SSL features with a bitmask, see CURLSSLOPT_* */ + CINIT(SSL_OPTIONS, LONG, 216), + + /* Set the SMTP auth originator */ + CINIT(MAIL_AUTH, STRINGPOINT, 217), + + /* Enable/disable SASL initial response */ + CINIT(SASL_IR, LONG, 218), + + /* Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_xferinfo_callback + * prototype defines. (Deprecates CURLOPT_PROGRESSFUNCTION) */ + CINIT(XFERINFOFUNCTION, FUNCTIONPOINT, 219), + + /* The XOAUTH2 bearer token */ + CINIT(XOAUTH2_BEARER, STRINGPOINT, 220), + + /* Set the interface string to use as outgoing network + * interface for DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_INTERFACE, STRINGPOINT, 221), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP4, STRINGPOINT, 222), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP6, STRINGPOINT, 223), + + /* Set authentication options directly */ + CINIT(LOGIN_OPTIONS, STRINGPOINT, 224), + + /* Enable/disable TLS NPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_NPN, LONG, 225), + + /* Enable/disable TLS ALPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_ALPN, LONG, 226), + + /* Time to wait for a response to a HTTP request containing an + * Expect: 100-continue header before sending the data anyway. */ + CINIT(EXPECT_100_TIMEOUT_MS, LONG, 227), + + /* This points to a linked list of headers used for proxy requests only, + struct curl_slist kind */ + CINIT(PROXYHEADER, OBJECTPOINT, 228), + + /* Pass in a bitmask of "header options" */ + CINIT(HEADEROPT, LONG, 229), + + /* The public key in DER form used to validate the peer public key + this option is used only if SSL_VERIFYPEER is true */ + CINIT(PINNEDPUBLICKEY, STRINGPOINT, 230), + + /* Path to Unix domain socket */ + CINIT(UNIX_SOCKET_PATH, STRINGPOINT, 231), + + /* Set if we should verify the certificate status. */ + CINIT(SSL_VERIFYSTATUS, LONG, 232), + + /* Set if we should enable TLS false start. */ + CINIT(SSL_FALSESTART, LONG, 233), + + /* Do not squash dot-dot sequences */ + CINIT(PATH_AS_IS, LONG, 234), + + /* Proxy Service Name */ + CINIT(PROXY_SERVICE_NAME, STRINGPOINT, 235), + + /* Service Name */ + CINIT(SERVICE_NAME, STRINGPOINT, 236), + + /* Wait/don't wait for pipe/mutex to clarify */ + CINIT(PIPEWAIT, LONG, 237), + + /* Set the protocol used when curl is given a URL without a protocol */ + CINIT(DEFAULT_PROTOCOL, STRINGPOINT, 238), + + /* Set stream weight, 1 - 256 (default is 16) */ + CINIT(STREAM_WEIGHT, LONG, 239), + + /* Set stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS, OBJECTPOINT, 240), + + /* Set E-xclusive stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS_E, OBJECTPOINT, 241), + + /* Do not send any tftp option requests to the server */ + CINIT(TFTP_NO_OPTIONS, LONG, 242), + + /* Linked-list of host:port:connect-to-host:connect-to-port, + overrides the URL's host:port (only for the network layer) */ + CINIT(CONNECT_TO, OBJECTPOINT, 243), + + /* Set TCP Fast Open */ + CINIT(TCP_FASTOPEN, LONG, 244), + + /* Continue to send data if the server responds early with an + * HTTP status code >= 300 */ + CINIT(KEEP_SENDING_ON_ERROR, LONG, 245), + + /* The CApath or CAfile used to validate the proxy certificate + this option is used only if PROXY_SSL_VERIFYPEER is true */ + CINIT(PROXY_CAINFO, STRINGPOINT, 246), + + /* The CApath directory used to validate the proxy certificate + this option is used only if PROXY_SSL_VERIFYPEER is true */ + CINIT(PROXY_CAPATH, STRINGPOINT, 247), + + /* Set if we should verify the proxy in ssl handshake, + set 1 to verify. */ + CINIT(PROXY_SSL_VERIFYPEER, LONG, 248), + + /* Set if we should verify the Common name from the proxy certificate in ssl + * handshake, set 1 to check existence, 2 to ensure that it matches + * the provided hostname. */ + CINIT(PROXY_SSL_VERIFYHOST, LONG, 249), + + /* What version to specifically try to use for proxy. + See CURL_SSLVERSION defines below. */ + CINIT(PROXY_SSLVERSION, LONG, 250), + + /* Set a username for authenticated TLS for proxy */ + CINIT(PROXY_TLSAUTH_USERNAME, STRINGPOINT, 251), + + /* Set a password for authenticated TLS for proxy */ + CINIT(PROXY_TLSAUTH_PASSWORD, STRINGPOINT, 252), + + /* Set authentication type for authenticated TLS for proxy */ + CINIT(PROXY_TLSAUTH_TYPE, STRINGPOINT, 253), + + /* name of the file keeping your private SSL-certificate for proxy */ + CINIT(PROXY_SSLCERT, STRINGPOINT, 254), + + /* type of the file keeping your SSL-certificate ("DER", "PEM", "ENG") for + proxy */ + CINIT(PROXY_SSLCERTTYPE, STRINGPOINT, 255), + + /* name of the file keeping your private SSL-key for proxy */ + CINIT(PROXY_SSLKEY, STRINGPOINT, 256), + + /* type of the file keeping your private SSL-key ("DER", "PEM", "ENG") for + proxy */ + CINIT(PROXY_SSLKEYTYPE, STRINGPOINT, 257), + + /* password for the SSL private key for proxy */ + CINIT(PROXY_KEYPASSWD, STRINGPOINT, 258), + + /* Specify which SSL ciphers to use for proxy */ + CINIT(PROXY_SSL_CIPHER_LIST, STRINGPOINT, 259), + + /* CRL file for proxy */ + CINIT(PROXY_CRLFILE, STRINGPOINT, 260), + + /* Enable/disable specific SSL features with a bitmask for proxy, see + CURLSSLOPT_* */ + CINIT(PROXY_SSL_OPTIONS, LONG, 261), + + /* Name of pre proxy to use. */ + CINIT(PRE_PROXY, STRINGPOINT, 262), + + /* The public key in DER form used to validate the proxy public key + this option is used only if PROXY_SSL_VERIFYPEER is true */ + CINIT(PROXY_PINNEDPUBLICKEY, STRINGPOINT, 263), + + /* Path to an abstract Unix domain socket */ + CINIT(ABSTRACT_UNIX_SOCKET, STRINGPOINT, 264), + + /* Suppress proxy CONNECT response headers from user callbacks */ + CINIT(SUPPRESS_CONNECT_HEADERS, LONG, 265), + + /* The request target, instead of extracted from the URL */ + CINIT(REQUEST_TARGET, STRINGPOINT, 266), + + /* bitmask of allowed auth methods for connections to SOCKS5 proxies */ + CINIT(SOCKS5_AUTH, LONG, 267), + + /* Enable/disable SSH compression */ + CINIT(SSH_COMPRESSION, LONG, 268), + + /* Post MIME data. */ + CINIT(MIMEPOST, OBJECTPOINT, 269), + + CURLOPT_LASTENTRY /* the last unused */ +} CURLoption; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2011 */ + +/* This was added in version 7.19.1 */ +#define CURLOPT_POST301 CURLOPT_POSTREDIR + +/* These are scheduled to disappear by 2009 */ + +/* The following were added in 7.17.0 */ +#define CURLOPT_SSLKEYPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_FTPAPPEND CURLOPT_APPEND +#define CURLOPT_FTPLISTONLY CURLOPT_DIRLISTONLY +#define CURLOPT_FTP_SSL CURLOPT_USE_SSL + +/* The following were added earlier */ + +#define CURLOPT_SSLCERTPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_KRB4LEVEL CURLOPT_KRBLEVEL + +#else +/* This is set if CURL_NO_OLDIES is defined at compile-time */ +#undef CURLOPT_DNS_USE_GLOBAL_CACHE /* soon obsolete */ +#endif + + + /* Below here follows defines for the CURLOPT_IPRESOLVE option. If a host + name resolves addresses using more than one IP protocol version, this + option might be handy to force libcurl to use a specific IP version. */ +#define CURL_IPRESOLVE_WHATEVER 0 /* default, resolves addresses to all IP + versions that your system allows */ +#define CURL_IPRESOLVE_V4 1 /* resolve to IPv4 addresses */ +#define CURL_IPRESOLVE_V6 2 /* resolve to IPv6 addresses */ + + /* three convenient "aliases" that follow the name scheme better */ +#define CURLOPT_RTSPHEADER CURLOPT_HTTPHEADER + + /* These enums are for use with the CURLOPT_HTTP_VERSION option. */ +enum { + CURL_HTTP_VERSION_NONE, /* setting this means we don't care, and that we'd + like the library to choose the best possible + for us! */ + CURL_HTTP_VERSION_1_0, /* please use HTTP 1.0 in the request */ + CURL_HTTP_VERSION_1_1, /* please use HTTP 1.1 in the request */ + CURL_HTTP_VERSION_2_0, /* please use HTTP 2 in the request */ + CURL_HTTP_VERSION_2TLS, /* use version 2 for HTTPS, version 1.1 for HTTP */ + CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE, /* please use HTTP 2 without HTTP/1.1 + Upgrade */ + + CURL_HTTP_VERSION_LAST /* *ILLEGAL* http version */ +}; + +/* Convenience definition simple because the name of the version is HTTP/2 and + not 2.0. The 2_0 version of the enum name was set while the version was + still planned to be 2.0 and we stick to it for compatibility. */ +#define CURL_HTTP_VERSION_2 CURL_HTTP_VERSION_2_0 + +/* + * Public API enums for RTSP requests + */ +enum { + CURL_RTSPREQ_NONE, /* first in list */ + CURL_RTSPREQ_OPTIONS, + CURL_RTSPREQ_DESCRIBE, + CURL_RTSPREQ_ANNOUNCE, + CURL_RTSPREQ_SETUP, + CURL_RTSPREQ_PLAY, + CURL_RTSPREQ_PAUSE, + CURL_RTSPREQ_TEARDOWN, + CURL_RTSPREQ_GET_PARAMETER, + CURL_RTSPREQ_SET_PARAMETER, + CURL_RTSPREQ_RECORD, + CURL_RTSPREQ_RECEIVE, + CURL_RTSPREQ_LAST /* last in list */ +}; + + /* These enums are for use with the CURLOPT_NETRC option. */ +enum CURL_NETRC_OPTION { + CURL_NETRC_IGNORED, /* The .netrc will never be read. + * This is the default. */ + CURL_NETRC_OPTIONAL, /* A user:password in the URL will be preferred + * to one in the .netrc. */ + CURL_NETRC_REQUIRED, /* A user:password in the URL will be ignored. + * Unless one is set programmatically, the .netrc + * will be queried. */ + CURL_NETRC_LAST +}; + +enum { + CURL_SSLVERSION_DEFAULT, + CURL_SSLVERSION_TLSv1, /* TLS 1.x */ + CURL_SSLVERSION_SSLv2, + CURL_SSLVERSION_SSLv3, + CURL_SSLVERSION_TLSv1_0, + CURL_SSLVERSION_TLSv1_1, + CURL_SSLVERSION_TLSv1_2, + CURL_SSLVERSION_TLSv1_3, + + CURL_SSLVERSION_LAST /* never use, keep last */ +}; + +enum { + CURL_SSLVERSION_MAX_NONE = 0, + CURL_SSLVERSION_MAX_DEFAULT = (CURL_SSLVERSION_TLSv1 << 16), + CURL_SSLVERSION_MAX_TLSv1_0 = (CURL_SSLVERSION_TLSv1_0 << 16), + CURL_SSLVERSION_MAX_TLSv1_1 = (CURL_SSLVERSION_TLSv1_1 << 16), + CURL_SSLVERSION_MAX_TLSv1_2 = (CURL_SSLVERSION_TLSv1_2 << 16), + CURL_SSLVERSION_MAX_TLSv1_3 = (CURL_SSLVERSION_TLSv1_3 << 16), + + /* never use, keep last */ + CURL_SSLVERSION_MAX_LAST = (CURL_SSLVERSION_LAST << 16) +}; + +enum CURL_TLSAUTH { + CURL_TLSAUTH_NONE, + CURL_TLSAUTH_SRP, + CURL_TLSAUTH_LAST /* never use, keep last */ +}; + +/* symbols to use with CURLOPT_POSTREDIR. + CURL_REDIR_POST_301, CURL_REDIR_POST_302 and CURL_REDIR_POST_303 + can be bitwise ORed so that CURL_REDIR_POST_301 | CURL_REDIR_POST_302 + | CURL_REDIR_POST_303 == CURL_REDIR_POST_ALL */ + +#define CURL_REDIR_GET_ALL 0 +#define CURL_REDIR_POST_301 1 +#define CURL_REDIR_POST_302 2 +#define CURL_REDIR_POST_303 4 +#define CURL_REDIR_POST_ALL \ + (CURL_REDIR_POST_301|CURL_REDIR_POST_302|CURL_REDIR_POST_303) + +typedef enum { + CURL_TIMECOND_NONE, + + CURL_TIMECOND_IFMODSINCE, + CURL_TIMECOND_IFUNMODSINCE, + CURL_TIMECOND_LASTMOD, + + CURL_TIMECOND_LAST +} curl_TimeCond; + +/* Special size_t value signaling a zero-terminated string. */ +#define CURL_ZERO_TERMINATED ((size_t) -1) + +/* curl_strequal() and curl_strnequal() are subject for removal in a future + release */ +CURL_EXTERN int curl_strequal(const char *s1, const char *s2); +CURL_EXTERN int curl_strnequal(const char *s1, const char *s2, size_t n); + +/* Mime/form handling support. */ +typedef struct curl_mime_s curl_mime; /* Mime context. */ +typedef struct curl_mimepart_s curl_mimepart; /* Mime part context. */ + +/* + * NAME curl_mime_init() + * + * DESCRIPTION + * + * Create a mime context and return its handle. The easy parameter is the + * target handle. + */ +CURL_EXTERN curl_mime *curl_mime_init(CURL *easy); + +/* + * NAME curl_mime_free() + * + * DESCRIPTION + * + * release a mime handle and its substructures. + */ +CURL_EXTERN void curl_mime_free(curl_mime *mime); + +/* + * NAME curl_mime_addpart() + * + * DESCRIPTION + * + * Append a new empty part to the given mime context and return a handle to + * the created part. + */ +CURL_EXTERN curl_mimepart *curl_mime_addpart(curl_mime *mime); + +/* + * NAME curl_mime_name() + * + * DESCRIPTION + * + * Set mime/form part name. + */ +CURL_EXTERN CURLcode curl_mime_name(curl_mimepart *part, const char *name); + +/* + * NAME curl_mime_filename() + * + * DESCRIPTION + * + * Set mime part remote file name. + */ +CURL_EXTERN CURLcode curl_mime_filename(curl_mimepart *part, + const char *filename); + +/* + * NAME curl_mime_type() + * + * DESCRIPTION + * + * Set mime part type. + */ +CURL_EXTERN CURLcode curl_mime_type(curl_mimepart *part, const char *mimetype); + +/* + * NAME curl_mime_encoder() + * + * DESCRIPTION + * + * Set mime data transfer encoder. + */ +CURL_EXTERN CURLcode curl_mime_encoder(curl_mimepart *part, + const char *encoding); + +/* + * NAME curl_mime_data() + * + * DESCRIPTION + * + * Set mime part data source from memory data, + */ +CURL_EXTERN CURLcode curl_mime_data(curl_mimepart *part, + const char *data, size_t datasize); + +/* + * NAME curl_mime_filedata() + * + * DESCRIPTION + * + * Set mime part data source from named file. + */ +CURL_EXTERN CURLcode curl_mime_filedata(curl_mimepart *part, + const char *filename); + +/* + * NAME curl_mime_data_cb() + * + * DESCRIPTION + * + * Set mime part data source from callback function. + */ +CURL_EXTERN CURLcode curl_mime_data_cb(curl_mimepart *part, + curl_off_t datasize, + curl_read_callback readfunc, + curl_seek_callback seekfunc, + curl_free_callback freefunc, + void *arg); + +/* + * NAME curl_mime_subparts() + * + * DESCRIPTION + * + * Set mime part data source from subparts. + */ +CURL_EXTERN CURLcode curl_mime_subparts(curl_mimepart *part, + curl_mime *subparts); +/* + * NAME curl_mime_headers() + * + * DESCRIPTION + * + * Set mime part headers. + */ +CURL_EXTERN CURLcode curl_mime_headers(curl_mimepart *part, + struct curl_slist *headers, + int take_ownership); + +/* Old form API. */ +/* name is uppercase CURLFORM_<name> */ +#ifdef CFINIT +#undef CFINIT +#endif + +#ifdef CURL_ISOCPP +#define CFINIT(name) CURLFORM_ ## name +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define CFINIT(name) CURLFORM_/**/name +#endif + +typedef enum { + CFINIT(NOTHING), /********* the first one is unused ************/ + + /* */ + CFINIT(COPYNAME), + CFINIT(PTRNAME), + CFINIT(NAMELENGTH), + CFINIT(COPYCONTENTS), + CFINIT(PTRCONTENTS), + CFINIT(CONTENTSLENGTH), + CFINIT(FILECONTENT), + CFINIT(ARRAY), + CFINIT(OBSOLETE), + CFINIT(FILE), + + CFINIT(BUFFER), + CFINIT(BUFFERPTR), + CFINIT(BUFFERLENGTH), + + CFINIT(CONTENTTYPE), + CFINIT(CONTENTHEADER), + CFINIT(FILENAME), + CFINIT(END), + CFINIT(OBSOLETE2), + + CFINIT(STREAM), + CFINIT(CONTENTLEN), /* added in 7.46.0, provide a curl_off_t length */ + + CURLFORM_LASTENTRY /* the last unused */ +} CURLformoption; + +#undef CFINIT /* done */ + +/* structure to be used as parameter for CURLFORM_ARRAY */ +struct curl_forms { + CURLformoption option; + const char *value; +}; + +/* use this for multipart formpost building */ +/* Returns code for curl_formadd() + * + * Returns: + * CURL_FORMADD_OK on success + * CURL_FORMADD_MEMORY if the FormInfo allocation fails + * CURL_FORMADD_OPTION_TWICE if one option is given twice for one Form + * CURL_FORMADD_NULL if a null pointer was given for a char + * CURL_FORMADD_MEMORY if the allocation of a FormInfo struct failed + * CURL_FORMADD_UNKNOWN_OPTION if an unknown option was used + * CURL_FORMADD_INCOMPLETE if the some FormInfo is not complete (or error) + * CURL_FORMADD_MEMORY if a curl_httppost struct cannot be allocated + * CURL_FORMADD_MEMORY if some allocation for string copying failed. + * CURL_FORMADD_ILLEGAL_ARRAY if an illegal option is used in an array + * + ***************************************************************************/ +typedef enum { + CURL_FORMADD_OK, /* first, no error */ + + CURL_FORMADD_MEMORY, + CURL_FORMADD_OPTION_TWICE, + CURL_FORMADD_NULL, + CURL_FORMADD_UNKNOWN_OPTION, + CURL_FORMADD_INCOMPLETE, + CURL_FORMADD_ILLEGAL_ARRAY, + CURL_FORMADD_DISABLED, /* libcurl was built with this disabled */ + + CURL_FORMADD_LAST /* last */ +} CURLFORMcode; + +/* + * NAME curl_formadd() + * + * DESCRIPTION + * + * Pretty advanced function for building multi-part formposts. Each invoke + * adds one part that together construct a full post. Then use + * CURLOPT_HTTPPOST to send it off to libcurl. + */ +CURL_EXTERN CURLFORMcode curl_formadd(struct curl_httppost **httppost, + struct curl_httppost **last_post, + ...); + +/* + * callback function for curl_formget() + * The void *arg pointer will be the one passed as second argument to + * curl_formget(). + * The character buffer passed to it must not be freed. + * Should return the buffer length passed to it as the argument "len" on + * success. + */ +typedef size_t (*curl_formget_callback)(void *arg, const char *buf, + size_t len); + +/* + * NAME curl_formget() + * + * DESCRIPTION + * + * Serialize a curl_httppost struct built with curl_formadd(). + * Accepts a void pointer as second argument which will be passed to + * the curl_formget_callback function. + * Returns 0 on success. + */ +CURL_EXTERN int curl_formget(struct curl_httppost *form, void *arg, + curl_formget_callback append); +/* + * NAME curl_formfree() + * + * DESCRIPTION + * + * Free a multipart formpost previously built with curl_formadd(). + */ +CURL_EXTERN void curl_formfree(struct curl_httppost *form); + +/* + * NAME curl_getenv() + * + * DESCRIPTION + * + * Returns a malloc()'ed string that MUST be curl_free()ed after usage is + * complete. DEPRECATED - see lib/README.curlx + */ +CURL_EXTERN char *curl_getenv(const char *variable); + +/* + * NAME curl_version() + * + * DESCRIPTION + * + * Returns a static ascii string of the libcurl version. + */ +CURL_EXTERN char *curl_version(void); + +/* + * NAME curl_easy_escape() + * + * DESCRIPTION + * + * Escapes URL strings (converts all letters consider illegal in URLs to their + * %XX versions). This function returns a new allocated string or NULL if an + * error occurred. + */ +CURL_EXTERN char *curl_easy_escape(CURL *handle, + const char *string, + int length); + +/* the previous version: */ +CURL_EXTERN char *curl_escape(const char *string, + int length); + + +/* + * NAME curl_easy_unescape() + * + * DESCRIPTION + * + * Unescapes URL encoding in strings (converts all %XX codes to their 8bit + * versions). This function returns a new allocated string or NULL if an error + * occurred. + * Conversion Note: On non-ASCII platforms the ASCII %XX codes are + * converted into the host encoding. + */ +CURL_EXTERN char *curl_easy_unescape(CURL *handle, + const char *string, + int length, + int *outlength); + +/* the previous version */ +CURL_EXTERN char *curl_unescape(const char *string, + int length); + +/* + * NAME curl_free() + * + * DESCRIPTION + * + * Provided for de-allocation in the same translation unit that did the + * allocation. Added in libcurl 7.10 + */ +CURL_EXTERN void curl_free(void *p); + +/* + * NAME curl_global_init() + * + * DESCRIPTION + * + * curl_global_init() should be invoked exactly once for each application that + * uses libcurl and before any call of other libcurl functions. + * + * This function is not thread-safe! + */ +CURL_EXTERN CURLcode curl_global_init(long flags); + +/* + * NAME curl_global_init_mem() + * + * DESCRIPTION + * + * curl_global_init() or curl_global_init_mem() should be invoked exactly once + * for each application that uses libcurl. This function can be used to + * initialize libcurl and set user defined memory management callback + * functions. Users can implement memory management routines to check for + * memory leaks, check for mis-use of the curl library etc. User registered + * callback routines with be invoked by this library instead of the system + * memory management routines like malloc, free etc. + */ +CURL_EXTERN CURLcode curl_global_init_mem(long flags, + curl_malloc_callback m, + curl_free_callback f, + curl_realloc_callback r, + curl_strdup_callback s, + curl_calloc_callback c); + +/* + * NAME curl_global_cleanup() + * + * DESCRIPTION + * + * curl_global_cleanup() should be invoked exactly once for each application + * that uses libcurl + */ +CURL_EXTERN void curl_global_cleanup(void); + +/* linked-list structure for the CURLOPT_QUOTE option (and other) */ +struct curl_slist { + char *data; + struct curl_slist *next; +}; + +/* + * NAME curl_global_sslset() + * + * DESCRIPTION + * + * When built with multiple SSL backends, curl_global_sslset() allows to + * choose one. This function can only be called once, and it must be called + * *before* curl_global_init(). + * + * The backend can be identified by the id (e.g. CURLSSLBACKEND_OPENSSL). The + * backend can also be specified via the name parameter (passing -1 as id). + * If both id and name are specified, the name will be ignored. If neither id + * nor name are specified, the function will fail with + * CURLSSLSET_UNKNOWN_BACKEND and set the "avail" pointer to the + * NULL-terminated list of available backends. + * + * Upon success, the function returns CURLSSLSET_OK. + * + * If the specified SSL backend is not available, the function returns + * CURLSSLSET_UNKNOWN_BACKEND and sets the "avail" pointer to a NULL-terminated + * list of available SSL backends. + * + * The SSL backend can be set only once. If it has already been set, a + * subsequent attempt to change it will result in a CURLSSLSET_TOO_LATE. + */ + +typedef struct { + curl_sslbackend id; + const char *name; +} curl_ssl_backend; + +typedef enum { + CURLSSLSET_OK = 0, + CURLSSLSET_UNKNOWN_BACKEND, + CURLSSLSET_TOO_LATE, + CURLSSLSET_NO_BACKENDS /* libcurl was built without any SSL support */ +} CURLsslset; + +CURL_EXTERN CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, + const curl_ssl_backend ***avail); + +/* + * NAME curl_slist_append() + * + * DESCRIPTION + * + * Appends a string to a linked list. If no list exists, it will be created + * first. Returns the new list, after appending. + */ +CURL_EXTERN struct curl_slist *curl_slist_append(struct curl_slist *, + const char *); + +/* + * NAME curl_slist_free_all() + * + * DESCRIPTION + * + * free a previously built curl_slist. + */ +CURL_EXTERN void curl_slist_free_all(struct curl_slist *); + +/* + * NAME curl_getdate() + * + * DESCRIPTION + * + * Returns the time, in seconds since 1 Jan 1970 of the time string given in + * the first argument. The time argument in the second parameter is unused + * and should be set to NULL. + */ +CURL_EXTERN time_t curl_getdate(const char *p, const time_t *unused); + +/* info about the certificate chain, only for OpenSSL builds. Asked + for with CURLOPT_CERTINFO / CURLINFO_CERTINFO */ +struct curl_certinfo { + int num_of_certs; /* number of certificates with information */ + struct curl_slist **certinfo; /* for each index in this array, there's a + linked list with textual information in the + format "name: value" */ +}; + +/* Information about the SSL library used and the respective internal SSL + handle, which can be used to obtain further information regarding the + connection. Asked for with CURLINFO_TLS_SSL_PTR or CURLINFO_TLS_SESSION. */ +struct curl_tlssessioninfo { + curl_sslbackend backend; + void *internals; +}; + +#define CURLINFO_STRING 0x100000 +#define CURLINFO_LONG 0x200000 +#define CURLINFO_DOUBLE 0x300000 +#define CURLINFO_SLIST 0x400000 +#define CURLINFO_PTR 0x400000 /* same as SLIST */ +#define CURLINFO_SOCKET 0x500000 +#define CURLINFO_OFF_T 0x600000 +#define CURLINFO_MASK 0x0fffff +#define CURLINFO_TYPEMASK 0xf00000 + +typedef enum { + CURLINFO_NONE, /* first, never use this */ + CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1, + CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2, + CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3, + CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4, + CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5, + CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6, + CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7, + CURLINFO_SIZE_UPLOAD_T = CURLINFO_OFF_T + 7, + CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8, + CURLINFO_SIZE_DOWNLOAD_T = CURLINFO_OFF_T + 8, + CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9, + CURLINFO_SPEED_DOWNLOAD_T = CURLINFO_OFF_T + 9, + CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10, + CURLINFO_SPEED_UPLOAD_T = CURLINFO_OFF_T + 10, + CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11, + CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12, + CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13, + CURLINFO_FILETIME = CURLINFO_LONG + 14, + CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15, + CURLINFO_CONTENT_LENGTH_DOWNLOAD_T = CURLINFO_OFF_T + 15, + CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16, + CURLINFO_CONTENT_LENGTH_UPLOAD_T = CURLINFO_OFF_T + 16, + CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17, + CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18, + CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19, + CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20, + CURLINFO_PRIVATE = CURLINFO_STRING + 21, + CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22, + CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23, + CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24, + CURLINFO_OS_ERRNO = CURLINFO_LONG + 25, + CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26, + CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27, + CURLINFO_COOKIELIST = CURLINFO_SLIST + 28, + CURLINFO_LASTSOCKET = CURLINFO_LONG + 29, + CURLINFO_FTP_ENTRY_PATH = CURLINFO_STRING + 30, + CURLINFO_REDIRECT_URL = CURLINFO_STRING + 31, + CURLINFO_PRIMARY_IP = CURLINFO_STRING + 32, + CURLINFO_APPCONNECT_TIME = CURLINFO_DOUBLE + 33, + CURLINFO_CERTINFO = CURLINFO_PTR + 34, + CURLINFO_CONDITION_UNMET = CURLINFO_LONG + 35, + CURLINFO_RTSP_SESSION_ID = CURLINFO_STRING + 36, + CURLINFO_RTSP_CLIENT_CSEQ = CURLINFO_LONG + 37, + CURLINFO_RTSP_SERVER_CSEQ = CURLINFO_LONG + 38, + CURLINFO_RTSP_CSEQ_RECV = CURLINFO_LONG + 39, + CURLINFO_PRIMARY_PORT = CURLINFO_LONG + 40, + CURLINFO_LOCAL_IP = CURLINFO_STRING + 41, + CURLINFO_LOCAL_PORT = CURLINFO_LONG + 42, + CURLINFO_TLS_SESSION = CURLINFO_PTR + 43, + CURLINFO_ACTIVESOCKET = CURLINFO_SOCKET + 44, + CURLINFO_TLS_SSL_PTR = CURLINFO_PTR + 45, + CURLINFO_HTTP_VERSION = CURLINFO_LONG + 46, + CURLINFO_PROXY_SSL_VERIFYRESULT = CURLINFO_LONG + 47, + CURLINFO_PROTOCOL = CURLINFO_LONG + 48, + CURLINFO_SCHEME = CURLINFO_STRING + 49, + /* Fill in new entries below here! */ + + CURLINFO_LASTONE = 49 +} CURLINFO; + +/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as + CURLINFO_HTTP_CODE */ +#define CURLINFO_HTTP_CODE CURLINFO_RESPONSE_CODE + +typedef enum { + CURLCLOSEPOLICY_NONE, /* first, never use this */ + + CURLCLOSEPOLICY_OLDEST, + CURLCLOSEPOLICY_LEAST_RECENTLY_USED, + CURLCLOSEPOLICY_LEAST_TRAFFIC, + CURLCLOSEPOLICY_SLOWEST, + CURLCLOSEPOLICY_CALLBACK, + + CURLCLOSEPOLICY_LAST /* last, never use this */ +} curl_closepolicy; + +#define CURL_GLOBAL_SSL (1<<0) /* no purpose since since 7.57.0 */ +#define CURL_GLOBAL_WIN32 (1<<1) +#define CURL_GLOBAL_ALL (CURL_GLOBAL_SSL|CURL_GLOBAL_WIN32) +#define CURL_GLOBAL_NOTHING 0 +#define CURL_GLOBAL_DEFAULT CURL_GLOBAL_ALL +#define CURL_GLOBAL_ACK_EINTR (1<<2) + + +/***************************************************************************** + * Setup defines, protos etc for the sharing stuff. + */ + +/* Different data locks for a single share */ +typedef enum { + CURL_LOCK_DATA_NONE = 0, + /* CURL_LOCK_DATA_SHARE is used internally to say that + * the locking is just made to change the internal state of the share + * itself. + */ + CURL_LOCK_DATA_SHARE, + CURL_LOCK_DATA_COOKIE, + CURL_LOCK_DATA_DNS, + CURL_LOCK_DATA_SSL_SESSION, + CURL_LOCK_DATA_CONNECT, + CURL_LOCK_DATA_LAST +} curl_lock_data; + +/* Different lock access types */ +typedef enum { + CURL_LOCK_ACCESS_NONE = 0, /* unspecified action */ + CURL_LOCK_ACCESS_SHARED = 1, /* for read perhaps */ + CURL_LOCK_ACCESS_SINGLE = 2, /* for write perhaps */ + CURL_LOCK_ACCESS_LAST /* never use */ +} curl_lock_access; + +typedef void (*curl_lock_function)(CURL *handle, + curl_lock_data data, + curl_lock_access locktype, + void *userptr); +typedef void (*curl_unlock_function)(CURL *handle, + curl_lock_data data, + void *userptr); + + +typedef enum { + CURLSHE_OK, /* all is fine */ + CURLSHE_BAD_OPTION, /* 1 */ + CURLSHE_IN_USE, /* 2 */ + CURLSHE_INVALID, /* 3 */ + CURLSHE_NOMEM, /* 4 out of memory */ + CURLSHE_NOT_BUILT_IN, /* 5 feature not present in lib */ + CURLSHE_LAST /* never use */ +} CURLSHcode; + +typedef enum { + CURLSHOPT_NONE, /* don't use */ + CURLSHOPT_SHARE, /* specify a data type to share */ + CURLSHOPT_UNSHARE, /* specify which data type to stop sharing */ + CURLSHOPT_LOCKFUNC, /* pass in a 'curl_lock_function' pointer */ + CURLSHOPT_UNLOCKFUNC, /* pass in a 'curl_unlock_function' pointer */ + CURLSHOPT_USERDATA, /* pass in a user data pointer used in the lock/unlock + callback functions */ + CURLSHOPT_LAST /* never use */ +} CURLSHoption; + +CURL_EXTERN CURLSH *curl_share_init(void); +CURL_EXTERN CURLSHcode curl_share_setopt(CURLSH *, CURLSHoption option, ...); +CURL_EXTERN CURLSHcode curl_share_cleanup(CURLSH *); + +/**************************************************************************** + * Structures for querying information about the curl library at runtime. + */ + +typedef enum { + CURLVERSION_FIRST, + CURLVERSION_SECOND, + CURLVERSION_THIRD, + CURLVERSION_FOURTH, + CURLVERSION_FIFTH, + CURLVERSION_LAST /* never actually use this */ +} CURLversion; + +/* The 'CURLVERSION_NOW' is the symbolic name meant to be used by + basically all programs ever that want to get version information. It is + meant to be a built-in version number for what kind of struct the caller + expects. If the struct ever changes, we redefine the NOW to another enum + from above. */ +#define CURLVERSION_NOW CURLVERSION_FIFTH + +typedef struct { + CURLversion age; /* age of the returned struct */ + const char *version; /* LIBCURL_VERSION */ + unsigned int version_num; /* LIBCURL_VERSION_NUM */ + const char *host; /* OS/host/cpu/machine when configured */ + int features; /* bitmask, see defines below */ + const char *ssl_version; /* human readable string */ + long ssl_version_num; /* not used anymore, always 0 */ + const char *libz_version; /* human readable string */ + /* protocols is terminated by an entry with a NULL protoname */ + const char * const *protocols; + + /* The fields below this were added in CURLVERSION_SECOND */ + const char *ares; + int ares_num; + + /* This field was added in CURLVERSION_THIRD */ + const char *libidn; + + /* These field were added in CURLVERSION_FOURTH */ + + /* Same as '_libiconv_version' if built with HAVE_ICONV */ + int iconv_ver_num; + + const char *libssh_version; /* human readable string */ + + /* These fields were added in CURLVERSION_FIFTH */ + + unsigned int brotli_ver_num; /* Numeric Brotli version + (MAJOR << 24) | (MINOR << 12) | PATCH */ + const char *brotli_version; /* human readable string. */ + +} curl_version_info_data; + +#define CURL_VERSION_IPV6 (1<<0) /* IPv6-enabled */ +#define CURL_VERSION_KERBEROS4 (1<<1) /* Kerberos V4 auth is supported + (deprecated) */ +#define CURL_VERSION_SSL (1<<2) /* SSL options are present */ +#define CURL_VERSION_LIBZ (1<<3) /* libz features are present */ +#define CURL_VERSION_NTLM (1<<4) /* NTLM auth is supported */ +#define CURL_VERSION_GSSNEGOTIATE (1<<5) /* Negotiate auth is supported + (deprecated) */ +#define CURL_VERSION_DEBUG (1<<6) /* Built with debug capabilities */ +#define CURL_VERSION_ASYNCHDNS (1<<7) /* Asynchronous DNS resolves */ +#define CURL_VERSION_SPNEGO (1<<8) /* SPNEGO auth is supported */ +#define CURL_VERSION_LARGEFILE (1<<9) /* Supports files larger than 2GB */ +#define CURL_VERSION_IDN (1<<10) /* Internationized Domain Names are + supported */ +#define CURL_VERSION_SSPI (1<<11) /* Built against Windows SSPI */ +#define CURL_VERSION_CONV (1<<12) /* Character conversions supported */ +#define CURL_VERSION_CURLDEBUG (1<<13) /* Debug memory tracking supported */ +#define CURL_VERSION_TLSAUTH_SRP (1<<14) /* TLS-SRP auth is supported */ +#define CURL_VERSION_NTLM_WB (1<<15) /* NTLM delegation to winbind helper + is supported */ +#define CURL_VERSION_HTTP2 (1<<16) /* HTTP2 support built-in */ +#define CURL_VERSION_GSSAPI (1<<17) /* Built against a GSS-API library */ +#define CURL_VERSION_KERBEROS5 (1<<18) /* Kerberos V5 auth is supported */ +#define CURL_VERSION_UNIX_SOCKETS (1<<19) /* Unix domain sockets support */ +#define CURL_VERSION_PSL (1<<20) /* Mozilla's Public Suffix List, used + for cookie domain verification */ +#define CURL_VERSION_HTTPS_PROXY (1<<21) /* HTTPS-proxy support built-in */ +#define CURL_VERSION_MULTI_SSL (1<<22) /* Multiple SSL backends available */ +#define CURL_VERSION_BROTLI (1<<23) /* Brotli features are present. */ + + /* + * NAME curl_version_info() + * + * DESCRIPTION + * + * This function returns a pointer to a static copy of the version info + * struct. See above. + */ +CURL_EXTERN curl_version_info_data *curl_version_info(CURLversion); + +/* + * NAME curl_easy_strerror() + * + * DESCRIPTION + * + * The curl_easy_strerror function may be used to turn a CURLcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_easy_strerror(CURLcode); + +/* + * NAME curl_share_strerror() + * + * DESCRIPTION + * + * The curl_share_strerror function may be used to turn a CURLSHcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_share_strerror(CURLSHcode); + +/* + * NAME curl_easy_pause() + * + * DESCRIPTION + * + * The curl_easy_pause function pauses or unpauses transfers. Select the new + * state by setting the bitmask, use the convenience defines below. + * + */ +CURL_EXTERN CURLcode curl_easy_pause(CURL *handle, int bitmask); + +#define CURLPAUSE_RECV (1<<0) +#define CURLPAUSE_RECV_CONT (0) + +#define CURLPAUSE_SEND (1<<2) +#define CURLPAUSE_SEND_CONT (0) + +#define CURLPAUSE_ALL (CURLPAUSE_RECV|CURLPAUSE_SEND) +#define CURLPAUSE_CONT (CURLPAUSE_RECV_CONT|CURLPAUSE_SEND_CONT) + +#ifdef __cplusplus +} +#endif + +/* unfortunately, the easy.h and multi.h include files need options and info + stuff before they can be included! */ +#include "easy.h" /* nothing in curl is fun without the easy stuff */ +#include "multi.h" + +/* the typechecker doesn't work in C++ (yet) */ +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && \ + !defined(__cplusplus) && !defined(CURL_DISABLE_TYPECHECK) +#include "typecheck-gcc.h" +#else +#if defined(__STDC__) && (__STDC__ >= 1) +/* This preprocessor magic that replaces a call with the exact same call is + only done to make sure application authors pass exactly three arguments + to these functions. */ +#define curl_easy_setopt(handle,opt,param) curl_easy_setopt(handle,opt,param) +#define curl_easy_getinfo(handle,info,arg) curl_easy_getinfo(handle,info,arg) +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) +#endif /* __STDC__ >= 1 */ +#endif /* gcc >= 4.3 && !__cplusplus */ + +#endif /* __CURL_CURL_H */ diff --git a/ext/curl-7.58.0/Win32/include/curl/curlver.h b/ext/curl-7.58.0/Win32/include/curl/curlver.h new file mode 100644 index 00000000..6d93cc11 --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/curlver.h @@ -0,0 +1,77 @@ +#ifndef __CURL_CURLVER_H +#define __CURL_CURLVER_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* This header file contains nothing but libcurl version info, generated by + a script at release-time. This was made its own header file in 7.11.2 */ + +/* This is the global package copyright */ +#define LIBCURL_COPYRIGHT "1996 - 2017 Daniel Stenberg, <daniel@haxx.se>." + +/* This is the version number of the libcurl package from which this header + file origins: */ +#define LIBCURL_VERSION "7.58.0" + +/* The numeric version number is also available "in parts" by using these + defines: */ +#define LIBCURL_VERSION_MAJOR 7 +#define LIBCURL_VERSION_MINOR 58 +#define LIBCURL_VERSION_PATCH 0 + +/* This is the numeric version of the libcurl version number, meant for easier + parsing and comparions by programs. The LIBCURL_VERSION_NUM define will + always follow this syntax: + + 0xXXYYZZ + + Where XX, YY and ZZ are the main version, release and patch numbers in + hexadecimal (using 8 bits each). All three numbers are always represented + using two digits. 1.2 would appear as "0x010200" while version 9.11.7 + appears as "0x090b07". + + This 6-digit (24 bits) hexadecimal number does not show pre-release number, + and it is always a greater number in a more recent release. It makes + comparisons with greater than and less than work. + + Note: This define is the full hex number and _does not_ use the + CURL_VERSION_BITS() macro since curl's own configure script greps for it + and needs it to contain the full number. +*/ +#define LIBCURL_VERSION_NUM 0x073a00 + +/* + * This is the date and time when the full source package was created. The + * timestamp is not stored in git, as the timestamp is properly set in the + * tarballs by the maketgz script. + * + * The format of the date follows this template: + * + * "2007-11-23" + */ +#define LIBCURL_TIMESTAMP "2018-01-24" + +#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) +#define CURL_AT_LEAST_VERSION(x,y,z) \ + (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) + +#endif /* __CURL_CURLVER_H */ diff --git a/ext/curl-7.58.0/Win32/include/curl/easy.h b/ext/curl-7.58.0/Win32/include/curl/easy.h new file mode 100644 index 00000000..752c5049 --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/easy.h @@ -0,0 +1,102 @@ +#ifndef __CURL_EASY_H +#define __CURL_EASY_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN CURL *curl_easy_init(void); +CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); +CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); +CURL_EXTERN void curl_easy_cleanup(CURL *curl); + +/* + * NAME curl_easy_getinfo() + * + * DESCRIPTION + * + * Request internal information from the curl session with this function. The + * third argument MUST be a pointer to a long, a pointer to a char * or a + * pointer to a double (as the documentation describes elsewhere). The data + * pointed to will be filled in accordingly and can be relied upon only if the + * function returns CURLE_OK. This function is intended to get used *AFTER* a + * performed transfer, all results from this function are undefined until the + * transfer is completed. + */ +CURL_EXTERN CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...); + + +/* + * NAME curl_easy_duphandle() + * + * DESCRIPTION + * + * Creates a new curl session handle with the same options set for the handle + * passed in. Duplicating a handle could only be a matter of cloning data and + * options, internal state info and things like persistent connections cannot + * be transferred. It is useful in multithreaded applications when you can run + * curl_easy_duphandle() for each new thread to avoid a series of identical + * curl_easy_setopt() invokes in every thread. + */ +CURL_EXTERN CURL *curl_easy_duphandle(CURL *curl); + +/* + * NAME curl_easy_reset() + * + * DESCRIPTION + * + * Re-initializes a CURL handle to the default values. This puts back the + * handle to the same state as it was in when it was just created. + * + * It does keep: live connections, the Session ID cache, the DNS cache and the + * cookies. + */ +CURL_EXTERN void curl_easy_reset(CURL *curl); + +/* + * NAME curl_easy_recv() + * + * DESCRIPTION + * + * Receives data from the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen, + size_t *n); + +/* + * NAME curl_easy_send() + * + * DESCRIPTION + * + * Sends data over the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer, + size_t buflen, size_t *n); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ext/curl-7.58.0/Win32/include/curl/mprintf.h b/ext/curl-7.58.0/Win32/include/curl/mprintf.h new file mode 100644 index 00000000..e20f546e --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/mprintf.h @@ -0,0 +1,50 @@ +#ifndef __CURL_MPRINTF_H +#define __CURL_MPRINTF_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include <stdarg.h> +#include <stdio.h> /* needed for FILE */ +#include "curl.h" /* for CURL_EXTERN */ + +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN int curl_mprintf(const char *format, ...); +CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...); +CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...); +CURL_EXTERN int curl_msnprintf(char *buffer, size_t maxlength, + const char *format, ...); +CURL_EXTERN int curl_mvprintf(const char *format, va_list args); +CURL_EXTERN int curl_mvfprintf(FILE *fd, const char *format, va_list args); +CURL_EXTERN int curl_mvsprintf(char *buffer, const char *format, va_list args); +CURL_EXTERN int curl_mvsnprintf(char *buffer, size_t maxlength, + const char *format, va_list args); +CURL_EXTERN char *curl_maprintf(const char *format, ...); +CURL_EXTERN char *curl_mvaprintf(const char *format, va_list args); + +#ifdef __cplusplus +} +#endif + +#endif /* __CURL_MPRINTF_H */ diff --git a/ext/curl-7.58.0/Win32/include/curl/multi.h b/ext/curl-7.58.0/Win32/include/curl/multi.h new file mode 100644 index 00000000..911c91dd --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/multi.h @@ -0,0 +1,439 @@ +#ifndef __CURL_MULTI_H +#define __CURL_MULTI_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +/* + This is an "external" header file. Don't give away any internals here! + + GOALS + + o Enable a "pull" interface. The application that uses libcurl decides where + and when to ask libcurl to get/send data. + + o Enable multiple simultaneous transfers in the same thread without making it + complicated for the application. + + o Enable the application to select() on its own file descriptors and curl's + file descriptors simultaneous easily. + +*/ + +/* + * This header file should not really need to include "curl.h" since curl.h + * itself includes this file and we expect user applications to do #include + * <curl/curl.h> without the need for especially including multi.h. + * + * For some reason we added this include here at one point, and rather than to + * break existing (wrongly written) libcurl applications, we leave it as-is + * but with this warning attached. + */ +#include "curl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(BUILDING_LIBCURL) || defined(CURL_STRICTER) +typedef struct Curl_multi CURLM; +#else +typedef void CURLM; +#endif + +typedef enum { + CURLM_CALL_MULTI_PERFORM = -1, /* please call curl_multi_perform() or + curl_multi_socket*() soon */ + CURLM_OK, + CURLM_BAD_HANDLE, /* the passed-in handle is not a valid CURLM handle */ + CURLM_BAD_EASY_HANDLE, /* an easy handle was not good/valid */ + CURLM_OUT_OF_MEMORY, /* if you ever get this, you're in deep sh*t */ + CURLM_INTERNAL_ERROR, /* this is a libcurl bug */ + CURLM_BAD_SOCKET, /* the passed in socket argument did not match */ + CURLM_UNKNOWN_OPTION, /* curl_multi_setopt() with unsupported option */ + CURLM_ADDED_ALREADY, /* an easy handle already added to a multi handle was + attempted to get added - again */ + CURLM_LAST +} CURLMcode; + +/* just to make code nicer when using curl_multi_socket() you can now check + for CURLM_CALL_MULTI_SOCKET too in the same style it works for + curl_multi_perform() and CURLM_CALL_MULTI_PERFORM */ +#define CURLM_CALL_MULTI_SOCKET CURLM_CALL_MULTI_PERFORM + +/* bitmask bits for CURLMOPT_PIPELINING */ +#define CURLPIPE_NOTHING 0L +#define CURLPIPE_HTTP1 1L +#define CURLPIPE_MULTIPLEX 2L + +typedef enum { + CURLMSG_NONE, /* first, not used */ + CURLMSG_DONE, /* This easy handle has completed. 'result' contains + the CURLcode of the transfer */ + CURLMSG_LAST /* last, not used */ +} CURLMSG; + +struct CURLMsg { + CURLMSG msg; /* what this message means */ + CURL *easy_handle; /* the handle it concerns */ + union { + void *whatever; /* message-specific data */ + CURLcode result; /* return code for transfer */ + } data; +}; +typedef struct CURLMsg CURLMsg; + +/* Based on poll(2) structure and values. + * We don't use pollfd and POLL* constants explicitly + * to cover platforms without poll(). */ +#define CURL_WAIT_POLLIN 0x0001 +#define CURL_WAIT_POLLPRI 0x0002 +#define CURL_WAIT_POLLOUT 0x0004 + +struct curl_waitfd { + curl_socket_t fd; + short events; + short revents; /* not supported yet */ +}; + +/* + * Name: curl_multi_init() + * + * Desc: inititalize multi-style curl usage + * + * Returns: a new CURLM handle to use in all 'curl_multi' functions. + */ +CURL_EXTERN CURLM *curl_multi_init(void); + +/* + * Name: curl_multi_add_handle() + * + * Desc: add a standard curl handle to the multi stack + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_add_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_remove_handle() + * + * Desc: removes a curl handle from the multi stack again + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_remove_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_fdset() + * + * Desc: Ask curl for its fd_set sets. The app can use these to select() or + * poll() on. We want curl_multi_perform() called as soon as one of + * them are ready. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_fdset(CURLM *multi_handle, + fd_set *read_fd_set, + fd_set *write_fd_set, + fd_set *exc_fd_set, + int *max_fd); + +/* + * Name: curl_multi_wait() + * + * Desc: Poll on all fds within a CURLM set as well as any + * additional fds passed to the function. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_wait(CURLM *multi_handle, + struct curl_waitfd extra_fds[], + unsigned int extra_nfds, + int timeout_ms, + int *ret); + + /* + * Name: curl_multi_perform() + * + * Desc: When the app thinks there's data available for curl it calls this + * function to read/write whatever there is right now. This returns + * as soon as the reads and writes are done. This function does not + * require that there actually is data available for reading or that + * data can be written, it can be called just in case. It returns + * the number of handles that still transfer data in the second + * argument's integer-pointer. + * + * Returns: CURLMcode type, general multi error code. *NOTE* that this only + * returns errors etc regarding the whole multi stack. There might + * still have occurred problems on invidual transfers even when this + * returns OK. + */ +CURL_EXTERN CURLMcode curl_multi_perform(CURLM *multi_handle, + int *running_handles); + + /* + * Name: curl_multi_cleanup() + * + * Desc: Cleans up and removes a whole multi stack. It does not free or + * touch any individual easy handles in any way. We need to define + * in what state those handles will be if this function is called + * in the middle of a transfer. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_cleanup(CURLM *multi_handle); + +/* + * Name: curl_multi_info_read() + * + * Desc: Ask the multi handle if there's any messages/informationals from + * the individual transfers. Messages include informationals such as + * error code from the transfer or just the fact that a transfer is + * completed. More details on these should be written down as well. + * + * Repeated calls to this function will return a new struct each + * time, until a special "end of msgs" struct is returned as a signal + * that there is no more to get at this point. + * + * The data the returned pointer points to will not survive calling + * curl_multi_cleanup(). + * + * The 'CURLMsg' struct is meant to be very simple and only contain + * very basic information. If more involved information is wanted, + * we will provide the particular "transfer handle" in that struct + * and that should/could/would be used in subsequent + * curl_easy_getinfo() calls (or similar). The point being that we + * must never expose complex structs to applications, as then we'll + * undoubtably get backwards compatibility problems in the future. + * + * Returns: A pointer to a filled-in struct, or NULL if it failed or ran out + * of structs. It also writes the number of messages left in the + * queue (after this read) in the integer the second argument points + * to. + */ +CURL_EXTERN CURLMsg *curl_multi_info_read(CURLM *multi_handle, + int *msgs_in_queue); + +/* + * Name: curl_multi_strerror() + * + * Desc: The curl_multi_strerror function may be used to turn a CURLMcode + * value into the equivalent human readable error string. This is + * useful for printing meaningful error messages. + * + * Returns: A pointer to a zero-terminated error message. + */ +CURL_EXTERN const char *curl_multi_strerror(CURLMcode); + +/* + * Name: curl_multi_socket() and + * curl_multi_socket_all() + * + * Desc: An alternative version of curl_multi_perform() that allows the + * application to pass in one of the file descriptors that have been + * detected to have "action" on them and let libcurl perform. + * See man page for details. + */ +#define CURL_POLL_NONE 0 +#define CURL_POLL_IN 1 +#define CURL_POLL_OUT 2 +#define CURL_POLL_INOUT 3 +#define CURL_POLL_REMOVE 4 + +#define CURL_SOCKET_TIMEOUT CURL_SOCKET_BAD + +#define CURL_CSELECT_IN 0x01 +#define CURL_CSELECT_OUT 0x02 +#define CURL_CSELECT_ERR 0x04 + +typedef int (*curl_socket_callback)(CURL *easy, /* easy handle */ + curl_socket_t s, /* socket */ + int what, /* see above */ + void *userp, /* private callback + pointer */ + void *socketp); /* private socket + pointer */ +/* + * Name: curl_multi_timer_callback + * + * Desc: Called by libcurl whenever the library detects a change in the + * maximum number of milliseconds the app is allowed to wait before + * curl_multi_socket() or curl_multi_perform() must be called + * (to allow libcurl's timed events to take place). + * + * Returns: The callback should return zero. + */ +typedef int (*curl_multi_timer_callback)(CURLM *multi, /* multi handle */ + long timeout_ms, /* see above */ + void *userp); /* private callback + pointer */ + +CURL_EXTERN CURLMcode curl_multi_socket(CURLM *multi_handle, curl_socket_t s, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_action(CURLM *multi_handle, + curl_socket_t s, + int ev_bitmask, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_all(CURLM *multi_handle, + int *running_handles); + +#ifndef CURL_ALLOW_OLD_MULTI_SOCKET +/* This macro below was added in 7.16.3 to push users who recompile to use + the new curl_multi_socket_action() instead of the old curl_multi_socket() +*/ +#define curl_multi_socket(x,y,z) curl_multi_socket_action(x,y,0,z) +#endif + +/* + * Name: curl_multi_timeout() + * + * Desc: Returns the maximum number of milliseconds the app is allowed to + * wait before curl_multi_socket() or curl_multi_perform() must be + * called (to allow libcurl's timed events to take place). + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_timeout(CURLM *multi_handle, + long *milliseconds); + +#undef CINIT /* re-using the same name as in curl.h */ + +#ifdef CURL_ISOCPP +#define CINIT(name,type,num) CURLMOPT_ ## name = CURLOPTTYPE_ ## type + num +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLMOPT_/**/name = type + number +#endif + +typedef enum { + /* This is the socket callback function pointer */ + CINIT(SOCKETFUNCTION, FUNCTIONPOINT, 1), + + /* This is the argument passed to the socket callback */ + CINIT(SOCKETDATA, OBJECTPOINT, 2), + + /* set to 1 to enable pipelining for this multi handle */ + CINIT(PIPELINING, LONG, 3), + + /* This is the timer callback function pointer */ + CINIT(TIMERFUNCTION, FUNCTIONPOINT, 4), + + /* This is the argument passed to the timer callback */ + CINIT(TIMERDATA, OBJECTPOINT, 5), + + /* maximum number of entries in the connection cache */ + CINIT(MAXCONNECTS, LONG, 6), + + /* maximum number of (pipelining) connections to one host */ + CINIT(MAX_HOST_CONNECTIONS, LONG, 7), + + /* maximum number of requests in a pipeline */ + CINIT(MAX_PIPELINE_LENGTH, LONG, 8), + + /* a connection with a content-length longer than this + will not be considered for pipelining */ + CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9), + + /* a connection with a chunk length longer than this + will not be considered for pipelining */ + CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10), + + /* a list of site names(+port) that are blacklisted from + pipelining */ + CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11), + + /* a list of server types that are blacklisted from + pipelining */ + CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12), + + /* maximum number of open connections in total */ + CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13), + + /* This is the server push callback function pointer */ + CINIT(PUSHFUNCTION, FUNCTIONPOINT, 14), + + /* This is the argument passed to the server push callback */ + CINIT(PUSHDATA, OBJECTPOINT, 15), + + CURLMOPT_LASTENTRY /* the last unused */ +} CURLMoption; + + +/* + * Name: curl_multi_setopt() + * + * Desc: Sets options for the multi handle. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_setopt(CURLM *multi_handle, + CURLMoption option, ...); + + +/* + * Name: curl_multi_assign() + * + * Desc: This function sets an association in the multi handle between the + * given socket and a private pointer of the application. This is + * (only) useful for curl_multi_socket uses. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle, + curl_socket_t sockfd, void *sockp); + + +/* + * Name: curl_push_callback + * + * Desc: This callback gets called when a new stream is being pushed by the + * server. It approves or denies the new stream. + * + * Returns: CURL_PUSH_OK or CURL_PUSH_DENY. + */ +#define CURL_PUSH_OK 0 +#define CURL_PUSH_DENY 1 + +struct curl_pushheaders; /* forward declaration only */ + +CURL_EXTERN char *curl_pushheader_bynum(struct curl_pushheaders *h, + size_t num); +CURL_EXTERN char *curl_pushheader_byname(struct curl_pushheaders *h, + const char *name); + +typedef int (*curl_push_callback)(CURL *parent, + CURL *easy, + size_t num_headers, + struct curl_pushheaders *headers, + void *userp); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif diff --git a/ext/curl-7.58.0/Win32/include/curl/stdcheaders.h b/ext/curl-7.58.0/Win32/include/curl/stdcheaders.h new file mode 100644 index 00000000..027b6f42 --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/stdcheaders.h @@ -0,0 +1,33 @@ +#ifndef __STDC_HEADERS_H +#define __STDC_HEADERS_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include <sys/types.h> + +size_t fread(void *, size_t, size_t, FILE *); +size_t fwrite(const void *, size_t, size_t, FILE *); + +int strcasecmp(const char *, const char *); +int strncasecmp(const char *, const char *, size_t); + +#endif /* __STDC_HEADERS_H */ diff --git a/ext/curl-7.58.0/Win32/include/curl/system.h b/ext/curl-7.58.0/Win32/include/curl/system.h new file mode 100644 index 00000000..07bbd9ca --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/system.h @@ -0,0 +1,473 @@ +#ifndef __CURL_SYSTEM_H +#define __CURL_SYSTEM_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* + * Try to keep one section per platform, compiler and architecture, otherwise, + * if an existing section is reused for a different one and later on the + * original is adjusted, probably the piggybacking one can be adversely + * changed. + * + * In order to differentiate between platforms/compilers/architectures use + * only compiler built in predefined preprocessor symbols. + * + * curl_off_t + * ---------- + * + * For any given platform/compiler curl_off_t must be typedef'ed to a 64-bit + * wide signed integral data type. The width of this data type must remain + * constant and independent of any possible large file support settings. + * + * As an exception to the above, curl_off_t shall be typedef'ed to a 32-bit + * wide signed integral data type if there is no 64-bit type. + * + * As a general rule, curl_off_t shall not be mapped to off_t. This rule shall + * only be violated if off_t is the only 64-bit data type available and the + * size of off_t is independent of large file support settings. Keep your + * build on the safe side avoiding an off_t gating. If you have a 64-bit + * off_t then take for sure that another 64-bit data type exists, dig deeper + * and you will find it. + * + */ + +#if defined(__DJGPP__) || defined(__GO32__) +# if defined(__DJGPP__) && (__DJGPP__ > 1) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__SALFORDC__) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__BORLANDC__) +# if (__BORLANDC__ < 0x520) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__TURBOC__) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__WATCOMC__) +# if defined(__386__) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__POCC__) +# if (__POCC__ < 280) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# elif defined(_MSC_VER) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# else +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__LCC__) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__SYMBIAN32__) +# if defined(__EABI__) /* Treat all ARM compilers equally */ +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__CW32__) +# pragma longlong on +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__VC32__) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T unsigned int + +#elif defined(__MWERKS__) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(_WIN32_WCE) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__MINGW32__) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_WS2TCPIP_H 1 + +#elif defined(__VMS) +# if defined(__VAX) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T unsigned int + +#elif defined(__OS400__) +# if defined(__ILEC400__) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 +# endif + +#elif defined(__MVS__) +# if defined(__IBMC__) || defined(__IBMCPP__) +# if defined(_ILP32) +# elif defined(_LP64) +# endif +# if defined(_LONG_LONG) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(_LP64) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 +# endif + +#elif defined(__370__) +# if defined(__IBMC__) || defined(__IBMCPP__) +# if defined(_ILP32) +# elif defined(_LP64) +# endif +# if defined(_LONG_LONG) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(_LP64) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 +# endif + +#elif defined(TPF) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__TINYC__) /* also known as tcc */ + +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + +#elif defined(__SUNPRO_C) /* Oracle Solaris Studio */ +# if !defined(__LP64) && (defined(__ILP32) || \ + defined(__i386) || defined(__sparcv8)) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__LP64) || \ + defined(__amd64) || defined(__sparcv9) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + +/* ===================================== */ +/* KEEP MSVC THE PENULTIMATE ENTRY */ +/* ===================================== */ + +#elif defined(_MSC_VER) +# if (_MSC_VER >= 900) && (_INTEGRAL_MAX_BITS >= 64) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +/* ===================================== */ +/* KEEP GENERIC GCC THE LAST ENTRY */ +/* ===================================== */ + +#elif defined(__GNUC__) +# if !defined(__LP64__) && \ + (defined(__ILP32__) || defined(__i386__) || defined(__hppa__) || \ + defined(__ppc__) || defined(__powerpc__) || defined(__arm__) || \ + defined(__sparc__) || defined(__mips__) || defined(__sh__) || \ + defined(__XTENSA__) || \ + (defined(__SIZEOF_LONG__) && __SIZEOF_LONG__ == 4) || \ + (defined(__LONG_MAX__) && __LONG_MAX__ == 2147483647L)) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__LP64__) || \ + defined(__x86_64__) || defined(__ppc64__) || defined(__sparc64__) || \ + (defined(__SIZEOF_LONG__) && __SIZEOF_LONG__ == 8) || \ + (defined(__LONG_MAX__) && __LONG_MAX__ == 9223372036854775807L) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + +#else +/* generic "safe guess" on old 32 bit style */ +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int +#endif + +#ifdef _AIX +/* AIX needs <sys/poll.h> */ +#define CURL_PULL_SYS_POLL_H +#endif + + +/* CURL_PULL_WS2TCPIP_H is defined above when inclusion of header file */ +/* ws2tcpip.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_WS2TCPIP_H +# include <winsock2.h> +# include <windows.h> +# include <ws2tcpip.h> +#endif + +/* CURL_PULL_SYS_TYPES_H is defined above when inclusion of header file */ +/* sys/types.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_SYS_TYPES_H +# include <sys/types.h> +#endif + +/* CURL_PULL_SYS_SOCKET_H is defined above when inclusion of header file */ +/* sys/socket.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_SYS_SOCKET_H +# include <sys/socket.h> +#endif + +/* CURL_PULL_SYS_POLL_H is defined above when inclusion of header file */ +/* sys/poll.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_SYS_POLL_H +# include <sys/poll.h> +#endif + +/* Data type definition of curl_socklen_t. */ +#ifdef CURL_TYPEOF_CURL_SOCKLEN_T + typedef CURL_TYPEOF_CURL_SOCKLEN_T curl_socklen_t; +#endif + +/* Data type definition of curl_off_t. */ + +#ifdef CURL_TYPEOF_CURL_OFF_T + typedef CURL_TYPEOF_CURL_OFF_T curl_off_t; +#endif + +/* + * CURL_ISOCPP and CURL_OFF_T_C definitions are done here in order to allow + * these to be visible and exported by the external libcurl interface API, + * while also making them visible to the library internals, simply including + * curl_setup.h, without actually needing to include curl.h internally. + * If some day this section would grow big enough, all this should be moved + * to its own header file. + */ + +/* + * Figure out if we can use the ## preprocessor operator, which is supported + * by ISO/ANSI C and C++. Some compilers support it without setting __STDC__ + * or __cplusplus so we need to carefully check for them too. + */ + +#if defined(__STDC__) || defined(_MSC_VER) || defined(__cplusplus) || \ + defined(__HP_aCC) || defined(__BORLANDC__) || defined(__LCC__) || \ + defined(__POCC__) || defined(__SALFORDC__) || defined(__HIGHC__) || \ + defined(__ILEC400__) + /* This compiler is believed to have an ISO compatible preprocessor */ +#define CURL_ISOCPP +#else + /* This compiler is believed NOT to have an ISO compatible preprocessor */ +#undef CURL_ISOCPP +#endif + +/* + * Macros for minimum-width signed and unsigned curl_off_t integer constants. + */ + +#if defined(__BORLANDC__) && (__BORLANDC__ == 0x0551) +# define __CURL_OFF_T_C_HLPR2(x) x +# define __CURL_OFF_T_C_HLPR1(x) __CURL_OFF_T_C_HLPR2(x) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_TU) +#else +# ifdef CURL_ISOCPP +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val ## Suffix +# else +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val/**/Suffix +# endif +# define __CURL_OFF_T_C_HLPR1(Val,Suffix) __CURL_OFF_T_C_HLPR2(Val,Suffix) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_TU) +#endif + +#endif /* __CURL_SYSTEM_H */ diff --git a/ext/curl-7.58.0/Win32/include/curl/typecheck-gcc.h b/ext/curl-7.58.0/Win32/include/curl/typecheck-gcc.h new file mode 100644 index 00000000..10c74c76 --- /dev/null +++ b/ext/curl-7.58.0/Win32/include/curl/typecheck-gcc.h @@ -0,0 +1,683 @@ +#ifndef __CURL_TYPECHECK_GCC_H +#define __CURL_TYPECHECK_GCC_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* wraps curl_easy_setopt() with typechecking */ + +/* To add a new kind of warning, add an + * if(_curl_is_sometype_option(_curl_opt)) + * if(!_curl_is_sometype(value)) + * _curl_easy_setopt_err_sometype(); + * block and define _curl_is_sometype_option, _curl_is_sometype and + * _curl_easy_setopt_err_sometype below + * + * NOTE: We use two nested 'if' statements here instead of the && operator, in + * order to work around gcc bug #32061. It affects only gcc 4.3.x/4.4.x + * when compiling with -Wlogical-op. + * + * To add an option that uses the same type as an existing option, you'll just + * need to extend the appropriate _curl_*_option macro + */ +#define curl_easy_setopt(handle, option, value) \ +__extension__ ({ \ + __typeof__(option) _curl_opt = option; \ + if(__builtin_constant_p(_curl_opt)) { \ + if(_curl_is_long_option(_curl_opt)) \ + if(!_curl_is_long(value)) \ + _curl_easy_setopt_err_long(); \ + if(_curl_is_off_t_option(_curl_opt)) \ + if(!_curl_is_off_t(value)) \ + _curl_easy_setopt_err_curl_off_t(); \ + if(_curl_is_string_option(_curl_opt)) \ + if(!_curl_is_string(value)) \ + _curl_easy_setopt_err_string(); \ + if(_curl_is_write_cb_option(_curl_opt)) \ + if(!_curl_is_write_cb(value)) \ + _curl_easy_setopt_err_write_callback(); \ + if((_curl_opt) == CURLOPT_READFUNCTION) \ + if(!_curl_is_read_cb(value)) \ + _curl_easy_setopt_err_read_cb(); \ + if((_curl_opt) == CURLOPT_IOCTLFUNCTION) \ + if(!_curl_is_ioctl_cb(value)) \ + _curl_easy_setopt_err_ioctl_cb(); \ + if((_curl_opt) == CURLOPT_SOCKOPTFUNCTION) \ + if(!_curl_is_sockopt_cb(value)) \ + _curl_easy_setopt_err_sockopt_cb(); \ + if((_curl_opt) == CURLOPT_OPENSOCKETFUNCTION) \ + if(!_curl_is_opensocket_cb(value)) \ + _curl_easy_setopt_err_opensocket_cb(); \ + if((_curl_opt) == CURLOPT_PROGRESSFUNCTION) \ + if(!_curl_is_progress_cb(value)) \ + _curl_easy_setopt_err_progress_cb(); \ + if((_curl_opt) == CURLOPT_DEBUGFUNCTION) \ + if(!_curl_is_debug_cb(value)) \ + _curl_easy_setopt_err_debug_cb(); \ + if((_curl_opt) == CURLOPT_SSL_CTX_FUNCTION) \ + if(!_curl_is_ssl_ctx_cb(value)) \ + _curl_easy_setopt_err_ssl_ctx_cb(); \ + if(_curl_is_conv_cb_option(_curl_opt)) \ + if(!_curl_is_conv_cb(value)) \ + _curl_easy_setopt_err_conv_cb(); \ + if((_curl_opt) == CURLOPT_SEEKFUNCTION) \ + if(!_curl_is_seek_cb(value)) \ + _curl_easy_setopt_err_seek_cb(); \ + if(_curl_is_cb_data_option(_curl_opt)) \ + if(!_curl_is_cb_data(value)) \ + _curl_easy_setopt_err_cb_data(); \ + if((_curl_opt) == CURLOPT_ERRORBUFFER) \ + if(!_curl_is_error_buffer(value)) \ + _curl_easy_setopt_err_error_buffer(); \ + if((_curl_opt) == CURLOPT_STDERR) \ + if(!_curl_is_FILE(value)) \ + _curl_easy_setopt_err_FILE(); \ + if(_curl_is_postfields_option(_curl_opt)) \ + if(!_curl_is_postfields(value)) \ + _curl_easy_setopt_err_postfields(); \ + if((_curl_opt) == CURLOPT_HTTPPOST) \ + if(!_curl_is_arr((value), struct curl_httppost)) \ + _curl_easy_setopt_err_curl_httpost(); \ + if((_curl_opt) == CURLOPT_MIMEPOST) \ + if(!_curl_is_ptr((value), curl_mime)) \ + _curl_easy_setopt_err_curl_mimepost(); \ + if(_curl_is_slist_option(_curl_opt)) \ + if(!_curl_is_arr((value), struct curl_slist)) \ + _curl_easy_setopt_err_curl_slist(); \ + if((_curl_opt) == CURLOPT_SHARE) \ + if(!_curl_is_ptr((value), CURLSH)) \ + _curl_easy_setopt_err_CURLSH(); \ + } \ + curl_easy_setopt(handle, _curl_opt, value); \ +}) + +/* wraps curl_easy_getinfo() with typechecking */ +/* FIXME: don't allow const pointers */ +#define curl_easy_getinfo(handle, info, arg) \ +__extension__ ({ \ + __typeof__(info) _curl_info = info; \ + if(__builtin_constant_p(_curl_info)) { \ + if(_curl_is_string_info(_curl_info)) \ + if(!_curl_is_arr((arg), char *)) \ + _curl_easy_getinfo_err_string(); \ + if(_curl_is_long_info(_curl_info)) \ + if(!_curl_is_arr((arg), long)) \ + _curl_easy_getinfo_err_long(); \ + if(_curl_is_double_info(_curl_info)) \ + if(!_curl_is_arr((arg), double)) \ + _curl_easy_getinfo_err_double(); \ + if(_curl_is_slist_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_slist *)) \ + _curl_easy_getinfo_err_curl_slist(); \ + if(_curl_is_tlssessioninfo_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_tlssessioninfo *)) \ + _curl_easy_getinfo_err_curl_tlssesssioninfo(); \ + if(_curl_is_certinfo_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_certinfo *)) \ + _curl_easy_getinfo_err_curl_certinfo(); \ + if(_curl_is_socket_info(_curl_info)) \ + if(!_curl_is_arr((arg), curl_socket_t)) \ + _curl_easy_getinfo_err_curl_socket(); \ + if(_curl_is_off_t_info(_curl_info)) \ + if(!_curl_is_arr((arg), curl_off_t)) \ + _curl_easy_getinfo_err_curl_off_t(); \ + } \ + curl_easy_getinfo(handle, _curl_info, arg); \ +}) + +/* TODO: typechecking for curl_share_setopt() and curl_multi_setopt(), + * for now just make sure that the functions are called with three + * arguments + */ +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) + + +/* the actual warnings, triggered by calling the _curl_easy_setopt_err* + * functions */ + +/* To define a new warning, use _CURL_WARNING(identifier, "message") */ +#define _CURL_WARNING(id, message) \ + static void __attribute__((__warning__(message))) \ + __attribute__((__unused__)) __attribute__((__noinline__)) \ + id(void) { __asm__(""); } + +_CURL_WARNING(_curl_easy_setopt_err_long, + "curl_easy_setopt expects a long argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_off_t, + "curl_easy_setopt expects a curl_off_t argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_string, + "curl_easy_setopt expects a " + "string ('char *' or char[]) argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_write_callback, + "curl_easy_setopt expects a curl_write_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_read_cb, + "curl_easy_setopt expects a curl_read_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ioctl_cb, + "curl_easy_setopt expects a curl_ioctl_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_sockopt_cb, + "curl_easy_setopt expects a curl_sockopt_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_opensocket_cb, + "curl_easy_setopt expects a " + "curl_opensocket_callback argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_progress_cb, + "curl_easy_setopt expects a curl_progress_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_debug_cb, + "curl_easy_setopt expects a curl_debug_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ssl_ctx_cb, + "curl_easy_setopt expects a curl_ssl_ctx_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_conv_cb, + "curl_easy_setopt expects a curl_conv_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_seek_cb, + "curl_easy_setopt expects a curl_seek_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_cb_data, + "curl_easy_setopt expects a " + "private data pointer as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_error_buffer, + "curl_easy_setopt expects a " + "char buffer of CURL_ERROR_SIZE as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_FILE, + "curl_easy_setopt expects a 'FILE *' argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_postfields, + "curl_easy_setopt expects a 'void *' or 'char *' argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_httpost, + "curl_easy_setopt expects a 'struct curl_httppost *' " + "argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_mimepost, + "curl_easy_setopt expects a 'curl_mime *' " + "argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_slist, + "curl_easy_setopt expects a 'struct curl_slist *' argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_CURLSH, + "curl_easy_setopt expects a CURLSH* argument for this option") + +_CURL_WARNING(_curl_easy_getinfo_err_string, + "curl_easy_getinfo expects a pointer to 'char *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_long, + "curl_easy_getinfo expects a pointer to long for this info") +_CURL_WARNING(_curl_easy_getinfo_err_double, + "curl_easy_getinfo expects a pointer to double for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_slist, + "curl_easy_getinfo expects a pointer to 'struct curl_slist *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_tlssesssioninfo, + "curl_easy_getinfo expects a pointer to " + "'struct curl_tlssessioninfo *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_certinfo, + "curl_easy_getinfo expects a pointer to " + "'struct curl_certinfo *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_socket, + "curl_easy_getinfo expects a pointer to curl_socket_t for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_off_t, + "curl_easy_getinfo expects a pointer to curl_off_t for this info") + +/* groups of curl_easy_setops options that take the same type of argument */ + +/* To add a new option to one of the groups, just add + * (option) == CURLOPT_SOMETHING + * to the or-expression. If the option takes a long or curl_off_t, you don't + * have to do anything + */ + +/* evaluates to true if option takes a long argument */ +#define _curl_is_long_option(option) \ + (0 < (option) && (option) < CURLOPTTYPE_OBJECTPOINT) + +#define _curl_is_off_t_option(option) \ + ((option) > CURLOPTTYPE_OFF_T) + +/* evaluates to true if option takes a char* argument */ +#define _curl_is_string_option(option) \ + ((option) == CURLOPT_ABSTRACT_UNIX_SOCKET || \ + (option) == CURLOPT_ACCEPT_ENCODING || \ + (option) == CURLOPT_CAINFO || \ + (option) == CURLOPT_CAPATH || \ + (option) == CURLOPT_COOKIE || \ + (option) == CURLOPT_COOKIEFILE || \ + (option) == CURLOPT_COOKIEJAR || \ + (option) == CURLOPT_COOKIELIST || \ + (option) == CURLOPT_CRLFILE || \ + (option) == CURLOPT_CUSTOMREQUEST || \ + (option) == CURLOPT_DEFAULT_PROTOCOL || \ + (option) == CURLOPT_DNS_INTERFACE || \ + (option) == CURLOPT_DNS_LOCAL_IP4 || \ + (option) == CURLOPT_DNS_LOCAL_IP6 || \ + (option) == CURLOPT_DNS_SERVERS || \ + (option) == CURLOPT_EGDSOCKET || \ + (option) == CURLOPT_FTPPORT || \ + (option) == CURLOPT_FTP_ACCOUNT || \ + (option) == CURLOPT_FTP_ALTERNATIVE_TO_USER || \ + (option) == CURLOPT_INTERFACE || \ + (option) == CURLOPT_ISSUERCERT || \ + (option) == CURLOPT_KEYPASSWD || \ + (option) == CURLOPT_KRBLEVEL || \ + (option) == CURLOPT_LOGIN_OPTIONS || \ + (option) == CURLOPT_MAIL_AUTH || \ + (option) == CURLOPT_MAIL_FROM || \ + (option) == CURLOPT_NETRC_FILE || \ + (option) == CURLOPT_NOPROXY || \ + (option) == CURLOPT_PASSWORD || \ + (option) == CURLOPT_PINNEDPUBLICKEY || \ + (option) == CURLOPT_PRE_PROXY || \ + (option) == CURLOPT_PROXY || \ + (option) == CURLOPT_PROXYPASSWORD || \ + (option) == CURLOPT_PROXYUSERNAME || \ + (option) == CURLOPT_PROXYUSERPWD || \ + (option) == CURLOPT_PROXY_CAINFO || \ + (option) == CURLOPT_PROXY_CAPATH || \ + (option) == CURLOPT_PROXY_CRLFILE || \ + (option) == CURLOPT_PROXY_KEYPASSWD || \ + (option) == CURLOPT_PROXY_PINNEDPUBLICKEY || \ + (option) == CURLOPT_PROXY_SERVICE_NAME || \ + (option) == CURLOPT_PROXY_SSLCERT || \ + (option) == CURLOPT_PROXY_SSLCERTTYPE || \ + (option) == CURLOPT_PROXY_SSLKEY || \ + (option) == CURLOPT_PROXY_SSLKEYTYPE || \ + (option) == CURLOPT_PROXY_SSL_CIPHER_LIST || \ + (option) == CURLOPT_PROXY_TLSAUTH_PASSWORD || \ + (option) == CURLOPT_PROXY_TLSAUTH_USERNAME || \ + (option) == CURLOPT_PROXY_TLSAUTH_TYPE || \ + (option) == CURLOPT_RANDOM_FILE || \ + (option) == CURLOPT_RANGE || \ + (option) == CURLOPT_REFERER || \ + (option) == CURLOPT_RTSP_SESSION_ID || \ + (option) == CURLOPT_RTSP_STREAM_URI || \ + (option) == CURLOPT_RTSP_TRANSPORT || \ + (option) == CURLOPT_SERVICE_NAME || \ + (option) == CURLOPT_SOCKS5_GSSAPI_SERVICE || \ + (option) == CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 || \ + (option) == CURLOPT_SSH_KNOWNHOSTS || \ + (option) == CURLOPT_SSH_PRIVATE_KEYFILE || \ + (option) == CURLOPT_SSH_PUBLIC_KEYFILE || \ + (option) == CURLOPT_SSLCERT || \ + (option) == CURLOPT_SSLCERTTYPE || \ + (option) == CURLOPT_SSLENGINE || \ + (option) == CURLOPT_SSLKEY || \ + (option) == CURLOPT_SSLKEYTYPE || \ + (option) == CURLOPT_SSL_CIPHER_LIST || \ + (option) == CURLOPT_TLSAUTH_PASSWORD || \ + (option) == CURLOPT_TLSAUTH_TYPE || \ + (option) == CURLOPT_TLSAUTH_USERNAME || \ + (option) == CURLOPT_UNIX_SOCKET_PATH || \ + (option) == CURLOPT_URL || \ + (option) == CURLOPT_USERAGENT || \ + (option) == CURLOPT_USERNAME || \ + (option) == CURLOPT_USERPWD || \ + (option) == CURLOPT_XOAUTH2_BEARER || \ + 0) + +/* evaluates to true if option takes a curl_write_callback argument */ +#define _curl_is_write_cb_option(option) \ + ((option) == CURLOPT_HEADERFUNCTION || \ + (option) == CURLOPT_WRITEFUNCTION) + +/* evaluates to true if option takes a curl_conv_callback argument */ +#define _curl_is_conv_cb_option(option) \ + ((option) == CURLOPT_CONV_TO_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_UTF8_FUNCTION) + +/* evaluates to true if option takes a data argument to pass to a callback */ +#define _curl_is_cb_data_option(option) \ + ((option) == CURLOPT_CHUNK_DATA || \ + (option) == CURLOPT_CLOSESOCKETDATA || \ + (option) == CURLOPT_DEBUGDATA || \ + (option) == CURLOPT_FNMATCH_DATA || \ + (option) == CURLOPT_HEADERDATA || \ + (option) == CURLOPT_INTERLEAVEDATA || \ + (option) == CURLOPT_IOCTLDATA || \ + (option) == CURLOPT_OPENSOCKETDATA || \ + (option) == CURLOPT_PRIVATE || \ + (option) == CURLOPT_PROGRESSDATA || \ + (option) == CURLOPT_READDATA || \ + (option) == CURLOPT_SEEKDATA || \ + (option) == CURLOPT_SOCKOPTDATA || \ + (option) == CURLOPT_SSH_KEYDATA || \ + (option) == CURLOPT_SSL_CTX_DATA || \ + (option) == CURLOPT_WRITEDATA || \ + 0) + +/* evaluates to true if option takes a POST data argument (void* or char*) */ +#define _curl_is_postfields_option(option) \ + ((option) == CURLOPT_POSTFIELDS || \ + (option) == CURLOPT_COPYPOSTFIELDS || \ + 0) + +/* evaluates to true if option takes a struct curl_slist * argument */ +#define _curl_is_slist_option(option) \ + ((option) == CURLOPT_HTTP200ALIASES || \ + (option) == CURLOPT_HTTPHEADER || \ + (option) == CURLOPT_MAIL_RCPT || \ + (option) == CURLOPT_POSTQUOTE || \ + (option) == CURLOPT_PREQUOTE || \ + (option) == CURLOPT_PROXYHEADER || \ + (option) == CURLOPT_QUOTE || \ + (option) == CURLOPT_RESOLVE || \ + (option) == CURLOPT_TELNETOPTIONS || \ + 0) + +/* groups of curl_easy_getinfo infos that take the same type of argument */ + +/* evaluates to true if info expects a pointer to char * argument */ +#define _curl_is_string_info(info) \ + (CURLINFO_STRING < (info) && (info) < CURLINFO_LONG) + +/* evaluates to true if info expects a pointer to long argument */ +#define _curl_is_long_info(info) \ + (CURLINFO_LONG < (info) && (info) < CURLINFO_DOUBLE) + +/* evaluates to true if info expects a pointer to double argument */ +#define _curl_is_double_info(info) \ + (CURLINFO_DOUBLE < (info) && (info) < CURLINFO_SLIST) + +/* true if info expects a pointer to struct curl_slist * argument */ +#define _curl_is_slist_info(info) \ + (((info) == CURLINFO_SSL_ENGINES) || ((info) == CURLINFO_COOKIELIST)) + +/* true if info expects a pointer to struct curl_tlssessioninfo * argument */ +#define _curl_is_tlssessioninfo_info(info) \ + (((info) == CURLINFO_TLS_SSL_PTR) || ((info) == CURLINFO_TLS_SESSION)) + +/* true if info expects a pointer to struct curl_certinfo * argument */ +#define _curl_is_certinfo_info(info) ((info) == CURLINFO_CERTINFO) + +/* true if info expects a pointer to struct curl_socket_t argument */ +#define _curl_is_socket_info(info) \ + (CURLINFO_SOCKET < (info) && (info) < CURLINFO_OFF_T) + +/* true if info expects a pointer to curl_off_t argument */ +#define _curl_is_off_t_info(info) \ + (CURLINFO_OFF_T < (info)) + + +/* typecheck helpers -- check whether given expression has requested type*/ + +/* For pointers, you can use the _curl_is_ptr/_curl_is_arr macros, + * otherwise define a new macro. Search for __builtin_types_compatible_p + * in the GCC manual. + * NOTE: these macros MUST NOT EVALUATE their arguments! The argument is + * the actual expression passed to the curl_easy_setopt macro. This + * means that you can only apply the sizeof and __typeof__ operators, no + * == or whatsoever. + */ + +/* XXX: should evaluate to true iff expr is a pointer */ +#define _curl_is_any_ptr(expr) \ + (sizeof(expr) == sizeof(void *)) + +/* evaluates to true if expr is NULL */ +/* XXX: must not evaluate expr, so this check is not accurate */ +#define _curl_is_NULL(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), __typeof__(NULL))) + +/* evaluates to true if expr is type*, const type* or NULL */ +#define _curl_is_ptr(expr, type) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), type *) || \ + __builtin_types_compatible_p(__typeof__(expr), const type *)) + +/* evaluates to true if expr is one of type[], type*, NULL or const type* */ +#define _curl_is_arr(expr, type) \ + (_curl_is_ptr((expr), type) || \ + __builtin_types_compatible_p(__typeof__(expr), type [])) + +/* evaluates to true if expr is a string */ +#define _curl_is_string(expr) \ + (_curl_is_arr((expr), char) || \ + _curl_is_arr((expr), signed char) || \ + _curl_is_arr((expr), unsigned char)) + +/* evaluates to true if expr is a long (no matter the signedness) + * XXX: for now, int is also accepted (and therefore short and char, which + * are promoted to int when passed to a variadic function) */ +#define _curl_is_long(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), long) || \ + __builtin_types_compatible_p(__typeof__(expr), signed long) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned long) || \ + __builtin_types_compatible_p(__typeof__(expr), int) || \ + __builtin_types_compatible_p(__typeof__(expr), signed int) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned int) || \ + __builtin_types_compatible_p(__typeof__(expr), short) || \ + __builtin_types_compatible_p(__typeof__(expr), signed short) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned short) || \ + __builtin_types_compatible_p(__typeof__(expr), char) || \ + __builtin_types_compatible_p(__typeof__(expr), signed char) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned char)) + +/* evaluates to true if expr is of type curl_off_t */ +#define _curl_is_off_t(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), curl_off_t)) + +/* evaluates to true if expr is abuffer suitable for CURLOPT_ERRORBUFFER */ +/* XXX: also check size of an char[] array? */ +#define _curl_is_error_buffer(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), char *) || \ + __builtin_types_compatible_p(__typeof__(expr), char[])) + +/* evaluates to true if expr is of type (const) void* or (const) FILE* */ +#if 0 +#define _curl_is_cb_data(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_ptr((expr), FILE)) +#else /* be less strict */ +#define _curl_is_cb_data(expr) \ + _curl_is_any_ptr(expr) +#endif + +/* evaluates to true if expr is of type FILE* */ +#define _curl_is_FILE(expr) \ + (_curl_is_NULL(expr) || \ + (__builtin_types_compatible_p(__typeof__(expr), FILE *))) + +/* evaluates to true if expr can be passed as POST data (void* or char*) */ +#define _curl_is_postfields(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_arr((expr), char)) + +/* FIXME: the whole callback checking is messy... + * The idea is to tolerate char vs. void and const vs. not const + * pointers in arguments at least + */ +/* helper: __builtin_types_compatible_p distinguishes between functions and + * function pointers, hide it */ +#define _curl_callback_compatible(func, type) \ + (__builtin_types_compatible_p(__typeof__(func), type) || \ + __builtin_types_compatible_p(__typeof__(func) *, type)) + +/* evaluates to true if expr is of type curl_read_callback or "similar" */ +#define _curl_is_read_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), __typeof__(fread) *) || \ + _curl_callback_compatible((expr), curl_read_callback) || \ + _curl_callback_compatible((expr), _curl_read_callback1) || \ + _curl_callback_compatible((expr), _curl_read_callback2) || \ + _curl_callback_compatible((expr), _curl_read_callback3) || \ + _curl_callback_compatible((expr), _curl_read_callback4) || \ + _curl_callback_compatible((expr), _curl_read_callback5) || \ + _curl_callback_compatible((expr), _curl_read_callback6)) +typedef size_t (*_curl_read_callback1)(char *, size_t, size_t, void *); +typedef size_t (*_curl_read_callback2)(char *, size_t, size_t, const void *); +typedef size_t (*_curl_read_callback3)(char *, size_t, size_t, FILE *); +typedef size_t (*_curl_read_callback4)(void *, size_t, size_t, void *); +typedef size_t (*_curl_read_callback5)(void *, size_t, size_t, const void *); +typedef size_t (*_curl_read_callback6)(void *, size_t, size_t, FILE *); + +/* evaluates to true if expr is of type curl_write_callback or "similar" */ +#define _curl_is_write_cb(expr) \ + (_curl_is_read_cb(expr) || \ + _curl_callback_compatible((expr), __typeof__(fwrite) *) || \ + _curl_callback_compatible((expr), curl_write_callback) || \ + _curl_callback_compatible((expr), _curl_write_callback1) || \ + _curl_callback_compatible((expr), _curl_write_callback2) || \ + _curl_callback_compatible((expr), _curl_write_callback3) || \ + _curl_callback_compatible((expr), _curl_write_callback4) || \ + _curl_callback_compatible((expr), _curl_write_callback5) || \ + _curl_callback_compatible((expr), _curl_write_callback6)) +typedef size_t (*_curl_write_callback1)(const char *, size_t, size_t, void *); +typedef size_t (*_curl_write_callback2)(const char *, size_t, size_t, + const void *); +typedef size_t (*_curl_write_callback3)(const char *, size_t, size_t, FILE *); +typedef size_t (*_curl_write_callback4)(const void *, size_t, size_t, void *); +typedef size_t (*_curl_write_callback5)(const void *, size_t, size_t, + const void *); +typedef size_t (*_curl_write_callback6)(const void *, size_t, size_t, FILE *); + +/* evaluates to true if expr is of type curl_ioctl_callback or "similar" */ +#define _curl_is_ioctl_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_ioctl_callback) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback1) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback2) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback3) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback4)) +typedef curlioerr (*_curl_ioctl_callback1)(CURL *, int, void *); +typedef curlioerr (*_curl_ioctl_callback2)(CURL *, int, const void *); +typedef curlioerr (*_curl_ioctl_callback3)(CURL *, curliocmd, void *); +typedef curlioerr (*_curl_ioctl_callback4)(CURL *, curliocmd, const void *); + +/* evaluates to true if expr is of type curl_sockopt_callback or "similar" */ +#define _curl_is_sockopt_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_sockopt_callback) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback1) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback2)) +typedef int (*_curl_sockopt_callback1)(void *, curl_socket_t, curlsocktype); +typedef int (*_curl_sockopt_callback2)(const void *, curl_socket_t, + curlsocktype); + +/* evaluates to true if expr is of type curl_opensocket_callback or + "similar" */ +#define _curl_is_opensocket_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_opensocket_callback) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback1) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback2) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback3) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback4)) +typedef curl_socket_t (*_curl_opensocket_callback1) + (void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (*_curl_opensocket_callback2) + (void *, curlsocktype, const struct curl_sockaddr *); +typedef curl_socket_t (*_curl_opensocket_callback3) + (const void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (*_curl_opensocket_callback4) + (const void *, curlsocktype, const struct curl_sockaddr *); + +/* evaluates to true if expr is of type curl_progress_callback or "similar" */ +#define _curl_is_progress_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_progress_callback) || \ + _curl_callback_compatible((expr), _curl_progress_callback1) || \ + _curl_callback_compatible((expr), _curl_progress_callback2)) +typedef int (*_curl_progress_callback1)(void *, + double, double, double, double); +typedef int (*_curl_progress_callback2)(const void *, + double, double, double, double); + +/* evaluates to true if expr is of type curl_debug_callback or "similar" */ +#define _curl_is_debug_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_debug_callback) || \ + _curl_callback_compatible((expr), _curl_debug_callback1) || \ + _curl_callback_compatible((expr), _curl_debug_callback2) || \ + _curl_callback_compatible((expr), _curl_debug_callback3) || \ + _curl_callback_compatible((expr), _curl_debug_callback4) || \ + _curl_callback_compatible((expr), _curl_debug_callback5) || \ + _curl_callback_compatible((expr), _curl_debug_callback6) || \ + _curl_callback_compatible((expr), _curl_debug_callback7) || \ + _curl_callback_compatible((expr), _curl_debug_callback8)) +typedef int (*_curl_debug_callback1) (CURL *, + curl_infotype, char *, size_t, void *); +typedef int (*_curl_debug_callback2) (CURL *, + curl_infotype, char *, size_t, const void *); +typedef int (*_curl_debug_callback3) (CURL *, + curl_infotype, const char *, size_t, void *); +typedef int (*_curl_debug_callback4) (CURL *, + curl_infotype, const char *, size_t, const void *); +typedef int (*_curl_debug_callback5) (CURL *, + curl_infotype, unsigned char *, size_t, void *); +typedef int (*_curl_debug_callback6) (CURL *, + curl_infotype, unsigned char *, size_t, const void *); +typedef int (*_curl_debug_callback7) (CURL *, + curl_infotype, const unsigned char *, size_t, void *); +typedef int (*_curl_debug_callback8) (CURL *, + curl_infotype, const unsigned char *, size_t, const void *); + +/* evaluates to true if expr is of type curl_ssl_ctx_callback or "similar" */ +/* this is getting even messier... */ +#define _curl_is_ssl_ctx_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_ssl_ctx_callback) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback1) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback2) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback3) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback4) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback5) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback6) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback7) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback8)) +typedef CURLcode (*_curl_ssl_ctx_callback1)(CURL *, void *, void *); +typedef CURLcode (*_curl_ssl_ctx_callback2)(CURL *, void *, const void *); +typedef CURLcode (*_curl_ssl_ctx_callback3)(CURL *, const void *, void *); +typedef CURLcode (*_curl_ssl_ctx_callback4)(CURL *, const void *, + const void *); +#ifdef HEADER_SSL_H +/* hack: if we included OpenSSL's ssl.h, we know about SSL_CTX + * this will of course break if we're included before OpenSSL headers... + */ +typedef CURLcode (*_curl_ssl_ctx_callback5)(CURL *, SSL_CTX, void *); +typedef CURLcode (*_curl_ssl_ctx_callback6)(CURL *, SSL_CTX, const void *); +typedef CURLcode (*_curl_ssl_ctx_callback7)(CURL *, const SSL_CTX, void *); +typedef CURLcode (*_curl_ssl_ctx_callback8)(CURL *, const SSL_CTX, + const void *); +#else +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback5; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback6; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback7; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback8; +#endif + +/* evaluates to true if expr is of type curl_conv_callback or "similar" */ +#define _curl_is_conv_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_conv_callback) || \ + _curl_callback_compatible((expr), _curl_conv_callback1) || \ + _curl_callback_compatible((expr), _curl_conv_callback2) || \ + _curl_callback_compatible((expr), _curl_conv_callback3) || \ + _curl_callback_compatible((expr), _curl_conv_callback4)) +typedef CURLcode (*_curl_conv_callback1)(char *, size_t length); +typedef CURLcode (*_curl_conv_callback2)(const char *, size_t length); +typedef CURLcode (*_curl_conv_callback3)(void *, size_t length); +typedef CURLcode (*_curl_conv_callback4)(const void *, size_t length); + +/* evaluates to true if expr is of type curl_seek_callback or "similar" */ +#define _curl_is_seek_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_seek_callback) || \ + _curl_callback_compatible((expr), _curl_seek_callback1) || \ + _curl_callback_compatible((expr), _curl_seek_callback2)) +typedef CURLcode (*_curl_seek_callback1)(void *, curl_off_t, int); +typedef CURLcode (*_curl_seek_callback2)(const void *, curl_off_t, int); + + +#endif /* __CURL_TYPECHECK_GCC_H */ diff --git a/ext/curl-7.58.0/Win32/lib/libcurl_a.lib b/ext/curl-7.58.0/Win32/lib/libcurl_a.lib Binary files differnew file mode 100644 index 00000000..241a0e8c --- /dev/null +++ b/ext/curl-7.58.0/Win32/lib/libcurl_a.lib diff --git a/ext/curl-7.58.0/Win32/lib/libcurl_a_debug.lib b/ext/curl-7.58.0/Win32/lib/libcurl_a_debug.lib Binary files differnew file mode 100644 index 00000000..ee9efe50 --- /dev/null +++ b/ext/curl-7.58.0/Win32/lib/libcurl_a_debug.lib diff --git a/ext/curl-7.58.0/x64/include/curl/curl.h b/ext/curl-7.58.0/x64/include/curl/curl.h new file mode 100644 index 00000000..7680acd1 --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/curl.h @@ -0,0 +1,2751 @@ +#ifndef __CURL_CURL_H +#define __CURL_CURL_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* + * If you have libcurl problems, all docs and details are found here: + * https://curl.haxx.se/libcurl/ + * + * curl-library mailing list subscription and unsubscription web interface: + * https://cool.haxx.se/mailman/listinfo/curl-library/ + */ + +#ifdef CURL_NO_OLDIES +#define CURL_STRICTER +#endif + +#include "curlver.h" /* libcurl version defines */ +#include "system.h" /* determine things run-time */ + +/* + * Define WIN32 when build target is Win32 API + */ + +#if (defined(_WIN32) || defined(__WIN32__)) && \ + !defined(WIN32) && !defined(__SYMBIAN32__) +#define WIN32 +#endif + +#include <stdio.h> +#include <limits.h> + +#if defined(__FreeBSD__) && (__FreeBSD__ >= 2) +/* Needed for __FreeBSD_version symbol definition */ +#include <osreldate.h> +#endif + +/* The include stuff here below is mainly for time_t! */ +#include <sys/types.h> +#include <time.h> + +#if defined(WIN32) && !defined(_WIN32_WCE) && !defined(__CYGWIN__) +#if !(defined(_WINSOCKAPI_) || defined(_WINSOCK_H) || \ + defined(__LWIP_OPT_H__) || defined(LWIP_HDR_OPT_H)) +/* The check above prevents the winsock2 inclusion if winsock.h already was + included, since they can't co-exist without problems */ +#include <winsock2.h> +#include <ws2tcpip.h> +#endif +#endif + +/* HP-UX systems version 9, 10 and 11 lack sys/select.h and so does oldish + libc5-based Linux systems. Only include it on systems that are known to + require it! */ +#if defined(_AIX) || defined(__NOVELL_LIBC__) || defined(__NetBSD__) || \ + defined(__minix) || defined(__SYMBIAN32__) || defined(__INTEGRITY) || \ + defined(ANDROID) || defined(__ANDROID__) || defined(__OpenBSD__) || \ + defined(__CYGWIN__) || \ + (defined(__FreeBSD_version) && (__FreeBSD_version < 800000)) +#include <sys/select.h> +#endif + +#if !defined(WIN32) && !defined(_WIN32_WCE) +#include <sys/socket.h> +#endif + +#if !defined(WIN32) && !defined(__WATCOMC__) && !defined(__VXWORKS__) +#include <sys/time.h> +#endif + +#ifdef __BEOS__ +#include <support/SupportDefs.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(BUILDING_LIBCURL) || defined(CURL_STRICTER) +typedef struct Curl_easy CURL; +typedef struct Curl_share CURLSH; +#else +typedef void CURL; +typedef void CURLSH; +#endif + +/* + * libcurl external API function linkage decorations. + */ + +#ifdef CURL_STATICLIB +# define CURL_EXTERN +#elif defined(WIN32) || defined(_WIN32) || defined(__SYMBIAN32__) +# if defined(BUILDING_LIBCURL) +# define CURL_EXTERN __declspec(dllexport) +# else +# define CURL_EXTERN __declspec(dllimport) +# endif +#elif defined(BUILDING_LIBCURL) && defined(CURL_HIDDEN_SYMBOLS) +# define CURL_EXTERN CURL_EXTERN_SYMBOL +#else +# define CURL_EXTERN +#endif + +#ifndef curl_socket_typedef +/* socket typedef */ +#if defined(WIN32) && !defined(__LWIP_OPT_H__) && !defined(LWIP_HDR_OPT_H) +typedef SOCKET curl_socket_t; +#define CURL_SOCKET_BAD INVALID_SOCKET +#else +typedef int curl_socket_t; +#define CURL_SOCKET_BAD -1 +#endif +#define curl_socket_typedef +#endif /* curl_socket_typedef */ + +/* enum for the different supported SSL backends */ +typedef enum { + CURLSSLBACKEND_NONE = 0, + CURLSSLBACKEND_OPENSSL = 1, + CURLSSLBACKEND_GNUTLS = 2, + CURLSSLBACKEND_NSS = 3, + CURLSSLBACKEND_OBSOLETE4 = 4, /* Was QSOSSL. */ + CURLSSLBACKEND_GSKIT = 5, + CURLSSLBACKEND_POLARSSL = 6, + CURLSSLBACKEND_WOLFSSL = 7, + CURLSSLBACKEND_SCHANNEL = 8, + CURLSSLBACKEND_DARWINSSL = 9, + CURLSSLBACKEND_AXTLS = 10, + CURLSSLBACKEND_MBEDTLS = 11 +} curl_sslbackend; + +/* aliases for library clones and renames */ +#define CURLSSLBACKEND_LIBRESSL CURLSSLBACKEND_OPENSSL +#define CURLSSLBACKEND_BORINGSSL CURLSSLBACKEND_OPENSSL +#define CURLSSLBACKEND_CYASSL CURLSSLBACKEND_WOLFSSL + +struct curl_httppost { + struct curl_httppost *next; /* next entry in the list */ + char *name; /* pointer to allocated name */ + long namelength; /* length of name length */ + char *contents; /* pointer to allocated data contents */ + long contentslength; /* length of contents field, see also + CURL_HTTPPOST_LARGE */ + char *buffer; /* pointer to allocated buffer contents */ + long bufferlength; /* length of buffer field */ + char *contenttype; /* Content-Type */ + struct curl_slist *contentheader; /* list of extra headers for this form */ + struct curl_httppost *more; /* if one field name has more than one + file, this link should link to following + files */ + long flags; /* as defined below */ + +/* specified content is a file name */ +#define CURL_HTTPPOST_FILENAME (1<<0) +/* specified content is a file name */ +#define CURL_HTTPPOST_READFILE (1<<1) +/* name is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRNAME (1<<2) +/* contents is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRCONTENTS (1<<3) +/* upload file from buffer */ +#define CURL_HTTPPOST_BUFFER (1<<4) +/* upload file from pointer contents */ +#define CURL_HTTPPOST_PTRBUFFER (1<<5) +/* upload file contents by using the regular read callback to get the data and + pass the given pointer as custom pointer */ +#define CURL_HTTPPOST_CALLBACK (1<<6) +/* use size in 'contentlen', added in 7.46.0 */ +#define CURL_HTTPPOST_LARGE (1<<7) + + char *showfilename; /* The file name to show. If not set, the + actual file name will be used (if this + is a file part) */ + void *userp; /* custom pointer used for + HTTPPOST_CALLBACK posts */ + curl_off_t contentlen; /* alternative length of contents + field. Used if CURL_HTTPPOST_LARGE is + set. Added in 7.46.0 */ +}; + +/* This is the CURLOPT_PROGRESSFUNCTION callback proto. It is now considered + deprecated but was the only choice up until 7.31.0 */ +typedef int (*curl_progress_callback)(void *clientp, + double dltotal, + double dlnow, + double ultotal, + double ulnow); + +/* This is the CURLOPT_XFERINFOFUNCTION callback proto. It was introduced in + 7.32.0, it avoids floating point and provides more detailed information. */ +typedef int (*curl_xferinfo_callback)(void *clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow); + +#ifndef CURL_MAX_READ_SIZE + /* The maximum receive buffer size configurable via CURLOPT_BUFFERSIZE. */ +#define CURL_MAX_READ_SIZE 524288 +#endif + +#ifndef CURL_MAX_WRITE_SIZE + /* Tests have proven that 20K is a very bad buffer size for uploads on + Windows, while 16K for some odd reason performed a lot better. + We do the ifndef check to allow this value to easier be changed at build + time for those who feel adventurous. The practical minimum is about + 400 bytes since libcurl uses a buffer of this size as a scratch area + (unrelated to network send operations). */ +#define CURL_MAX_WRITE_SIZE 16384 +#endif + +#ifndef CURL_MAX_HTTP_HEADER +/* The only reason to have a max limit for this is to avoid the risk of a bad + server feeding libcurl with a never-ending header that will cause reallocs + infinitely */ +#define CURL_MAX_HTTP_HEADER (100*1024) +#endif + +/* This is a magic return code for the write callback that, when returned, + will signal libcurl to pause receiving on the current transfer. */ +#define CURL_WRITEFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_write_callback)(char *buffer, + size_t size, + size_t nitems, + void *outstream); + + + +/* enumeration of file types */ +typedef enum { + CURLFILETYPE_FILE = 0, + CURLFILETYPE_DIRECTORY, + CURLFILETYPE_SYMLINK, + CURLFILETYPE_DEVICE_BLOCK, + CURLFILETYPE_DEVICE_CHAR, + CURLFILETYPE_NAMEDPIPE, + CURLFILETYPE_SOCKET, + CURLFILETYPE_DOOR, /* is possible only on Sun Solaris now */ + + CURLFILETYPE_UNKNOWN /* should never occur */ +} curlfiletype; + +#define CURLFINFOFLAG_KNOWN_FILENAME (1<<0) +#define CURLFINFOFLAG_KNOWN_FILETYPE (1<<1) +#define CURLFINFOFLAG_KNOWN_TIME (1<<2) +#define CURLFINFOFLAG_KNOWN_PERM (1<<3) +#define CURLFINFOFLAG_KNOWN_UID (1<<4) +#define CURLFINFOFLAG_KNOWN_GID (1<<5) +#define CURLFINFOFLAG_KNOWN_SIZE (1<<6) +#define CURLFINFOFLAG_KNOWN_HLINKCOUNT (1<<7) + +/* Content of this structure depends on information which is known and is + achievable (e.g. by FTP LIST parsing). Please see the url_easy_setopt(3) man + page for callbacks returning this structure -- some fields are mandatory, + some others are optional. The FLAG field has special meaning. */ +struct curl_fileinfo { + char *filename; + curlfiletype filetype; + time_t time; + unsigned int perm; + int uid; + int gid; + curl_off_t size; + long int hardlinks; + + struct { + /* If some of these fields is not NULL, it is a pointer to b_data. */ + char *time; + char *perm; + char *user; + char *group; + char *target; /* pointer to the target filename of a symlink */ + } strings; + + unsigned int flags; + + /* used internally */ + char *b_data; + size_t b_size; + size_t b_used; +}; + +/* return codes for CURLOPT_CHUNK_BGN_FUNCTION */ +#define CURL_CHUNK_BGN_FUNC_OK 0 +#define CURL_CHUNK_BGN_FUNC_FAIL 1 /* tell the lib to end the task */ +#define CURL_CHUNK_BGN_FUNC_SKIP 2 /* skip this chunk over */ + +/* if splitting of data transfer is enabled, this callback is called before + download of an individual chunk started. Note that parameter "remains" works + only for FTP wildcard downloading (for now), otherwise is not used */ +typedef long (*curl_chunk_bgn_callback)(const void *transfer_info, + void *ptr, + int remains); + +/* return codes for CURLOPT_CHUNK_END_FUNCTION */ +#define CURL_CHUNK_END_FUNC_OK 0 +#define CURL_CHUNK_END_FUNC_FAIL 1 /* tell the lib to end the task */ + +/* If splitting of data transfer is enabled this callback is called after + download of an individual chunk finished. + Note! After this callback was set then it have to be called FOR ALL chunks. + Even if downloading of this chunk was skipped in CHUNK_BGN_FUNC. + This is the reason why we don't need "transfer_info" parameter in this + callback and we are not interested in "remains" parameter too. */ +typedef long (*curl_chunk_end_callback)(void *ptr); + +/* return codes for FNMATCHFUNCTION */ +#define CURL_FNMATCHFUNC_MATCH 0 /* string corresponds to the pattern */ +#define CURL_FNMATCHFUNC_NOMATCH 1 /* pattern doesn't match the string */ +#define CURL_FNMATCHFUNC_FAIL 2 /* an error occurred */ + +/* callback type for wildcard downloading pattern matching. If the + string matches the pattern, return CURL_FNMATCHFUNC_MATCH value, etc. */ +typedef int (*curl_fnmatch_callback)(void *ptr, + const char *pattern, + const char *string); + +/* These are the return codes for the seek callbacks */ +#define CURL_SEEKFUNC_OK 0 +#define CURL_SEEKFUNC_FAIL 1 /* fail the entire transfer */ +#define CURL_SEEKFUNC_CANTSEEK 2 /* tell libcurl seeking can't be done, so + libcurl might try other means instead */ +typedef int (*curl_seek_callback)(void *instream, + curl_off_t offset, + int origin); /* 'whence' */ + +/* This is a return code for the read callback that, when returned, will + signal libcurl to immediately abort the current transfer. */ +#define CURL_READFUNC_ABORT 0x10000000 +/* This is a return code for the read callback that, when returned, will + signal libcurl to pause sending data on the current transfer. */ +#define CURL_READFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_read_callback)(char *buffer, + size_t size, + size_t nitems, + void *instream); + +typedef enum { + CURLSOCKTYPE_IPCXN, /* socket created for a specific IP connection */ + CURLSOCKTYPE_ACCEPT, /* socket created by accept() call */ + CURLSOCKTYPE_LAST /* never use */ +} curlsocktype; + +/* The return code from the sockopt_callback can signal information back + to libcurl: */ +#define CURL_SOCKOPT_OK 0 +#define CURL_SOCKOPT_ERROR 1 /* causes libcurl to abort and return + CURLE_ABORTED_BY_CALLBACK */ +#define CURL_SOCKOPT_ALREADY_CONNECTED 2 + +typedef int (*curl_sockopt_callback)(void *clientp, + curl_socket_t curlfd, + curlsocktype purpose); + +struct curl_sockaddr { + int family; + int socktype; + int protocol; + unsigned int addrlen; /* addrlen was a socklen_t type before 7.18.0 but it + turned really ugly and painful on the systems that + lack this type */ + struct sockaddr addr; +}; + +typedef curl_socket_t +(*curl_opensocket_callback)(void *clientp, + curlsocktype purpose, + struct curl_sockaddr *address); + +typedef int +(*curl_closesocket_callback)(void *clientp, curl_socket_t item); + +typedef enum { + CURLIOE_OK, /* I/O operation successful */ + CURLIOE_UNKNOWNCMD, /* command was unknown to callback */ + CURLIOE_FAILRESTART, /* failed to restart the read */ + CURLIOE_LAST /* never use */ +} curlioerr; + +typedef enum { + CURLIOCMD_NOP, /* no operation */ + CURLIOCMD_RESTARTREAD, /* restart the read stream from start */ + CURLIOCMD_LAST /* never use */ +} curliocmd; + +typedef curlioerr (*curl_ioctl_callback)(CURL *handle, + int cmd, + void *clientp); + +#ifndef CURL_DID_MEMORY_FUNC_TYPEDEFS +/* + * The following typedef's are signatures of malloc, free, realloc, strdup and + * calloc respectively. Function pointers of these types can be passed to the + * curl_global_init_mem() function to set user defined memory management + * callback routines. + */ +typedef void *(*curl_malloc_callback)(size_t size); +typedef void (*curl_free_callback)(void *ptr); +typedef void *(*curl_realloc_callback)(void *ptr, size_t size); +typedef char *(*curl_strdup_callback)(const char *str); +typedef void *(*curl_calloc_callback)(size_t nmemb, size_t size); + +#define CURL_DID_MEMORY_FUNC_TYPEDEFS +#endif + +/* the kind of data that is passed to information_callback*/ +typedef enum { + CURLINFO_TEXT = 0, + CURLINFO_HEADER_IN, /* 1 */ + CURLINFO_HEADER_OUT, /* 2 */ + CURLINFO_DATA_IN, /* 3 */ + CURLINFO_DATA_OUT, /* 4 */ + CURLINFO_SSL_DATA_IN, /* 5 */ + CURLINFO_SSL_DATA_OUT, /* 6 */ + CURLINFO_END +} curl_infotype; + +typedef int (*curl_debug_callback) + (CURL *handle, /* the handle/transfer this concerns */ + curl_infotype type, /* what kind of data */ + char *data, /* points to the data */ + size_t size, /* size of the data pointed to */ + void *userptr); /* whatever the user please */ + +/* All possible error codes from all sorts of curl functions. Future versions + may return other values, stay prepared. + + Always add new return codes last. Never *EVER* remove any. The return + codes must remain the same! + */ + +typedef enum { + CURLE_OK = 0, + CURLE_UNSUPPORTED_PROTOCOL, /* 1 */ + CURLE_FAILED_INIT, /* 2 */ + CURLE_URL_MALFORMAT, /* 3 */ + CURLE_NOT_BUILT_IN, /* 4 - [was obsoleted in August 2007 for + 7.17.0, reused in April 2011 for 7.21.5] */ + CURLE_COULDNT_RESOLVE_PROXY, /* 5 */ + CURLE_COULDNT_RESOLVE_HOST, /* 6 */ + CURLE_COULDNT_CONNECT, /* 7 */ + CURLE_WEIRD_SERVER_REPLY, /* 8 */ + CURLE_REMOTE_ACCESS_DENIED, /* 9 a service was denied by the server + due to lack of access - when login fails + this is not returned. */ + CURLE_FTP_ACCEPT_FAILED, /* 10 - [was obsoleted in April 2006 for + 7.15.4, reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASS_REPLY, /* 11 */ + CURLE_FTP_ACCEPT_TIMEOUT, /* 12 - timeout occurred accepting server + [was obsoleted in August 2007 for 7.17.0, + reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASV_REPLY, /* 13 */ + CURLE_FTP_WEIRD_227_FORMAT, /* 14 */ + CURLE_FTP_CANT_GET_HOST, /* 15 */ + CURLE_HTTP2, /* 16 - A problem in the http2 framing layer. + [was obsoleted in August 2007 for 7.17.0, + reused in July 2014 for 7.38.0] */ + CURLE_FTP_COULDNT_SET_TYPE, /* 17 */ + CURLE_PARTIAL_FILE, /* 18 */ + CURLE_FTP_COULDNT_RETR_FILE, /* 19 */ + CURLE_OBSOLETE20, /* 20 - NOT USED */ + CURLE_QUOTE_ERROR, /* 21 - quote command failure */ + CURLE_HTTP_RETURNED_ERROR, /* 22 */ + CURLE_WRITE_ERROR, /* 23 */ + CURLE_OBSOLETE24, /* 24 - NOT USED */ + CURLE_UPLOAD_FAILED, /* 25 - failed upload "command" */ + CURLE_READ_ERROR, /* 26 - couldn't open/read from file */ + CURLE_OUT_OF_MEMORY, /* 27 */ + /* Note: CURLE_OUT_OF_MEMORY may sometimes indicate a conversion error + instead of a memory allocation error if CURL_DOES_CONVERSIONS + is defined + */ + CURLE_OPERATION_TIMEDOUT, /* 28 - the timeout time was reached */ + CURLE_OBSOLETE29, /* 29 - NOT USED */ + CURLE_FTP_PORT_FAILED, /* 30 - FTP PORT operation failed */ + CURLE_FTP_COULDNT_USE_REST, /* 31 - the REST command failed */ + CURLE_OBSOLETE32, /* 32 - NOT USED */ + CURLE_RANGE_ERROR, /* 33 - RANGE "command" didn't work */ + CURLE_HTTP_POST_ERROR, /* 34 */ + CURLE_SSL_CONNECT_ERROR, /* 35 - wrong when connecting with SSL */ + CURLE_BAD_DOWNLOAD_RESUME, /* 36 - couldn't resume download */ + CURLE_FILE_COULDNT_READ_FILE, /* 37 */ + CURLE_LDAP_CANNOT_BIND, /* 38 */ + CURLE_LDAP_SEARCH_FAILED, /* 39 */ + CURLE_OBSOLETE40, /* 40 - NOT USED */ + CURLE_FUNCTION_NOT_FOUND, /* 41 - NOT USED starting with 7.53.0 */ + CURLE_ABORTED_BY_CALLBACK, /* 42 */ + CURLE_BAD_FUNCTION_ARGUMENT, /* 43 */ + CURLE_OBSOLETE44, /* 44 - NOT USED */ + CURLE_INTERFACE_FAILED, /* 45 - CURLOPT_INTERFACE failed */ + CURLE_OBSOLETE46, /* 46 - NOT USED */ + CURLE_TOO_MANY_REDIRECTS, /* 47 - catch endless re-direct loops */ + CURLE_UNKNOWN_OPTION, /* 48 - User specified an unknown option */ + CURLE_TELNET_OPTION_SYNTAX, /* 49 - Malformed telnet option */ + CURLE_OBSOLETE50, /* 50 - NOT USED */ + CURLE_PEER_FAILED_VERIFICATION, /* 51 - peer's certificate or fingerprint + wasn't verified fine */ + CURLE_GOT_NOTHING, /* 52 - when this is a specific error */ + CURLE_SSL_ENGINE_NOTFOUND, /* 53 - SSL crypto engine not found */ + CURLE_SSL_ENGINE_SETFAILED, /* 54 - can not set SSL crypto engine as + default */ + CURLE_SEND_ERROR, /* 55 - failed sending network data */ + CURLE_RECV_ERROR, /* 56 - failure in receiving network data */ + CURLE_OBSOLETE57, /* 57 - NOT IN USE */ + CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */ + CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */ + CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */ + CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized/bad encoding */ + CURLE_LDAP_INVALID_URL, /* 62 - Invalid LDAP URL */ + CURLE_FILESIZE_EXCEEDED, /* 63 - Maximum file size exceeded */ + CURLE_USE_SSL_FAILED, /* 64 - Requested FTP SSL level failed */ + CURLE_SEND_FAIL_REWIND, /* 65 - Sending the data requires a rewind + that failed */ + CURLE_SSL_ENGINE_INITFAILED, /* 66 - failed to initialise ENGINE */ + CURLE_LOGIN_DENIED, /* 67 - user, password or similar was not + accepted and we failed to login */ + CURLE_TFTP_NOTFOUND, /* 68 - file not found on server */ + CURLE_TFTP_PERM, /* 69 - permission problem on server */ + CURLE_REMOTE_DISK_FULL, /* 70 - out of disk space on server */ + CURLE_TFTP_ILLEGAL, /* 71 - Illegal TFTP operation */ + CURLE_TFTP_UNKNOWNID, /* 72 - Unknown transfer ID */ + CURLE_REMOTE_FILE_EXISTS, /* 73 - File already exists */ + CURLE_TFTP_NOSUCHUSER, /* 74 - No such user */ + CURLE_CONV_FAILED, /* 75 - conversion failed */ + CURLE_CONV_REQD, /* 76 - caller must register conversion + callbacks using curl_easy_setopt options + CURLOPT_CONV_FROM_NETWORK_FUNCTION, + CURLOPT_CONV_TO_NETWORK_FUNCTION, and + CURLOPT_CONV_FROM_UTF8_FUNCTION */ + CURLE_SSL_CACERT_BADFILE, /* 77 - could not load CACERT file, missing + or wrong format */ + CURLE_REMOTE_FILE_NOT_FOUND, /* 78 - remote file not found */ + CURLE_SSH, /* 79 - error from the SSH layer, somewhat + generic so the error message will be of + interest when this has happened */ + + CURLE_SSL_SHUTDOWN_FAILED, /* 80 - Failed to shut down the SSL + connection */ + CURLE_AGAIN, /* 81 - socket is not ready for send/recv, + wait till it's ready and try again (Added + in 7.18.2) */ + CURLE_SSL_CRL_BADFILE, /* 82 - could not load CRL file, missing or + wrong format (Added in 7.19.0) */ + CURLE_SSL_ISSUER_ERROR, /* 83 - Issuer check failed. (Added in + 7.19.0) */ + CURLE_FTP_PRET_FAILED, /* 84 - a PRET command failed */ + CURLE_RTSP_CSEQ_ERROR, /* 85 - mismatch of RTSP CSeq numbers */ + CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Ids */ + CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ + CURLE_CHUNK_FAILED, /* 88 - chunk callback reported error */ + CURLE_NO_CONNECTION_AVAILABLE, /* 89 - No connection available, the + session will be queued */ + CURLE_SSL_PINNEDPUBKEYNOTMATCH, /* 90 - specified pinned public key did not + match */ + CURLE_SSL_INVALIDCERTSTATUS, /* 91 - invalid certificate status */ + CURLE_HTTP2_STREAM, /* 92 - stream error in HTTP/2 framing layer + */ + CURL_LAST /* never use! */ +} CURLcode; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Previously obsolete error code re-used in 7.38.0 */ +#define CURLE_OBSOLETE16 CURLE_HTTP2 + +/* Previously obsolete error codes re-used in 7.24.0 */ +#define CURLE_OBSOLETE10 CURLE_FTP_ACCEPT_FAILED +#define CURLE_OBSOLETE12 CURLE_FTP_ACCEPT_TIMEOUT + +/* compatibility with older names */ +#define CURLOPT_ENCODING CURLOPT_ACCEPT_ENCODING +#define CURLE_FTP_WEIRD_SERVER_REPLY CURLE_WEIRD_SERVER_REPLY + +/* The following were added in 7.21.5, April 2011 */ +#define CURLE_UNKNOWN_TELNET_OPTION CURLE_UNKNOWN_OPTION + +/* The following were added in 7.17.1 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_SSL_PEER_CERTIFICATE CURLE_PEER_FAILED_VERIFICATION + +/* The following were added in 7.17.0 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_OBSOLETE CURLE_OBSOLETE50 /* no one should be using this! */ +#define CURLE_BAD_PASSWORD_ENTERED CURLE_OBSOLETE46 +#define CURLE_BAD_CALLING_ORDER CURLE_OBSOLETE44 +#define CURLE_FTP_USER_PASSWORD_INCORRECT CURLE_OBSOLETE10 +#define CURLE_FTP_CANT_RECONNECT CURLE_OBSOLETE16 +#define CURLE_FTP_COULDNT_GET_SIZE CURLE_OBSOLETE32 +#define CURLE_FTP_COULDNT_SET_ASCII CURLE_OBSOLETE29 +#define CURLE_FTP_WEIRD_USER_REPLY CURLE_OBSOLETE12 +#define CURLE_FTP_WRITE_ERROR CURLE_OBSOLETE20 +#define CURLE_LIBRARY_NOT_FOUND CURLE_OBSOLETE40 +#define CURLE_MALFORMAT_USER CURLE_OBSOLETE24 +#define CURLE_SHARE_IN_USE CURLE_OBSOLETE57 +#define CURLE_URL_MALFORMAT_USER CURLE_NOT_BUILT_IN + +#define CURLE_FTP_ACCESS_DENIED CURLE_REMOTE_ACCESS_DENIED +#define CURLE_FTP_COULDNT_SET_BINARY CURLE_FTP_COULDNT_SET_TYPE +#define CURLE_FTP_QUOTE_ERROR CURLE_QUOTE_ERROR +#define CURLE_TFTP_DISKFULL CURLE_REMOTE_DISK_FULL +#define CURLE_TFTP_EXISTS CURLE_REMOTE_FILE_EXISTS +#define CURLE_HTTP_RANGE_ERROR CURLE_RANGE_ERROR +#define CURLE_FTP_SSL_FAILED CURLE_USE_SSL_FAILED + +/* The following were added earlier */ + +#define CURLE_OPERATION_TIMEOUTED CURLE_OPERATION_TIMEDOUT + +#define CURLE_HTTP_NOT_FOUND CURLE_HTTP_RETURNED_ERROR +#define CURLE_HTTP_PORT_FAILED CURLE_INTERFACE_FAILED +#define CURLE_FTP_COULDNT_STOR_FILE CURLE_UPLOAD_FAILED + +#define CURLE_FTP_PARTIAL_FILE CURLE_PARTIAL_FILE +#define CURLE_FTP_BAD_DOWNLOAD_RESUME CURLE_BAD_DOWNLOAD_RESUME + +/* This was the error code 50 in 7.7.3 and a few earlier versions, this + is no longer used by libcurl but is instead #defined here only to not + make programs break */ +#define CURLE_ALREADY_COMPLETE 99999 + +/* Provide defines for really old option names */ +#define CURLOPT_FILE CURLOPT_WRITEDATA /* name changed in 7.9.7 */ +#define CURLOPT_INFILE CURLOPT_READDATA /* name changed in 7.9.7 */ +#define CURLOPT_WRITEHEADER CURLOPT_HEADERDATA + +/* Since long deprecated options with no code in the lib that does anything + with them. */ +#define CURLOPT_WRITEINFO CURLOPT_OBSOLETE40 +#define CURLOPT_CLOSEPOLICY CURLOPT_OBSOLETE72 + +#endif /*!CURL_NO_OLDIES*/ + +/* This prototype applies to all conversion callbacks */ +typedef CURLcode (*curl_conv_callback)(char *buffer, size_t length); + +typedef CURLcode (*curl_ssl_ctx_callback)(CURL *curl, /* easy handle */ + void *ssl_ctx, /* actually an + OpenSSL SSL_CTX */ + void *userptr); + +typedef enum { + CURLPROXY_HTTP = 0, /* added in 7.10, new in 7.19.4 default is to use + CONNECT HTTP/1.1 */ + CURLPROXY_HTTP_1_0 = 1, /* added in 7.19.4, force to use CONNECT + HTTP/1.0 */ + CURLPROXY_HTTPS = 2, /* added in 7.52.0 */ + CURLPROXY_SOCKS4 = 4, /* support added in 7.15.2, enum existed already + in 7.10 */ + CURLPROXY_SOCKS5 = 5, /* added in 7.10 */ + CURLPROXY_SOCKS4A = 6, /* added in 7.18.0 */ + CURLPROXY_SOCKS5_HOSTNAME = 7 /* Use the SOCKS5 protocol but pass along the + host name rather than the IP address. added + in 7.18.0 */ +} curl_proxytype; /* this enum was added in 7.10 */ + +/* + * Bitmasks for CURLOPT_HTTPAUTH and CURLOPT_PROXYAUTH options: + * + * CURLAUTH_NONE - No HTTP authentication + * CURLAUTH_BASIC - HTTP Basic authentication (default) + * CURLAUTH_DIGEST - HTTP Digest authentication + * CURLAUTH_NEGOTIATE - HTTP Negotiate (SPNEGO) authentication + * CURLAUTH_GSSNEGOTIATE - Alias for CURLAUTH_NEGOTIATE (deprecated) + * CURLAUTH_NTLM - HTTP NTLM authentication + * CURLAUTH_DIGEST_IE - HTTP Digest authentication with IE flavour + * CURLAUTH_NTLM_WB - HTTP NTLM authentication delegated to winbind helper + * CURLAUTH_ONLY - Use together with a single other type to force no + * authentication or just that single type + * CURLAUTH_ANY - All fine types set + * CURLAUTH_ANYSAFE - All fine types except Basic + */ + +#define CURLAUTH_NONE ((unsigned long)0) +#define CURLAUTH_BASIC (((unsigned long)1)<<0) +#define CURLAUTH_DIGEST (((unsigned long)1)<<1) +#define CURLAUTH_NEGOTIATE (((unsigned long)1)<<2) +/* Deprecated since the advent of CURLAUTH_NEGOTIATE */ +#define CURLAUTH_GSSNEGOTIATE CURLAUTH_NEGOTIATE +/* Used for CURLOPT_SOCKS5_AUTH to stay terminologically correct */ +#define CURLAUTH_GSSAPI CURLAUTH_NEGOTIATE +#define CURLAUTH_NTLM (((unsigned long)1)<<3) +#define CURLAUTH_DIGEST_IE (((unsigned long)1)<<4) +#define CURLAUTH_NTLM_WB (((unsigned long)1)<<5) +#define CURLAUTH_ONLY (((unsigned long)1)<<31) +#define CURLAUTH_ANY (~CURLAUTH_DIGEST_IE) +#define CURLAUTH_ANYSAFE (~(CURLAUTH_BASIC|CURLAUTH_DIGEST_IE)) + +#define CURLSSH_AUTH_ANY ~0 /* all types supported by the server */ +#define CURLSSH_AUTH_NONE 0 /* none allowed, silly but complete */ +#define CURLSSH_AUTH_PUBLICKEY (1<<0) /* public/private key files */ +#define CURLSSH_AUTH_PASSWORD (1<<1) /* password */ +#define CURLSSH_AUTH_HOST (1<<2) /* host key files */ +#define CURLSSH_AUTH_KEYBOARD (1<<3) /* keyboard interactive */ +#define CURLSSH_AUTH_AGENT (1<<4) /* agent (ssh-agent, pageant...) */ +#define CURLSSH_AUTH_GSSAPI (1<<5) /* gssapi (kerberos, ...) */ +#define CURLSSH_AUTH_DEFAULT CURLSSH_AUTH_ANY + +#define CURLGSSAPI_DELEGATION_NONE 0 /* no delegation (default) */ +#define CURLGSSAPI_DELEGATION_POLICY_FLAG (1<<0) /* if permitted by policy */ +#define CURLGSSAPI_DELEGATION_FLAG (1<<1) /* delegate always */ + +#define CURL_ERROR_SIZE 256 + +enum curl_khtype { + CURLKHTYPE_UNKNOWN, + CURLKHTYPE_RSA1, + CURLKHTYPE_RSA, + CURLKHTYPE_DSS, + CURLKHTYPE_ECDSA, + CURLKHTYPE_ED25519 +}; + +struct curl_khkey { + const char *key; /* points to a zero-terminated string encoded with base64 + if len is zero, otherwise to the "raw" data */ + size_t len; + enum curl_khtype keytype; +}; + +/* this is the set of return values expected from the curl_sshkeycallback + callback */ +enum curl_khstat { + CURLKHSTAT_FINE_ADD_TO_FILE, + CURLKHSTAT_FINE, + CURLKHSTAT_REJECT, /* reject the connection, return an error */ + CURLKHSTAT_DEFER, /* do not accept it, but we can't answer right now so + this causes a CURLE_DEFER error but otherwise the + connection will be left intact etc */ + CURLKHSTAT_LAST /* not for use, only a marker for last-in-list */ +}; + +/* this is the set of status codes pass in to the callback */ +enum curl_khmatch { + CURLKHMATCH_OK, /* match */ + CURLKHMATCH_MISMATCH, /* host found, key mismatch! */ + CURLKHMATCH_MISSING, /* no matching host/key found */ + CURLKHMATCH_LAST /* not for use, only a marker for last-in-list */ +}; + +typedef int + (*curl_sshkeycallback) (CURL *easy, /* easy handle */ + const struct curl_khkey *knownkey, /* known */ + const struct curl_khkey *foundkey, /* found */ + enum curl_khmatch, /* libcurl's view on the keys */ + void *clientp); /* custom pointer passed from app */ + +/* parameter for the CURLOPT_USE_SSL option */ +typedef enum { + CURLUSESSL_NONE, /* do not attempt to use SSL */ + CURLUSESSL_TRY, /* try using SSL, proceed anyway otherwise */ + CURLUSESSL_CONTROL, /* SSL for the control connection or fail */ + CURLUSESSL_ALL, /* SSL for all communication or fail */ + CURLUSESSL_LAST /* not an option, never use */ +} curl_usessl; + +/* Definition of bits for the CURLOPT_SSL_OPTIONS argument: */ + +/* - ALLOW_BEAST tells libcurl to allow the BEAST SSL vulnerability in the + name of improving interoperability with older servers. Some SSL libraries + have introduced work-arounds for this flaw but those work-arounds sometimes + make the SSL communication fail. To regain functionality with those broken + servers, a user can this way allow the vulnerability back. */ +#define CURLSSLOPT_ALLOW_BEAST (1<<0) + +/* - NO_REVOKE tells libcurl to disable certificate revocation checks for those + SSL backends where such behavior is present. */ +#define CURLSSLOPT_NO_REVOKE (1<<1) + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2009 */ + +#define CURLFTPSSL_NONE CURLUSESSL_NONE +#define CURLFTPSSL_TRY CURLUSESSL_TRY +#define CURLFTPSSL_CONTROL CURLUSESSL_CONTROL +#define CURLFTPSSL_ALL CURLUSESSL_ALL +#define CURLFTPSSL_LAST CURLUSESSL_LAST +#define curl_ftpssl curl_usessl +#endif /*!CURL_NO_OLDIES*/ + +/* parameter for the CURLOPT_FTP_SSL_CCC option */ +typedef enum { + CURLFTPSSL_CCC_NONE, /* do not send CCC */ + CURLFTPSSL_CCC_PASSIVE, /* Let the server initiate the shutdown */ + CURLFTPSSL_CCC_ACTIVE, /* Initiate the shutdown */ + CURLFTPSSL_CCC_LAST /* not an option, never use */ +} curl_ftpccc; + +/* parameter for the CURLOPT_FTPSSLAUTH option */ +typedef enum { + CURLFTPAUTH_DEFAULT, /* let libcurl decide */ + CURLFTPAUTH_SSL, /* use "AUTH SSL" */ + CURLFTPAUTH_TLS, /* use "AUTH TLS" */ + CURLFTPAUTH_LAST /* not an option, never use */ +} curl_ftpauth; + +/* parameter for the CURLOPT_FTP_CREATE_MISSING_DIRS option */ +typedef enum { + CURLFTP_CREATE_DIR_NONE, /* do NOT create missing dirs! */ + CURLFTP_CREATE_DIR, /* (FTP/SFTP) if CWD fails, try MKD and then CWD + again if MKD succeeded, for SFTP this does + similar magic */ + CURLFTP_CREATE_DIR_RETRY, /* (FTP only) if CWD fails, try MKD and then CWD + again even if MKD failed! */ + CURLFTP_CREATE_DIR_LAST /* not an option, never use */ +} curl_ftpcreatedir; + +/* parameter for the CURLOPT_FTP_FILEMETHOD option */ +typedef enum { + CURLFTPMETHOD_DEFAULT, /* let libcurl pick */ + CURLFTPMETHOD_MULTICWD, /* single CWD operation for each path part */ + CURLFTPMETHOD_NOCWD, /* no CWD at all */ + CURLFTPMETHOD_SINGLECWD, /* one CWD to full dir, then work on file */ + CURLFTPMETHOD_LAST /* not an option, never use */ +} curl_ftpmethod; + +/* bitmask defines for CURLOPT_HEADEROPT */ +#define CURLHEADER_UNIFIED 0 +#define CURLHEADER_SEPARATE (1<<0) + +/* CURLPROTO_ defines are for the CURLOPT_*PROTOCOLS options */ +#define CURLPROTO_HTTP (1<<0) +#define CURLPROTO_HTTPS (1<<1) +#define CURLPROTO_FTP (1<<2) +#define CURLPROTO_FTPS (1<<3) +#define CURLPROTO_SCP (1<<4) +#define CURLPROTO_SFTP (1<<5) +#define CURLPROTO_TELNET (1<<6) +#define CURLPROTO_LDAP (1<<7) +#define CURLPROTO_LDAPS (1<<8) +#define CURLPROTO_DICT (1<<9) +#define CURLPROTO_FILE (1<<10) +#define CURLPROTO_TFTP (1<<11) +#define CURLPROTO_IMAP (1<<12) +#define CURLPROTO_IMAPS (1<<13) +#define CURLPROTO_POP3 (1<<14) +#define CURLPROTO_POP3S (1<<15) +#define CURLPROTO_SMTP (1<<16) +#define CURLPROTO_SMTPS (1<<17) +#define CURLPROTO_RTSP (1<<18) +#define CURLPROTO_RTMP (1<<19) +#define CURLPROTO_RTMPT (1<<20) +#define CURLPROTO_RTMPE (1<<21) +#define CURLPROTO_RTMPTE (1<<22) +#define CURLPROTO_RTMPS (1<<23) +#define CURLPROTO_RTMPTS (1<<24) +#define CURLPROTO_GOPHER (1<<25) +#define CURLPROTO_SMB (1<<26) +#define CURLPROTO_SMBS (1<<27) +#define CURLPROTO_ALL (~0) /* enable everything */ + +/* long may be 32 or 64 bits, but we should never depend on anything else + but 32 */ +#define CURLOPTTYPE_LONG 0 +#define CURLOPTTYPE_OBJECTPOINT 10000 +#define CURLOPTTYPE_STRINGPOINT 10000 +#define CURLOPTTYPE_FUNCTIONPOINT 20000 +#define CURLOPTTYPE_OFF_T 30000 + +/* *STRINGPOINT is an alias for OBJECTPOINT to allow tools to extract the + string options from the header file */ + +/* name is uppercase CURLOPT_<name>, + type is one of the defined CURLOPTTYPE_<type> + number is unique identifier */ +#ifdef CINIT +#undef CINIT +#endif + +#ifdef CURL_ISOCPP +#define CINIT(na,t,nu) CURLOPT_ ## na = CURLOPTTYPE_ ## t + nu +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define STRINGPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLOPT_/**/name = type + number +#endif + +/* + * This macro-mania below setups the CURLOPT_[what] enum, to be used with + * curl_easy_setopt(). The first argument in the CINIT() macro is the [what] + * word. + */ + +typedef enum { + /* This is the FILE * or void * the regular output should be written to. */ + CINIT(WRITEDATA, OBJECTPOINT, 1), + + /* The full URL to get/put */ + CINIT(URL, STRINGPOINT, 2), + + /* Port number to connect to, if other than default. */ + CINIT(PORT, LONG, 3), + + /* Name of proxy to use. */ + CINIT(PROXY, STRINGPOINT, 4), + + /* "user:password;options" to use when fetching. */ + CINIT(USERPWD, STRINGPOINT, 5), + + /* "user:password" to use with proxy. */ + CINIT(PROXYUSERPWD, STRINGPOINT, 6), + + /* Range to get, specified as an ASCII string. */ + CINIT(RANGE, STRINGPOINT, 7), + + /* not used */ + + /* Specified file stream to upload from (use as input): */ + CINIT(READDATA, OBJECTPOINT, 9), + + /* Buffer to receive error messages in, must be at least CURL_ERROR_SIZE + * bytes big. */ + CINIT(ERRORBUFFER, OBJECTPOINT, 10), + + /* Function that will be called to store the output (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11), + + /* Function that will be called to read the input (instead of fread). The + * parameters will use fread() syntax, make sure to follow them. */ + CINIT(READFUNCTION, FUNCTIONPOINT, 12), + + /* Time-out the read operation after this amount of seconds */ + CINIT(TIMEOUT, LONG, 13), + + /* If the CURLOPT_INFILE is used, this can be used to inform libcurl about + * how large the file being sent really is. That allows better error + * checking and better verifies that the upload was successful. -1 means + * unknown size. + * + * For large file support, there is also a _LARGE version of the key + * which takes an off_t type, allowing platforms with larger off_t + * sizes to handle larger files. See below for INFILESIZE_LARGE. + */ + CINIT(INFILESIZE, LONG, 14), + + /* POST static input fields. */ + CINIT(POSTFIELDS, OBJECTPOINT, 15), + + /* Set the referrer page (needed by some CGIs) */ + CINIT(REFERER, STRINGPOINT, 16), + + /* Set the FTP PORT string (interface name, named or numerical IP address) + Use i.e '-' to use default address. */ + CINIT(FTPPORT, STRINGPOINT, 17), + + /* Set the User-Agent string (examined by some CGIs) */ + CINIT(USERAGENT, STRINGPOINT, 18), + + /* If the download receives less than "low speed limit" bytes/second + * during "low speed time" seconds, the operations is aborted. + * You could i.e if you have a pretty high speed connection, abort if + * it is less than 2000 bytes/sec during 20 seconds. + */ + + /* Set the "low speed limit" */ + CINIT(LOW_SPEED_LIMIT, LONG, 19), + + /* Set the "low speed time" */ + CINIT(LOW_SPEED_TIME, LONG, 20), + + /* Set the continuation offset. + * + * Note there is also a _LARGE version of this key which uses + * off_t types, allowing for large file offsets on platforms which + * use larger-than-32-bit off_t's. Look below for RESUME_FROM_LARGE. + */ + CINIT(RESUME_FROM, LONG, 21), + + /* Set cookie in request: */ + CINIT(COOKIE, STRINGPOINT, 22), + + /* This points to a linked list of headers, struct curl_slist kind. This + list is also used for RTSP (in spite of its name) */ + CINIT(HTTPHEADER, OBJECTPOINT, 23), + + /* This points to a linked list of post entries, struct curl_httppost */ + CINIT(HTTPPOST, OBJECTPOINT, 24), + + /* name of the file keeping your private SSL-certificate */ + CINIT(SSLCERT, STRINGPOINT, 25), + + /* password for the SSL or SSH private key */ + CINIT(KEYPASSWD, STRINGPOINT, 26), + + /* send TYPE parameter? */ + CINIT(CRLF, LONG, 27), + + /* send linked-list of QUOTE commands */ + CINIT(QUOTE, OBJECTPOINT, 28), + + /* send FILE * or void * to store headers to, if you use a callback it + is simply passed to the callback unmodified */ + CINIT(HEADERDATA, OBJECTPOINT, 29), + + /* point to a file to read the initial cookies from, also enables + "cookie awareness" */ + CINIT(COOKIEFILE, STRINGPOINT, 31), + + /* What version to specifically try to use. + See CURL_SSLVERSION defines below. */ + CINIT(SSLVERSION, LONG, 32), + + /* What kind of HTTP time condition to use, see defines */ + CINIT(TIMECONDITION, LONG, 33), + + /* Time to use with the above condition. Specified in number of seconds + since 1 Jan 1970 */ + CINIT(TIMEVALUE, LONG, 34), + + /* 35 = OBSOLETE */ + + /* Custom request, for customizing the get command like + HTTP: DELETE, TRACE and others + FTP: to use a different list command + */ + CINIT(CUSTOMREQUEST, STRINGPOINT, 36), + + /* FILE handle to use instead of stderr */ + CINIT(STDERR, OBJECTPOINT, 37), + + /* 38 is not used */ + + /* send linked-list of post-transfer QUOTE commands */ + CINIT(POSTQUOTE, OBJECTPOINT, 39), + + CINIT(OBSOLETE40, OBJECTPOINT, 40), /* OBSOLETE, do not use! */ + + CINIT(VERBOSE, LONG, 41), /* talk a lot */ + CINIT(HEADER, LONG, 42), /* throw the header out too */ + CINIT(NOPROGRESS, LONG, 43), /* shut off the progress meter */ + CINIT(NOBODY, LONG, 44), /* use HEAD to get http document */ + CINIT(FAILONERROR, LONG, 45), /* no output on http error codes >= 400 */ + CINIT(UPLOAD, LONG, 46), /* this is an upload */ + CINIT(POST, LONG, 47), /* HTTP POST method */ + CINIT(DIRLISTONLY, LONG, 48), /* bare names when listing directories */ + + CINIT(APPEND, LONG, 50), /* Append instead of overwrite on upload! */ + + /* Specify whether to read the user+password from the .netrc or the URL. + * This must be one of the CURL_NETRC_* enums below. */ + CINIT(NETRC, LONG, 51), + + CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */ + + CINIT(TRANSFERTEXT, LONG, 53), /* transfer data in text/ASCII format */ + CINIT(PUT, LONG, 54), /* HTTP PUT */ + + /* 55 = OBSOLETE */ + + /* DEPRECATED + * Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_progress_callback + * prototype defines. */ + CINIT(PROGRESSFUNCTION, FUNCTIONPOINT, 56), + + /* Data passed to the CURLOPT_PROGRESSFUNCTION and CURLOPT_XFERINFOFUNCTION + callbacks */ + CINIT(PROGRESSDATA, OBJECTPOINT, 57), +#define CURLOPT_XFERINFODATA CURLOPT_PROGRESSDATA + + /* We want the referrer field set automatically when following locations */ + CINIT(AUTOREFERER, LONG, 58), + + /* Port of the proxy, can be set in the proxy string as well with: + "[host]:[port]" */ + CINIT(PROXYPORT, LONG, 59), + + /* size of the POST input data, if strlen() is not good to use */ + CINIT(POSTFIELDSIZE, LONG, 60), + + /* tunnel non-http operations through a HTTP proxy */ + CINIT(HTTPPROXYTUNNEL, LONG, 61), + + /* Set the interface string to use as outgoing network interface */ + CINIT(INTERFACE, STRINGPOINT, 62), + + /* Set the krb4/5 security level, this also enables krb4/5 awareness. This + * is a string, 'clear', 'safe', 'confidential' or 'private'. If the string + * is set but doesn't match one of these, 'private' will be used. */ + CINIT(KRBLEVEL, STRINGPOINT, 63), + + /* Set if we should verify the peer in ssl handshake, set 1 to verify. */ + CINIT(SSL_VERIFYPEER, LONG, 64), + + /* The CApath or CAfile used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAINFO, STRINGPOINT, 65), + + /* 66 = OBSOLETE */ + /* 67 = OBSOLETE */ + + /* Maximum number of http redirects to follow */ + CINIT(MAXREDIRS, LONG, 68), + + /* Pass a long set to 1 to get the date of the requested document (if + possible)! Pass a zero to shut it off. */ + CINIT(FILETIME, LONG, 69), + + /* This points to a linked list of telnet options */ + CINIT(TELNETOPTIONS, OBJECTPOINT, 70), + + /* Max amount of cached alive connections */ + CINIT(MAXCONNECTS, LONG, 71), + + CINIT(OBSOLETE72, LONG, 72), /* OBSOLETE, do not use! */ + + /* 73 = OBSOLETE */ + + /* Set to explicitly use a new connection for the upcoming transfer. + Do not use this unless you're absolutely sure of this, as it makes the + operation slower and is less friendly for the network. */ + CINIT(FRESH_CONNECT, LONG, 74), + + /* Set to explicitly forbid the upcoming transfer's connection to be re-used + when done. Do not use this unless you're absolutely sure of this, as it + makes the operation slower and is less friendly for the network. */ + CINIT(FORBID_REUSE, LONG, 75), + + /* Set to a file name that contains random data for libcurl to use to + seed the random engine when doing SSL connects. */ + CINIT(RANDOM_FILE, STRINGPOINT, 76), + + /* Set to the Entropy Gathering Daemon socket pathname */ + CINIT(EGDSOCKET, STRINGPOINT, 77), + + /* Time-out connect operations after this amount of seconds, if connects are + OK within this time, then fine... This only aborts the connect phase. */ + CINIT(CONNECTTIMEOUT, LONG, 78), + + /* Function that will be called to store headers (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(HEADERFUNCTION, FUNCTIONPOINT, 79), + + /* Set this to force the HTTP request to get back to GET. Only really usable + if POST, PUT or a custom request have been used first. + */ + CINIT(HTTPGET, LONG, 80), + + /* Set if we should verify the Common name from the peer certificate in ssl + * handshake, set 1 to check existence, 2 to ensure that it matches the + * provided hostname. */ + CINIT(SSL_VERIFYHOST, LONG, 81), + + /* Specify which file name to write all known cookies in after completed + operation. Set file name to "-" (dash) to make it go to stdout. */ + CINIT(COOKIEJAR, STRINGPOINT, 82), + + /* Specify which SSL ciphers to use */ + CINIT(SSL_CIPHER_LIST, STRINGPOINT, 83), + + /* Specify which HTTP version to use! This must be set to one of the + CURL_HTTP_VERSION* enums set below. */ + CINIT(HTTP_VERSION, LONG, 84), + + /* Specifically switch on or off the FTP engine's use of the EPSV command. By + default, that one will always be attempted before the more traditional + PASV command. */ + CINIT(FTP_USE_EPSV, LONG, 85), + + /* type of the file keeping your SSL-certificate ("DER", "PEM", "ENG") */ + CINIT(SSLCERTTYPE, STRINGPOINT, 86), + + /* name of the file keeping your private SSL-key */ + CINIT(SSLKEY, STRINGPOINT, 87), + + /* type of the file keeping your private SSL-key ("DER", "PEM", "ENG") */ + CINIT(SSLKEYTYPE, STRINGPOINT, 88), + + /* crypto engine for the SSL-sub system */ + CINIT(SSLENGINE, STRINGPOINT, 89), + + /* set the crypto engine for the SSL-sub system as default + the param has no meaning... + */ + CINIT(SSLENGINE_DEFAULT, LONG, 90), + + /* Non-zero value means to use the global dns cache */ + CINIT(DNS_USE_GLOBAL_CACHE, LONG, 91), /* DEPRECATED, do not use! */ + + /* DNS cache timeout */ + CINIT(DNS_CACHE_TIMEOUT, LONG, 92), + + /* send linked-list of pre-transfer QUOTE commands */ + CINIT(PREQUOTE, OBJECTPOINT, 93), + + /* set the debug function */ + CINIT(DEBUGFUNCTION, FUNCTIONPOINT, 94), + + /* set the data for the debug function */ + CINIT(DEBUGDATA, OBJECTPOINT, 95), + + /* mark this as start of a cookie session */ + CINIT(COOKIESESSION, LONG, 96), + + /* The CApath directory used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAPATH, STRINGPOINT, 97), + + /* Instruct libcurl to use a smaller receive buffer */ + CINIT(BUFFERSIZE, LONG, 98), + + /* Instruct libcurl to not use any signal/alarm handlers, even when using + timeouts. This option is useful for multi-threaded applications. + See libcurl-the-guide for more background information. */ + CINIT(NOSIGNAL, LONG, 99), + + /* Provide a CURLShare for mutexing non-ts data */ + CINIT(SHARE, OBJECTPOINT, 100), + + /* indicates type of proxy. accepted values are CURLPROXY_HTTP (default), + CURLPROXY_HTTPS, CURLPROXY_SOCKS4, CURLPROXY_SOCKS4A and + CURLPROXY_SOCKS5. */ + CINIT(PROXYTYPE, LONG, 101), + + /* Set the Accept-Encoding string. Use this to tell a server you would like + the response to be compressed. Before 7.21.6, this was known as + CURLOPT_ENCODING */ + CINIT(ACCEPT_ENCODING, STRINGPOINT, 102), + + /* Set pointer to private data */ + CINIT(PRIVATE, OBJECTPOINT, 103), + + /* Set aliases for HTTP 200 in the HTTP Response header */ + CINIT(HTTP200ALIASES, OBJECTPOINT, 104), + + /* Continue to send authentication (user+password) when following locations, + even when hostname changed. This can potentially send off the name + and password to whatever host the server decides. */ + CINIT(UNRESTRICTED_AUTH, LONG, 105), + + /* Specifically switch on or off the FTP engine's use of the EPRT command ( + it also disables the LPRT attempt). By default, those ones will always be + attempted before the good old traditional PORT command. */ + CINIT(FTP_USE_EPRT, LONG, 106), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_USERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(HTTPAUTH, LONG, 107), + + /* Set the ssl context callback function, currently only for OpenSSL ssl_ctx + in second argument. The function must be matching the + curl_ssl_ctx_callback proto. */ + CINIT(SSL_CTX_FUNCTION, FUNCTIONPOINT, 108), + + /* Set the userdata for the ssl context callback function's third + argument */ + CINIT(SSL_CTX_DATA, OBJECTPOINT, 109), + + /* FTP Option that causes missing dirs to be created on the remote server. + In 7.19.4 we introduced the convenience enums for this option using the + CURLFTP_CREATE_DIR prefix. + */ + CINIT(FTP_CREATE_MISSING_DIRS, LONG, 110), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_PROXYUSERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(PROXYAUTH, LONG, 111), + + /* FTP option that changes the timeout, in seconds, associated with + getting a response. This is different from transfer timeout time and + essentially places a demand on the FTP server to acknowledge commands + in a timely manner. */ + CINIT(FTP_RESPONSE_TIMEOUT, LONG, 112), +#define CURLOPT_SERVER_RESPONSE_TIMEOUT CURLOPT_FTP_RESPONSE_TIMEOUT + + /* Set this option to one of the CURL_IPRESOLVE_* defines (see below) to + tell libcurl to resolve names to those IP versions only. This only has + affect on systems with support for more than one, i.e IPv4 _and_ IPv6. */ + CINIT(IPRESOLVE, LONG, 113), + + /* Set this option to limit the size of a file that will be downloaded from + an HTTP or FTP server. + + Note there is also _LARGE version which adds large file support for + platforms which have larger off_t sizes. See MAXFILESIZE_LARGE below. */ + CINIT(MAXFILESIZE, LONG, 114), + + /* See the comment for INFILESIZE above, but in short, specifies + * the size of the file being uploaded. -1 means unknown. + */ + CINIT(INFILESIZE_LARGE, OFF_T, 115), + + /* Sets the continuation offset. There is also a LONG version of this; + * look above for RESUME_FROM. + */ + CINIT(RESUME_FROM_LARGE, OFF_T, 116), + + /* Sets the maximum size of data that will be downloaded from + * an HTTP or FTP server. See MAXFILESIZE above for the LONG version. + */ + CINIT(MAXFILESIZE_LARGE, OFF_T, 117), + + /* Set this option to the file name of your .netrc file you want libcurl + to parse (using the CURLOPT_NETRC option). If not set, libcurl will do + a poor attempt to find the user's home directory and check for a .netrc + file in there. */ + CINIT(NETRC_FILE, STRINGPOINT, 118), + + /* Enable SSL/TLS for FTP, pick one of: + CURLUSESSL_TRY - try using SSL, proceed anyway otherwise + CURLUSESSL_CONTROL - SSL for the control connection or fail + CURLUSESSL_ALL - SSL for all communication or fail + */ + CINIT(USE_SSL, LONG, 119), + + /* The _LARGE version of the standard POSTFIELDSIZE option */ + CINIT(POSTFIELDSIZE_LARGE, OFF_T, 120), + + /* Enable/disable the TCP Nagle algorithm */ + CINIT(TCP_NODELAY, LONG, 121), + + /* 122 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 123 OBSOLETE. Gone in 7.16.0 */ + /* 124 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 125 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 126 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 127 OBSOLETE. Gone in 7.16.0 */ + /* 128 OBSOLETE. Gone in 7.16.0 */ + + /* When FTP over SSL/TLS is selected (with CURLOPT_USE_SSL), this option + can be used to change libcurl's default action which is to first try + "AUTH SSL" and then "AUTH TLS" in this order, and proceed when a OK + response has been received. + + Available parameters are: + CURLFTPAUTH_DEFAULT - let libcurl decide + CURLFTPAUTH_SSL - try "AUTH SSL" first, then TLS + CURLFTPAUTH_TLS - try "AUTH TLS" first, then SSL + */ + CINIT(FTPSSLAUTH, LONG, 129), + + CINIT(IOCTLFUNCTION, FUNCTIONPOINT, 130), + CINIT(IOCTLDATA, OBJECTPOINT, 131), + + /* 132 OBSOLETE. Gone in 7.16.0 */ + /* 133 OBSOLETE. Gone in 7.16.0 */ + + /* zero terminated string for pass on to the FTP server when asked for + "account" info */ + CINIT(FTP_ACCOUNT, STRINGPOINT, 134), + + /* feed cookie into cookie engine */ + CINIT(COOKIELIST, STRINGPOINT, 135), + + /* ignore Content-Length */ + CINIT(IGNORE_CONTENT_LENGTH, LONG, 136), + + /* Set to non-zero to skip the IP address received in a 227 PASV FTP server + response. Typically used for FTP-SSL purposes but is not restricted to + that. libcurl will then instead use the same IP address it used for the + control connection. */ + CINIT(FTP_SKIP_PASV_IP, LONG, 137), + + /* Select "file method" to use when doing FTP, see the curl_ftpmethod + above. */ + CINIT(FTP_FILEMETHOD, LONG, 138), + + /* Local port number to bind the socket to */ + CINIT(LOCALPORT, LONG, 139), + + /* Number of ports to try, including the first one set with LOCALPORT. + Thus, setting it to 1 will make no additional attempts but the first. + */ + CINIT(LOCALPORTRANGE, LONG, 140), + + /* no transfer, set up connection and let application use the socket by + extracting it with CURLINFO_LASTSOCKET */ + CINIT(CONNECT_ONLY, LONG, 141), + + /* Function that will be called to convert from the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_FROM_NETWORK_FUNCTION, FUNCTIONPOINT, 142), + + /* Function that will be called to convert to the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_TO_NETWORK_FUNCTION, FUNCTIONPOINT, 143), + + /* Function that will be called to convert from UTF8 + (instead of using the iconv calls in libcurl) + Note that this is used only for SSL certificate processing */ + CINIT(CONV_FROM_UTF8_FUNCTION, FUNCTIONPOINT, 144), + + /* if the connection proceeds too quickly then need to slow it down */ + /* limit-rate: maximum number of bytes per second to send or receive */ + CINIT(MAX_SEND_SPEED_LARGE, OFF_T, 145), + CINIT(MAX_RECV_SPEED_LARGE, OFF_T, 146), + + /* Pointer to command string to send if USER/PASS fails. */ + CINIT(FTP_ALTERNATIVE_TO_USER, STRINGPOINT, 147), + + /* callback function for setting socket options */ + CINIT(SOCKOPTFUNCTION, FUNCTIONPOINT, 148), + CINIT(SOCKOPTDATA, OBJECTPOINT, 149), + + /* set to 0 to disable session ID re-use for this transfer, default is + enabled (== 1) */ + CINIT(SSL_SESSIONID_CACHE, LONG, 150), + + /* allowed SSH authentication methods */ + CINIT(SSH_AUTH_TYPES, LONG, 151), + + /* Used by scp/sftp to do public/private key authentication */ + CINIT(SSH_PUBLIC_KEYFILE, STRINGPOINT, 152), + CINIT(SSH_PRIVATE_KEYFILE, STRINGPOINT, 153), + + /* Send CCC (Clear Command Channel) after authentication */ + CINIT(FTP_SSL_CCC, LONG, 154), + + /* Same as TIMEOUT and CONNECTTIMEOUT, but with ms resolution */ + CINIT(TIMEOUT_MS, LONG, 155), + CINIT(CONNECTTIMEOUT_MS, LONG, 156), + + /* set to zero to disable the libcurl's decoding and thus pass the raw body + data to the application even when it is encoded/compressed */ + CINIT(HTTP_TRANSFER_DECODING, LONG, 157), + CINIT(HTTP_CONTENT_DECODING, LONG, 158), + + /* Permission used when creating new files and directories on the remote + server for protocols that support it, SFTP/SCP/FILE */ + CINIT(NEW_FILE_PERMS, LONG, 159), + CINIT(NEW_DIRECTORY_PERMS, LONG, 160), + + /* Set the behaviour of POST when redirecting. Values must be set to one + of CURL_REDIR* defines below. This used to be called CURLOPT_POST301 */ + CINIT(POSTREDIR, LONG, 161), + + /* used by scp/sftp to verify the host's public key */ + CINIT(SSH_HOST_PUBLIC_KEY_MD5, STRINGPOINT, 162), + + /* Callback function for opening socket (instead of socket(2)). Optionally, + callback is able change the address or refuse to connect returning + CURL_SOCKET_BAD. The callback should have type + curl_opensocket_callback */ + CINIT(OPENSOCKETFUNCTION, FUNCTIONPOINT, 163), + CINIT(OPENSOCKETDATA, OBJECTPOINT, 164), + + /* POST volatile input fields. */ + CINIT(COPYPOSTFIELDS, OBJECTPOINT, 165), + + /* set transfer mode (;type=<a|i>) when doing FTP via an HTTP proxy */ + CINIT(PROXY_TRANSFER_MODE, LONG, 166), + + /* Callback function for seeking in the input stream */ + CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167), + CINIT(SEEKDATA, OBJECTPOINT, 168), + + /* CRL file */ + CINIT(CRLFILE, STRINGPOINT, 169), + + /* Issuer certificate */ + CINIT(ISSUERCERT, STRINGPOINT, 170), + + /* (IPv6) Address scope */ + CINIT(ADDRESS_SCOPE, LONG, 171), + + /* Collect certificate chain info and allow it to get retrievable with + CURLINFO_CERTINFO after the transfer is complete. */ + CINIT(CERTINFO, LONG, 172), + + /* "name" and "pwd" to use when fetching. */ + CINIT(USERNAME, STRINGPOINT, 173), + CINIT(PASSWORD, STRINGPOINT, 174), + + /* "name" and "pwd" to use with Proxy when fetching. */ + CINIT(PROXYUSERNAME, STRINGPOINT, 175), + CINIT(PROXYPASSWORD, STRINGPOINT, 176), + + /* Comma separated list of hostnames defining no-proxy zones. These should + match both hostnames directly, and hostnames within a domain. For + example, local.com will match local.com and www.local.com, but NOT + notlocal.com or www.notlocal.com. For compatibility with other + implementations of this, .local.com will be considered to be the same as + local.com. A single * is the only valid wildcard, and effectively + disables the use of proxy. */ + CINIT(NOPROXY, STRINGPOINT, 177), + + /* block size for TFTP transfers */ + CINIT(TFTP_BLKSIZE, LONG, 178), + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_SERVICE, STRINGPOINT, 179), /* DEPRECATED, do not use! */ + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_NEC, LONG, 180), + + /* set the bitmask for the protocols that are allowed to be used for the + transfer, which thus helps the app which takes URLs from users or other + external inputs and want to restrict what protocol(s) to deal + with. Defaults to CURLPROTO_ALL. */ + CINIT(PROTOCOLS, LONG, 181), + + /* set the bitmask for the protocols that libcurl is allowed to follow to, + as a subset of the CURLOPT_PROTOCOLS ones. That means the protocol needs + to be set in both bitmasks to be allowed to get redirected to. Defaults + to all protocols except FILE and SCP. */ + CINIT(REDIR_PROTOCOLS, LONG, 182), + + /* set the SSH knownhost file name to use */ + CINIT(SSH_KNOWNHOSTS, STRINGPOINT, 183), + + /* set the SSH host key callback, must point to a curl_sshkeycallback + function */ + CINIT(SSH_KEYFUNCTION, FUNCTIONPOINT, 184), + + /* set the SSH host key callback custom pointer */ + CINIT(SSH_KEYDATA, OBJECTPOINT, 185), + + /* set the SMTP mail originator */ + CINIT(MAIL_FROM, STRINGPOINT, 186), + + /* set the list of SMTP mail receiver(s) */ + CINIT(MAIL_RCPT, OBJECTPOINT, 187), + + /* FTP: send PRET before PASV */ + CINIT(FTP_USE_PRET, LONG, 188), + + /* RTSP request method (OPTIONS, SETUP, PLAY, etc...) */ + CINIT(RTSP_REQUEST, LONG, 189), + + /* The RTSP session identifier */ + CINIT(RTSP_SESSION_ID, STRINGPOINT, 190), + + /* The RTSP stream URI */ + CINIT(RTSP_STREAM_URI, STRINGPOINT, 191), + + /* The Transport: header to use in RTSP requests */ + CINIT(RTSP_TRANSPORT, STRINGPOINT, 192), + + /* Manually initialize the client RTSP CSeq for this handle */ + CINIT(RTSP_CLIENT_CSEQ, LONG, 193), + + /* Manually initialize the server RTSP CSeq for this handle */ + CINIT(RTSP_SERVER_CSEQ, LONG, 194), + + /* The stream to pass to INTERLEAVEFUNCTION. */ + CINIT(INTERLEAVEDATA, OBJECTPOINT, 195), + + /* Let the application define a custom write method for RTP data */ + CINIT(INTERLEAVEFUNCTION, FUNCTIONPOINT, 196), + + /* Turn on wildcard matching */ + CINIT(WILDCARDMATCH, LONG, 197), + + /* Directory matching callback called before downloading of an + individual file (chunk) started */ + CINIT(CHUNK_BGN_FUNCTION, FUNCTIONPOINT, 198), + + /* Directory matching callback called after the file (chunk) + was downloaded, or skipped */ + CINIT(CHUNK_END_FUNCTION, FUNCTIONPOINT, 199), + + /* Change match (fnmatch-like) callback for wildcard matching */ + CINIT(FNMATCH_FUNCTION, FUNCTIONPOINT, 200), + + /* Let the application define custom chunk data pointer */ + CINIT(CHUNK_DATA, OBJECTPOINT, 201), + + /* FNMATCH_FUNCTION user pointer */ + CINIT(FNMATCH_DATA, OBJECTPOINT, 202), + + /* send linked-list of name:port:address sets */ + CINIT(RESOLVE, OBJECTPOINT, 203), + + /* Set a username for authenticated TLS */ + CINIT(TLSAUTH_USERNAME, STRINGPOINT, 204), + + /* Set a password for authenticated TLS */ + CINIT(TLSAUTH_PASSWORD, STRINGPOINT, 205), + + /* Set authentication type for authenticated TLS */ + CINIT(TLSAUTH_TYPE, STRINGPOINT, 206), + + /* Set to 1 to enable the "TE:" header in HTTP requests to ask for + compressed transfer-encoded responses. Set to 0 to disable the use of TE: + in outgoing requests. The current default is 0, but it might change in a + future libcurl release. + + libcurl will ask for the compressed methods it knows of, and if that + isn't any, it will not ask for transfer-encoding at all even if this + option is set to 1. + + */ + CINIT(TRANSFER_ENCODING, LONG, 207), + + /* Callback function for closing socket (instead of close(2)). The callback + should have type curl_closesocket_callback */ + CINIT(CLOSESOCKETFUNCTION, FUNCTIONPOINT, 208), + CINIT(CLOSESOCKETDATA, OBJECTPOINT, 209), + + /* allow GSSAPI credential delegation */ + CINIT(GSSAPI_DELEGATION, LONG, 210), + + /* Set the name servers to use for DNS resolution */ + CINIT(DNS_SERVERS, STRINGPOINT, 211), + + /* Time-out accept operations (currently for FTP only) after this amount + of milliseconds. */ + CINIT(ACCEPTTIMEOUT_MS, LONG, 212), + + /* Set TCP keepalive */ + CINIT(TCP_KEEPALIVE, LONG, 213), + + /* non-universal keepalive knobs (Linux, AIX, HP-UX, more) */ + CINIT(TCP_KEEPIDLE, LONG, 214), + CINIT(TCP_KEEPINTVL, LONG, 215), + + /* Enable/disable specific SSL features with a bitmask, see CURLSSLOPT_* */ + CINIT(SSL_OPTIONS, LONG, 216), + + /* Set the SMTP auth originator */ + CINIT(MAIL_AUTH, STRINGPOINT, 217), + + /* Enable/disable SASL initial response */ + CINIT(SASL_IR, LONG, 218), + + /* Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_xferinfo_callback + * prototype defines. (Deprecates CURLOPT_PROGRESSFUNCTION) */ + CINIT(XFERINFOFUNCTION, FUNCTIONPOINT, 219), + + /* The XOAUTH2 bearer token */ + CINIT(XOAUTH2_BEARER, STRINGPOINT, 220), + + /* Set the interface string to use as outgoing network + * interface for DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_INTERFACE, STRINGPOINT, 221), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP4, STRINGPOINT, 222), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP6, STRINGPOINT, 223), + + /* Set authentication options directly */ + CINIT(LOGIN_OPTIONS, STRINGPOINT, 224), + + /* Enable/disable TLS NPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_NPN, LONG, 225), + + /* Enable/disable TLS ALPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_ALPN, LONG, 226), + + /* Time to wait for a response to a HTTP request containing an + * Expect: 100-continue header before sending the data anyway. */ + CINIT(EXPECT_100_TIMEOUT_MS, LONG, 227), + + /* This points to a linked list of headers used for proxy requests only, + struct curl_slist kind */ + CINIT(PROXYHEADER, OBJECTPOINT, 228), + + /* Pass in a bitmask of "header options" */ + CINIT(HEADEROPT, LONG, 229), + + /* The public key in DER form used to validate the peer public key + this option is used only if SSL_VERIFYPEER is true */ + CINIT(PINNEDPUBLICKEY, STRINGPOINT, 230), + + /* Path to Unix domain socket */ + CINIT(UNIX_SOCKET_PATH, STRINGPOINT, 231), + + /* Set if we should verify the certificate status. */ + CINIT(SSL_VERIFYSTATUS, LONG, 232), + + /* Set if we should enable TLS false start. */ + CINIT(SSL_FALSESTART, LONG, 233), + + /* Do not squash dot-dot sequences */ + CINIT(PATH_AS_IS, LONG, 234), + + /* Proxy Service Name */ + CINIT(PROXY_SERVICE_NAME, STRINGPOINT, 235), + + /* Service Name */ + CINIT(SERVICE_NAME, STRINGPOINT, 236), + + /* Wait/don't wait for pipe/mutex to clarify */ + CINIT(PIPEWAIT, LONG, 237), + + /* Set the protocol used when curl is given a URL without a protocol */ + CINIT(DEFAULT_PROTOCOL, STRINGPOINT, 238), + + /* Set stream weight, 1 - 256 (default is 16) */ + CINIT(STREAM_WEIGHT, LONG, 239), + + /* Set stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS, OBJECTPOINT, 240), + + /* Set E-xclusive stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS_E, OBJECTPOINT, 241), + + /* Do not send any tftp option requests to the server */ + CINIT(TFTP_NO_OPTIONS, LONG, 242), + + /* Linked-list of host:port:connect-to-host:connect-to-port, + overrides the URL's host:port (only for the network layer) */ + CINIT(CONNECT_TO, OBJECTPOINT, 243), + + /* Set TCP Fast Open */ + CINIT(TCP_FASTOPEN, LONG, 244), + + /* Continue to send data if the server responds early with an + * HTTP status code >= 300 */ + CINIT(KEEP_SENDING_ON_ERROR, LONG, 245), + + /* The CApath or CAfile used to validate the proxy certificate + this option is used only if PROXY_SSL_VERIFYPEER is true */ + CINIT(PROXY_CAINFO, STRINGPOINT, 246), + + /* The CApath directory used to validate the proxy certificate + this option is used only if PROXY_SSL_VERIFYPEER is true */ + CINIT(PROXY_CAPATH, STRINGPOINT, 247), + + /* Set if we should verify the proxy in ssl handshake, + set 1 to verify. */ + CINIT(PROXY_SSL_VERIFYPEER, LONG, 248), + + /* Set if we should verify the Common name from the proxy certificate in ssl + * handshake, set 1 to check existence, 2 to ensure that it matches + * the provided hostname. */ + CINIT(PROXY_SSL_VERIFYHOST, LONG, 249), + + /* What version to specifically try to use for proxy. + See CURL_SSLVERSION defines below. */ + CINIT(PROXY_SSLVERSION, LONG, 250), + + /* Set a username for authenticated TLS for proxy */ + CINIT(PROXY_TLSAUTH_USERNAME, STRINGPOINT, 251), + + /* Set a password for authenticated TLS for proxy */ + CINIT(PROXY_TLSAUTH_PASSWORD, STRINGPOINT, 252), + + /* Set authentication type for authenticated TLS for proxy */ + CINIT(PROXY_TLSAUTH_TYPE, STRINGPOINT, 253), + + /* name of the file keeping your private SSL-certificate for proxy */ + CINIT(PROXY_SSLCERT, STRINGPOINT, 254), + + /* type of the file keeping your SSL-certificate ("DER", "PEM", "ENG") for + proxy */ + CINIT(PROXY_SSLCERTTYPE, STRINGPOINT, 255), + + /* name of the file keeping your private SSL-key for proxy */ + CINIT(PROXY_SSLKEY, STRINGPOINT, 256), + + /* type of the file keeping your private SSL-key ("DER", "PEM", "ENG") for + proxy */ + CINIT(PROXY_SSLKEYTYPE, STRINGPOINT, 257), + + /* password for the SSL private key for proxy */ + CINIT(PROXY_KEYPASSWD, STRINGPOINT, 258), + + /* Specify which SSL ciphers to use for proxy */ + CINIT(PROXY_SSL_CIPHER_LIST, STRINGPOINT, 259), + + /* CRL file for proxy */ + CINIT(PROXY_CRLFILE, STRINGPOINT, 260), + + /* Enable/disable specific SSL features with a bitmask for proxy, see + CURLSSLOPT_* */ + CINIT(PROXY_SSL_OPTIONS, LONG, 261), + + /* Name of pre proxy to use. */ + CINIT(PRE_PROXY, STRINGPOINT, 262), + + /* The public key in DER form used to validate the proxy public key + this option is used only if PROXY_SSL_VERIFYPEER is true */ + CINIT(PROXY_PINNEDPUBLICKEY, STRINGPOINT, 263), + + /* Path to an abstract Unix domain socket */ + CINIT(ABSTRACT_UNIX_SOCKET, STRINGPOINT, 264), + + /* Suppress proxy CONNECT response headers from user callbacks */ + CINIT(SUPPRESS_CONNECT_HEADERS, LONG, 265), + + /* The request target, instead of extracted from the URL */ + CINIT(REQUEST_TARGET, STRINGPOINT, 266), + + /* bitmask of allowed auth methods for connections to SOCKS5 proxies */ + CINIT(SOCKS5_AUTH, LONG, 267), + + /* Enable/disable SSH compression */ + CINIT(SSH_COMPRESSION, LONG, 268), + + /* Post MIME data. */ + CINIT(MIMEPOST, OBJECTPOINT, 269), + + CURLOPT_LASTENTRY /* the last unused */ +} CURLoption; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2011 */ + +/* This was added in version 7.19.1 */ +#define CURLOPT_POST301 CURLOPT_POSTREDIR + +/* These are scheduled to disappear by 2009 */ + +/* The following were added in 7.17.0 */ +#define CURLOPT_SSLKEYPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_FTPAPPEND CURLOPT_APPEND +#define CURLOPT_FTPLISTONLY CURLOPT_DIRLISTONLY +#define CURLOPT_FTP_SSL CURLOPT_USE_SSL + +/* The following were added earlier */ + +#define CURLOPT_SSLCERTPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_KRB4LEVEL CURLOPT_KRBLEVEL + +#else +/* This is set if CURL_NO_OLDIES is defined at compile-time */ +#undef CURLOPT_DNS_USE_GLOBAL_CACHE /* soon obsolete */ +#endif + + + /* Below here follows defines for the CURLOPT_IPRESOLVE option. If a host + name resolves addresses using more than one IP protocol version, this + option might be handy to force libcurl to use a specific IP version. */ +#define CURL_IPRESOLVE_WHATEVER 0 /* default, resolves addresses to all IP + versions that your system allows */ +#define CURL_IPRESOLVE_V4 1 /* resolve to IPv4 addresses */ +#define CURL_IPRESOLVE_V6 2 /* resolve to IPv6 addresses */ + + /* three convenient "aliases" that follow the name scheme better */ +#define CURLOPT_RTSPHEADER CURLOPT_HTTPHEADER + + /* These enums are for use with the CURLOPT_HTTP_VERSION option. */ +enum { + CURL_HTTP_VERSION_NONE, /* setting this means we don't care, and that we'd + like the library to choose the best possible + for us! */ + CURL_HTTP_VERSION_1_0, /* please use HTTP 1.0 in the request */ + CURL_HTTP_VERSION_1_1, /* please use HTTP 1.1 in the request */ + CURL_HTTP_VERSION_2_0, /* please use HTTP 2 in the request */ + CURL_HTTP_VERSION_2TLS, /* use version 2 for HTTPS, version 1.1 for HTTP */ + CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE, /* please use HTTP 2 without HTTP/1.1 + Upgrade */ + + CURL_HTTP_VERSION_LAST /* *ILLEGAL* http version */ +}; + +/* Convenience definition simple because the name of the version is HTTP/2 and + not 2.0. The 2_0 version of the enum name was set while the version was + still planned to be 2.0 and we stick to it for compatibility. */ +#define CURL_HTTP_VERSION_2 CURL_HTTP_VERSION_2_0 + +/* + * Public API enums for RTSP requests + */ +enum { + CURL_RTSPREQ_NONE, /* first in list */ + CURL_RTSPREQ_OPTIONS, + CURL_RTSPREQ_DESCRIBE, + CURL_RTSPREQ_ANNOUNCE, + CURL_RTSPREQ_SETUP, + CURL_RTSPREQ_PLAY, + CURL_RTSPREQ_PAUSE, + CURL_RTSPREQ_TEARDOWN, + CURL_RTSPREQ_GET_PARAMETER, + CURL_RTSPREQ_SET_PARAMETER, + CURL_RTSPREQ_RECORD, + CURL_RTSPREQ_RECEIVE, + CURL_RTSPREQ_LAST /* last in list */ +}; + + /* These enums are for use with the CURLOPT_NETRC option. */ +enum CURL_NETRC_OPTION { + CURL_NETRC_IGNORED, /* The .netrc will never be read. + * This is the default. */ + CURL_NETRC_OPTIONAL, /* A user:password in the URL will be preferred + * to one in the .netrc. */ + CURL_NETRC_REQUIRED, /* A user:password in the URL will be ignored. + * Unless one is set programmatically, the .netrc + * will be queried. */ + CURL_NETRC_LAST +}; + +enum { + CURL_SSLVERSION_DEFAULT, + CURL_SSLVERSION_TLSv1, /* TLS 1.x */ + CURL_SSLVERSION_SSLv2, + CURL_SSLVERSION_SSLv3, + CURL_SSLVERSION_TLSv1_0, + CURL_SSLVERSION_TLSv1_1, + CURL_SSLVERSION_TLSv1_2, + CURL_SSLVERSION_TLSv1_3, + + CURL_SSLVERSION_LAST /* never use, keep last */ +}; + +enum { + CURL_SSLVERSION_MAX_NONE = 0, + CURL_SSLVERSION_MAX_DEFAULT = (CURL_SSLVERSION_TLSv1 << 16), + CURL_SSLVERSION_MAX_TLSv1_0 = (CURL_SSLVERSION_TLSv1_0 << 16), + CURL_SSLVERSION_MAX_TLSv1_1 = (CURL_SSLVERSION_TLSv1_1 << 16), + CURL_SSLVERSION_MAX_TLSv1_2 = (CURL_SSLVERSION_TLSv1_2 << 16), + CURL_SSLVERSION_MAX_TLSv1_3 = (CURL_SSLVERSION_TLSv1_3 << 16), + + /* never use, keep last */ + CURL_SSLVERSION_MAX_LAST = (CURL_SSLVERSION_LAST << 16) +}; + +enum CURL_TLSAUTH { + CURL_TLSAUTH_NONE, + CURL_TLSAUTH_SRP, + CURL_TLSAUTH_LAST /* never use, keep last */ +}; + +/* symbols to use with CURLOPT_POSTREDIR. + CURL_REDIR_POST_301, CURL_REDIR_POST_302 and CURL_REDIR_POST_303 + can be bitwise ORed so that CURL_REDIR_POST_301 | CURL_REDIR_POST_302 + | CURL_REDIR_POST_303 == CURL_REDIR_POST_ALL */ + +#define CURL_REDIR_GET_ALL 0 +#define CURL_REDIR_POST_301 1 +#define CURL_REDIR_POST_302 2 +#define CURL_REDIR_POST_303 4 +#define CURL_REDIR_POST_ALL \ + (CURL_REDIR_POST_301|CURL_REDIR_POST_302|CURL_REDIR_POST_303) + +typedef enum { + CURL_TIMECOND_NONE, + + CURL_TIMECOND_IFMODSINCE, + CURL_TIMECOND_IFUNMODSINCE, + CURL_TIMECOND_LASTMOD, + + CURL_TIMECOND_LAST +} curl_TimeCond; + +/* Special size_t value signaling a zero-terminated string. */ +#define CURL_ZERO_TERMINATED ((size_t) -1) + +/* curl_strequal() and curl_strnequal() are subject for removal in a future + release */ +CURL_EXTERN int curl_strequal(const char *s1, const char *s2); +CURL_EXTERN int curl_strnequal(const char *s1, const char *s2, size_t n); + +/* Mime/form handling support. */ +typedef struct curl_mime_s curl_mime; /* Mime context. */ +typedef struct curl_mimepart_s curl_mimepart; /* Mime part context. */ + +/* + * NAME curl_mime_init() + * + * DESCRIPTION + * + * Create a mime context and return its handle. The easy parameter is the + * target handle. + */ +CURL_EXTERN curl_mime *curl_mime_init(CURL *easy); + +/* + * NAME curl_mime_free() + * + * DESCRIPTION + * + * release a mime handle and its substructures. + */ +CURL_EXTERN void curl_mime_free(curl_mime *mime); + +/* + * NAME curl_mime_addpart() + * + * DESCRIPTION + * + * Append a new empty part to the given mime context and return a handle to + * the created part. + */ +CURL_EXTERN curl_mimepart *curl_mime_addpart(curl_mime *mime); + +/* + * NAME curl_mime_name() + * + * DESCRIPTION + * + * Set mime/form part name. + */ +CURL_EXTERN CURLcode curl_mime_name(curl_mimepart *part, const char *name); + +/* + * NAME curl_mime_filename() + * + * DESCRIPTION + * + * Set mime part remote file name. + */ +CURL_EXTERN CURLcode curl_mime_filename(curl_mimepart *part, + const char *filename); + +/* + * NAME curl_mime_type() + * + * DESCRIPTION + * + * Set mime part type. + */ +CURL_EXTERN CURLcode curl_mime_type(curl_mimepart *part, const char *mimetype); + +/* + * NAME curl_mime_encoder() + * + * DESCRIPTION + * + * Set mime data transfer encoder. + */ +CURL_EXTERN CURLcode curl_mime_encoder(curl_mimepart *part, + const char *encoding); + +/* + * NAME curl_mime_data() + * + * DESCRIPTION + * + * Set mime part data source from memory data, + */ +CURL_EXTERN CURLcode curl_mime_data(curl_mimepart *part, + const char *data, size_t datasize); + +/* + * NAME curl_mime_filedata() + * + * DESCRIPTION + * + * Set mime part data source from named file. + */ +CURL_EXTERN CURLcode curl_mime_filedata(curl_mimepart *part, + const char *filename); + +/* + * NAME curl_mime_data_cb() + * + * DESCRIPTION + * + * Set mime part data source from callback function. + */ +CURL_EXTERN CURLcode curl_mime_data_cb(curl_mimepart *part, + curl_off_t datasize, + curl_read_callback readfunc, + curl_seek_callback seekfunc, + curl_free_callback freefunc, + void *arg); + +/* + * NAME curl_mime_subparts() + * + * DESCRIPTION + * + * Set mime part data source from subparts. + */ +CURL_EXTERN CURLcode curl_mime_subparts(curl_mimepart *part, + curl_mime *subparts); +/* + * NAME curl_mime_headers() + * + * DESCRIPTION + * + * Set mime part headers. + */ +CURL_EXTERN CURLcode curl_mime_headers(curl_mimepart *part, + struct curl_slist *headers, + int take_ownership); + +/* Old form API. */ +/* name is uppercase CURLFORM_<name> */ +#ifdef CFINIT +#undef CFINIT +#endif + +#ifdef CURL_ISOCPP +#define CFINIT(name) CURLFORM_ ## name +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define CFINIT(name) CURLFORM_/**/name +#endif + +typedef enum { + CFINIT(NOTHING), /********* the first one is unused ************/ + + /* */ + CFINIT(COPYNAME), + CFINIT(PTRNAME), + CFINIT(NAMELENGTH), + CFINIT(COPYCONTENTS), + CFINIT(PTRCONTENTS), + CFINIT(CONTENTSLENGTH), + CFINIT(FILECONTENT), + CFINIT(ARRAY), + CFINIT(OBSOLETE), + CFINIT(FILE), + + CFINIT(BUFFER), + CFINIT(BUFFERPTR), + CFINIT(BUFFERLENGTH), + + CFINIT(CONTENTTYPE), + CFINIT(CONTENTHEADER), + CFINIT(FILENAME), + CFINIT(END), + CFINIT(OBSOLETE2), + + CFINIT(STREAM), + CFINIT(CONTENTLEN), /* added in 7.46.0, provide a curl_off_t length */ + + CURLFORM_LASTENTRY /* the last unused */ +} CURLformoption; + +#undef CFINIT /* done */ + +/* structure to be used as parameter for CURLFORM_ARRAY */ +struct curl_forms { + CURLformoption option; + const char *value; +}; + +/* use this for multipart formpost building */ +/* Returns code for curl_formadd() + * + * Returns: + * CURL_FORMADD_OK on success + * CURL_FORMADD_MEMORY if the FormInfo allocation fails + * CURL_FORMADD_OPTION_TWICE if one option is given twice for one Form + * CURL_FORMADD_NULL if a null pointer was given for a char + * CURL_FORMADD_MEMORY if the allocation of a FormInfo struct failed + * CURL_FORMADD_UNKNOWN_OPTION if an unknown option was used + * CURL_FORMADD_INCOMPLETE if the some FormInfo is not complete (or error) + * CURL_FORMADD_MEMORY if a curl_httppost struct cannot be allocated + * CURL_FORMADD_MEMORY if some allocation for string copying failed. + * CURL_FORMADD_ILLEGAL_ARRAY if an illegal option is used in an array + * + ***************************************************************************/ +typedef enum { + CURL_FORMADD_OK, /* first, no error */ + + CURL_FORMADD_MEMORY, + CURL_FORMADD_OPTION_TWICE, + CURL_FORMADD_NULL, + CURL_FORMADD_UNKNOWN_OPTION, + CURL_FORMADD_INCOMPLETE, + CURL_FORMADD_ILLEGAL_ARRAY, + CURL_FORMADD_DISABLED, /* libcurl was built with this disabled */ + + CURL_FORMADD_LAST /* last */ +} CURLFORMcode; + +/* + * NAME curl_formadd() + * + * DESCRIPTION + * + * Pretty advanced function for building multi-part formposts. Each invoke + * adds one part that together construct a full post. Then use + * CURLOPT_HTTPPOST to send it off to libcurl. + */ +CURL_EXTERN CURLFORMcode curl_formadd(struct curl_httppost **httppost, + struct curl_httppost **last_post, + ...); + +/* + * callback function for curl_formget() + * The void *arg pointer will be the one passed as second argument to + * curl_formget(). + * The character buffer passed to it must not be freed. + * Should return the buffer length passed to it as the argument "len" on + * success. + */ +typedef size_t (*curl_formget_callback)(void *arg, const char *buf, + size_t len); + +/* + * NAME curl_formget() + * + * DESCRIPTION + * + * Serialize a curl_httppost struct built with curl_formadd(). + * Accepts a void pointer as second argument which will be passed to + * the curl_formget_callback function. + * Returns 0 on success. + */ +CURL_EXTERN int curl_formget(struct curl_httppost *form, void *arg, + curl_formget_callback append); +/* + * NAME curl_formfree() + * + * DESCRIPTION + * + * Free a multipart formpost previously built with curl_formadd(). + */ +CURL_EXTERN void curl_formfree(struct curl_httppost *form); + +/* + * NAME curl_getenv() + * + * DESCRIPTION + * + * Returns a malloc()'ed string that MUST be curl_free()ed after usage is + * complete. DEPRECATED - see lib/README.curlx + */ +CURL_EXTERN char *curl_getenv(const char *variable); + +/* + * NAME curl_version() + * + * DESCRIPTION + * + * Returns a static ascii string of the libcurl version. + */ +CURL_EXTERN char *curl_version(void); + +/* + * NAME curl_easy_escape() + * + * DESCRIPTION + * + * Escapes URL strings (converts all letters consider illegal in URLs to their + * %XX versions). This function returns a new allocated string or NULL if an + * error occurred. + */ +CURL_EXTERN char *curl_easy_escape(CURL *handle, + const char *string, + int length); + +/* the previous version: */ +CURL_EXTERN char *curl_escape(const char *string, + int length); + + +/* + * NAME curl_easy_unescape() + * + * DESCRIPTION + * + * Unescapes URL encoding in strings (converts all %XX codes to their 8bit + * versions). This function returns a new allocated string or NULL if an error + * occurred. + * Conversion Note: On non-ASCII platforms the ASCII %XX codes are + * converted into the host encoding. + */ +CURL_EXTERN char *curl_easy_unescape(CURL *handle, + const char *string, + int length, + int *outlength); + +/* the previous version */ +CURL_EXTERN char *curl_unescape(const char *string, + int length); + +/* + * NAME curl_free() + * + * DESCRIPTION + * + * Provided for de-allocation in the same translation unit that did the + * allocation. Added in libcurl 7.10 + */ +CURL_EXTERN void curl_free(void *p); + +/* + * NAME curl_global_init() + * + * DESCRIPTION + * + * curl_global_init() should be invoked exactly once for each application that + * uses libcurl and before any call of other libcurl functions. + * + * This function is not thread-safe! + */ +CURL_EXTERN CURLcode curl_global_init(long flags); + +/* + * NAME curl_global_init_mem() + * + * DESCRIPTION + * + * curl_global_init() or curl_global_init_mem() should be invoked exactly once + * for each application that uses libcurl. This function can be used to + * initialize libcurl and set user defined memory management callback + * functions. Users can implement memory management routines to check for + * memory leaks, check for mis-use of the curl library etc. User registered + * callback routines with be invoked by this library instead of the system + * memory management routines like malloc, free etc. + */ +CURL_EXTERN CURLcode curl_global_init_mem(long flags, + curl_malloc_callback m, + curl_free_callback f, + curl_realloc_callback r, + curl_strdup_callback s, + curl_calloc_callback c); + +/* + * NAME curl_global_cleanup() + * + * DESCRIPTION + * + * curl_global_cleanup() should be invoked exactly once for each application + * that uses libcurl + */ +CURL_EXTERN void curl_global_cleanup(void); + +/* linked-list structure for the CURLOPT_QUOTE option (and other) */ +struct curl_slist { + char *data; + struct curl_slist *next; +}; + +/* + * NAME curl_global_sslset() + * + * DESCRIPTION + * + * When built with multiple SSL backends, curl_global_sslset() allows to + * choose one. This function can only be called once, and it must be called + * *before* curl_global_init(). + * + * The backend can be identified by the id (e.g. CURLSSLBACKEND_OPENSSL). The + * backend can also be specified via the name parameter (passing -1 as id). + * If both id and name are specified, the name will be ignored. If neither id + * nor name are specified, the function will fail with + * CURLSSLSET_UNKNOWN_BACKEND and set the "avail" pointer to the + * NULL-terminated list of available backends. + * + * Upon success, the function returns CURLSSLSET_OK. + * + * If the specified SSL backend is not available, the function returns + * CURLSSLSET_UNKNOWN_BACKEND and sets the "avail" pointer to a NULL-terminated + * list of available SSL backends. + * + * The SSL backend can be set only once. If it has already been set, a + * subsequent attempt to change it will result in a CURLSSLSET_TOO_LATE. + */ + +typedef struct { + curl_sslbackend id; + const char *name; +} curl_ssl_backend; + +typedef enum { + CURLSSLSET_OK = 0, + CURLSSLSET_UNKNOWN_BACKEND, + CURLSSLSET_TOO_LATE, + CURLSSLSET_NO_BACKENDS /* libcurl was built without any SSL support */ +} CURLsslset; + +CURL_EXTERN CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, + const curl_ssl_backend ***avail); + +/* + * NAME curl_slist_append() + * + * DESCRIPTION + * + * Appends a string to a linked list. If no list exists, it will be created + * first. Returns the new list, after appending. + */ +CURL_EXTERN struct curl_slist *curl_slist_append(struct curl_slist *, + const char *); + +/* + * NAME curl_slist_free_all() + * + * DESCRIPTION + * + * free a previously built curl_slist. + */ +CURL_EXTERN void curl_slist_free_all(struct curl_slist *); + +/* + * NAME curl_getdate() + * + * DESCRIPTION + * + * Returns the time, in seconds since 1 Jan 1970 of the time string given in + * the first argument. The time argument in the second parameter is unused + * and should be set to NULL. + */ +CURL_EXTERN time_t curl_getdate(const char *p, const time_t *unused); + +/* info about the certificate chain, only for OpenSSL builds. Asked + for with CURLOPT_CERTINFO / CURLINFO_CERTINFO */ +struct curl_certinfo { + int num_of_certs; /* number of certificates with information */ + struct curl_slist **certinfo; /* for each index in this array, there's a + linked list with textual information in the + format "name: value" */ +}; + +/* Information about the SSL library used and the respective internal SSL + handle, which can be used to obtain further information regarding the + connection. Asked for with CURLINFO_TLS_SSL_PTR or CURLINFO_TLS_SESSION. */ +struct curl_tlssessioninfo { + curl_sslbackend backend; + void *internals; +}; + +#define CURLINFO_STRING 0x100000 +#define CURLINFO_LONG 0x200000 +#define CURLINFO_DOUBLE 0x300000 +#define CURLINFO_SLIST 0x400000 +#define CURLINFO_PTR 0x400000 /* same as SLIST */ +#define CURLINFO_SOCKET 0x500000 +#define CURLINFO_OFF_T 0x600000 +#define CURLINFO_MASK 0x0fffff +#define CURLINFO_TYPEMASK 0xf00000 + +typedef enum { + CURLINFO_NONE, /* first, never use this */ + CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1, + CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2, + CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3, + CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4, + CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5, + CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6, + CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7, + CURLINFO_SIZE_UPLOAD_T = CURLINFO_OFF_T + 7, + CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8, + CURLINFO_SIZE_DOWNLOAD_T = CURLINFO_OFF_T + 8, + CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9, + CURLINFO_SPEED_DOWNLOAD_T = CURLINFO_OFF_T + 9, + CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10, + CURLINFO_SPEED_UPLOAD_T = CURLINFO_OFF_T + 10, + CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11, + CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12, + CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13, + CURLINFO_FILETIME = CURLINFO_LONG + 14, + CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15, + CURLINFO_CONTENT_LENGTH_DOWNLOAD_T = CURLINFO_OFF_T + 15, + CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16, + CURLINFO_CONTENT_LENGTH_UPLOAD_T = CURLINFO_OFF_T + 16, + CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17, + CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18, + CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19, + CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20, + CURLINFO_PRIVATE = CURLINFO_STRING + 21, + CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22, + CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23, + CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24, + CURLINFO_OS_ERRNO = CURLINFO_LONG + 25, + CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26, + CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27, + CURLINFO_COOKIELIST = CURLINFO_SLIST + 28, + CURLINFO_LASTSOCKET = CURLINFO_LONG + 29, + CURLINFO_FTP_ENTRY_PATH = CURLINFO_STRING + 30, + CURLINFO_REDIRECT_URL = CURLINFO_STRING + 31, + CURLINFO_PRIMARY_IP = CURLINFO_STRING + 32, + CURLINFO_APPCONNECT_TIME = CURLINFO_DOUBLE + 33, + CURLINFO_CERTINFO = CURLINFO_PTR + 34, + CURLINFO_CONDITION_UNMET = CURLINFO_LONG + 35, + CURLINFO_RTSP_SESSION_ID = CURLINFO_STRING + 36, + CURLINFO_RTSP_CLIENT_CSEQ = CURLINFO_LONG + 37, + CURLINFO_RTSP_SERVER_CSEQ = CURLINFO_LONG + 38, + CURLINFO_RTSP_CSEQ_RECV = CURLINFO_LONG + 39, + CURLINFO_PRIMARY_PORT = CURLINFO_LONG + 40, + CURLINFO_LOCAL_IP = CURLINFO_STRING + 41, + CURLINFO_LOCAL_PORT = CURLINFO_LONG + 42, + CURLINFO_TLS_SESSION = CURLINFO_PTR + 43, + CURLINFO_ACTIVESOCKET = CURLINFO_SOCKET + 44, + CURLINFO_TLS_SSL_PTR = CURLINFO_PTR + 45, + CURLINFO_HTTP_VERSION = CURLINFO_LONG + 46, + CURLINFO_PROXY_SSL_VERIFYRESULT = CURLINFO_LONG + 47, + CURLINFO_PROTOCOL = CURLINFO_LONG + 48, + CURLINFO_SCHEME = CURLINFO_STRING + 49, + /* Fill in new entries below here! */ + + CURLINFO_LASTONE = 49 +} CURLINFO; + +/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as + CURLINFO_HTTP_CODE */ +#define CURLINFO_HTTP_CODE CURLINFO_RESPONSE_CODE + +typedef enum { + CURLCLOSEPOLICY_NONE, /* first, never use this */ + + CURLCLOSEPOLICY_OLDEST, + CURLCLOSEPOLICY_LEAST_RECENTLY_USED, + CURLCLOSEPOLICY_LEAST_TRAFFIC, + CURLCLOSEPOLICY_SLOWEST, + CURLCLOSEPOLICY_CALLBACK, + + CURLCLOSEPOLICY_LAST /* last, never use this */ +} curl_closepolicy; + +#define CURL_GLOBAL_SSL (1<<0) /* no purpose since since 7.57.0 */ +#define CURL_GLOBAL_WIN32 (1<<1) +#define CURL_GLOBAL_ALL (CURL_GLOBAL_SSL|CURL_GLOBAL_WIN32) +#define CURL_GLOBAL_NOTHING 0 +#define CURL_GLOBAL_DEFAULT CURL_GLOBAL_ALL +#define CURL_GLOBAL_ACK_EINTR (1<<2) + + +/***************************************************************************** + * Setup defines, protos etc for the sharing stuff. + */ + +/* Different data locks for a single share */ +typedef enum { + CURL_LOCK_DATA_NONE = 0, + /* CURL_LOCK_DATA_SHARE is used internally to say that + * the locking is just made to change the internal state of the share + * itself. + */ + CURL_LOCK_DATA_SHARE, + CURL_LOCK_DATA_COOKIE, + CURL_LOCK_DATA_DNS, + CURL_LOCK_DATA_SSL_SESSION, + CURL_LOCK_DATA_CONNECT, + CURL_LOCK_DATA_LAST +} curl_lock_data; + +/* Different lock access types */ +typedef enum { + CURL_LOCK_ACCESS_NONE = 0, /* unspecified action */ + CURL_LOCK_ACCESS_SHARED = 1, /* for read perhaps */ + CURL_LOCK_ACCESS_SINGLE = 2, /* for write perhaps */ + CURL_LOCK_ACCESS_LAST /* never use */ +} curl_lock_access; + +typedef void (*curl_lock_function)(CURL *handle, + curl_lock_data data, + curl_lock_access locktype, + void *userptr); +typedef void (*curl_unlock_function)(CURL *handle, + curl_lock_data data, + void *userptr); + + +typedef enum { + CURLSHE_OK, /* all is fine */ + CURLSHE_BAD_OPTION, /* 1 */ + CURLSHE_IN_USE, /* 2 */ + CURLSHE_INVALID, /* 3 */ + CURLSHE_NOMEM, /* 4 out of memory */ + CURLSHE_NOT_BUILT_IN, /* 5 feature not present in lib */ + CURLSHE_LAST /* never use */ +} CURLSHcode; + +typedef enum { + CURLSHOPT_NONE, /* don't use */ + CURLSHOPT_SHARE, /* specify a data type to share */ + CURLSHOPT_UNSHARE, /* specify which data type to stop sharing */ + CURLSHOPT_LOCKFUNC, /* pass in a 'curl_lock_function' pointer */ + CURLSHOPT_UNLOCKFUNC, /* pass in a 'curl_unlock_function' pointer */ + CURLSHOPT_USERDATA, /* pass in a user data pointer used in the lock/unlock + callback functions */ + CURLSHOPT_LAST /* never use */ +} CURLSHoption; + +CURL_EXTERN CURLSH *curl_share_init(void); +CURL_EXTERN CURLSHcode curl_share_setopt(CURLSH *, CURLSHoption option, ...); +CURL_EXTERN CURLSHcode curl_share_cleanup(CURLSH *); + +/**************************************************************************** + * Structures for querying information about the curl library at runtime. + */ + +typedef enum { + CURLVERSION_FIRST, + CURLVERSION_SECOND, + CURLVERSION_THIRD, + CURLVERSION_FOURTH, + CURLVERSION_FIFTH, + CURLVERSION_LAST /* never actually use this */ +} CURLversion; + +/* The 'CURLVERSION_NOW' is the symbolic name meant to be used by + basically all programs ever that want to get version information. It is + meant to be a built-in version number for what kind of struct the caller + expects. If the struct ever changes, we redefine the NOW to another enum + from above. */ +#define CURLVERSION_NOW CURLVERSION_FIFTH + +typedef struct { + CURLversion age; /* age of the returned struct */ + const char *version; /* LIBCURL_VERSION */ + unsigned int version_num; /* LIBCURL_VERSION_NUM */ + const char *host; /* OS/host/cpu/machine when configured */ + int features; /* bitmask, see defines below */ + const char *ssl_version; /* human readable string */ + long ssl_version_num; /* not used anymore, always 0 */ + const char *libz_version; /* human readable string */ + /* protocols is terminated by an entry with a NULL protoname */ + const char * const *protocols; + + /* The fields below this were added in CURLVERSION_SECOND */ + const char *ares; + int ares_num; + + /* This field was added in CURLVERSION_THIRD */ + const char *libidn; + + /* These field were added in CURLVERSION_FOURTH */ + + /* Same as '_libiconv_version' if built with HAVE_ICONV */ + int iconv_ver_num; + + const char *libssh_version; /* human readable string */ + + /* These fields were added in CURLVERSION_FIFTH */ + + unsigned int brotli_ver_num; /* Numeric Brotli version + (MAJOR << 24) | (MINOR << 12) | PATCH */ + const char *brotli_version; /* human readable string. */ + +} curl_version_info_data; + +#define CURL_VERSION_IPV6 (1<<0) /* IPv6-enabled */ +#define CURL_VERSION_KERBEROS4 (1<<1) /* Kerberos V4 auth is supported + (deprecated) */ +#define CURL_VERSION_SSL (1<<2) /* SSL options are present */ +#define CURL_VERSION_LIBZ (1<<3) /* libz features are present */ +#define CURL_VERSION_NTLM (1<<4) /* NTLM auth is supported */ +#define CURL_VERSION_GSSNEGOTIATE (1<<5) /* Negotiate auth is supported + (deprecated) */ +#define CURL_VERSION_DEBUG (1<<6) /* Built with debug capabilities */ +#define CURL_VERSION_ASYNCHDNS (1<<7) /* Asynchronous DNS resolves */ +#define CURL_VERSION_SPNEGO (1<<8) /* SPNEGO auth is supported */ +#define CURL_VERSION_LARGEFILE (1<<9) /* Supports files larger than 2GB */ +#define CURL_VERSION_IDN (1<<10) /* Internationized Domain Names are + supported */ +#define CURL_VERSION_SSPI (1<<11) /* Built against Windows SSPI */ +#define CURL_VERSION_CONV (1<<12) /* Character conversions supported */ +#define CURL_VERSION_CURLDEBUG (1<<13) /* Debug memory tracking supported */ +#define CURL_VERSION_TLSAUTH_SRP (1<<14) /* TLS-SRP auth is supported */ +#define CURL_VERSION_NTLM_WB (1<<15) /* NTLM delegation to winbind helper + is supported */ +#define CURL_VERSION_HTTP2 (1<<16) /* HTTP2 support built-in */ +#define CURL_VERSION_GSSAPI (1<<17) /* Built against a GSS-API library */ +#define CURL_VERSION_KERBEROS5 (1<<18) /* Kerberos V5 auth is supported */ +#define CURL_VERSION_UNIX_SOCKETS (1<<19) /* Unix domain sockets support */ +#define CURL_VERSION_PSL (1<<20) /* Mozilla's Public Suffix List, used + for cookie domain verification */ +#define CURL_VERSION_HTTPS_PROXY (1<<21) /* HTTPS-proxy support built-in */ +#define CURL_VERSION_MULTI_SSL (1<<22) /* Multiple SSL backends available */ +#define CURL_VERSION_BROTLI (1<<23) /* Brotli features are present. */ + + /* + * NAME curl_version_info() + * + * DESCRIPTION + * + * This function returns a pointer to a static copy of the version info + * struct. See above. + */ +CURL_EXTERN curl_version_info_data *curl_version_info(CURLversion); + +/* + * NAME curl_easy_strerror() + * + * DESCRIPTION + * + * The curl_easy_strerror function may be used to turn a CURLcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_easy_strerror(CURLcode); + +/* + * NAME curl_share_strerror() + * + * DESCRIPTION + * + * The curl_share_strerror function may be used to turn a CURLSHcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_share_strerror(CURLSHcode); + +/* + * NAME curl_easy_pause() + * + * DESCRIPTION + * + * The curl_easy_pause function pauses or unpauses transfers. Select the new + * state by setting the bitmask, use the convenience defines below. + * + */ +CURL_EXTERN CURLcode curl_easy_pause(CURL *handle, int bitmask); + +#define CURLPAUSE_RECV (1<<0) +#define CURLPAUSE_RECV_CONT (0) + +#define CURLPAUSE_SEND (1<<2) +#define CURLPAUSE_SEND_CONT (0) + +#define CURLPAUSE_ALL (CURLPAUSE_RECV|CURLPAUSE_SEND) +#define CURLPAUSE_CONT (CURLPAUSE_RECV_CONT|CURLPAUSE_SEND_CONT) + +#ifdef __cplusplus +} +#endif + +/* unfortunately, the easy.h and multi.h include files need options and info + stuff before they can be included! */ +#include "easy.h" /* nothing in curl is fun without the easy stuff */ +#include "multi.h" + +/* the typechecker doesn't work in C++ (yet) */ +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && \ + !defined(__cplusplus) && !defined(CURL_DISABLE_TYPECHECK) +#include "typecheck-gcc.h" +#else +#if defined(__STDC__) && (__STDC__ >= 1) +/* This preprocessor magic that replaces a call with the exact same call is + only done to make sure application authors pass exactly three arguments + to these functions. */ +#define curl_easy_setopt(handle,opt,param) curl_easy_setopt(handle,opt,param) +#define curl_easy_getinfo(handle,info,arg) curl_easy_getinfo(handle,info,arg) +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) +#endif /* __STDC__ >= 1 */ +#endif /* gcc >= 4.3 && !__cplusplus */ + +#endif /* __CURL_CURL_H */ diff --git a/ext/curl-7.58.0/x64/include/curl/curlver.h b/ext/curl-7.58.0/x64/include/curl/curlver.h new file mode 100644 index 00000000..6d93cc11 --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/curlver.h @@ -0,0 +1,77 @@ +#ifndef __CURL_CURLVER_H +#define __CURL_CURLVER_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* This header file contains nothing but libcurl version info, generated by + a script at release-time. This was made its own header file in 7.11.2 */ + +/* This is the global package copyright */ +#define LIBCURL_COPYRIGHT "1996 - 2017 Daniel Stenberg, <daniel@haxx.se>." + +/* This is the version number of the libcurl package from which this header + file origins: */ +#define LIBCURL_VERSION "7.58.0" + +/* The numeric version number is also available "in parts" by using these + defines: */ +#define LIBCURL_VERSION_MAJOR 7 +#define LIBCURL_VERSION_MINOR 58 +#define LIBCURL_VERSION_PATCH 0 + +/* This is the numeric version of the libcurl version number, meant for easier + parsing and comparions by programs. The LIBCURL_VERSION_NUM define will + always follow this syntax: + + 0xXXYYZZ + + Where XX, YY and ZZ are the main version, release and patch numbers in + hexadecimal (using 8 bits each). All three numbers are always represented + using two digits. 1.2 would appear as "0x010200" while version 9.11.7 + appears as "0x090b07". + + This 6-digit (24 bits) hexadecimal number does not show pre-release number, + and it is always a greater number in a more recent release. It makes + comparisons with greater than and less than work. + + Note: This define is the full hex number and _does not_ use the + CURL_VERSION_BITS() macro since curl's own configure script greps for it + and needs it to contain the full number. +*/ +#define LIBCURL_VERSION_NUM 0x073a00 + +/* + * This is the date and time when the full source package was created. The + * timestamp is not stored in git, as the timestamp is properly set in the + * tarballs by the maketgz script. + * + * The format of the date follows this template: + * + * "2007-11-23" + */ +#define LIBCURL_TIMESTAMP "2018-01-24" + +#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) +#define CURL_AT_LEAST_VERSION(x,y,z) \ + (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) + +#endif /* __CURL_CURLVER_H */ diff --git a/ext/curl-7.58.0/x64/include/curl/easy.h b/ext/curl-7.58.0/x64/include/curl/easy.h new file mode 100644 index 00000000..752c5049 --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/easy.h @@ -0,0 +1,102 @@ +#ifndef __CURL_EASY_H +#define __CURL_EASY_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN CURL *curl_easy_init(void); +CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); +CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); +CURL_EXTERN void curl_easy_cleanup(CURL *curl); + +/* + * NAME curl_easy_getinfo() + * + * DESCRIPTION + * + * Request internal information from the curl session with this function. The + * third argument MUST be a pointer to a long, a pointer to a char * or a + * pointer to a double (as the documentation describes elsewhere). The data + * pointed to will be filled in accordingly and can be relied upon only if the + * function returns CURLE_OK. This function is intended to get used *AFTER* a + * performed transfer, all results from this function are undefined until the + * transfer is completed. + */ +CURL_EXTERN CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...); + + +/* + * NAME curl_easy_duphandle() + * + * DESCRIPTION + * + * Creates a new curl session handle with the same options set for the handle + * passed in. Duplicating a handle could only be a matter of cloning data and + * options, internal state info and things like persistent connections cannot + * be transferred. It is useful in multithreaded applications when you can run + * curl_easy_duphandle() for each new thread to avoid a series of identical + * curl_easy_setopt() invokes in every thread. + */ +CURL_EXTERN CURL *curl_easy_duphandle(CURL *curl); + +/* + * NAME curl_easy_reset() + * + * DESCRIPTION + * + * Re-initializes a CURL handle to the default values. This puts back the + * handle to the same state as it was in when it was just created. + * + * It does keep: live connections, the Session ID cache, the DNS cache and the + * cookies. + */ +CURL_EXTERN void curl_easy_reset(CURL *curl); + +/* + * NAME curl_easy_recv() + * + * DESCRIPTION + * + * Receives data from the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen, + size_t *n); + +/* + * NAME curl_easy_send() + * + * DESCRIPTION + * + * Sends data over the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer, + size_t buflen, size_t *n); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ext/curl-7.58.0/x64/include/curl/mprintf.h b/ext/curl-7.58.0/x64/include/curl/mprintf.h new file mode 100644 index 00000000..e20f546e --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/mprintf.h @@ -0,0 +1,50 @@ +#ifndef __CURL_MPRINTF_H +#define __CURL_MPRINTF_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include <stdarg.h> +#include <stdio.h> /* needed for FILE */ +#include "curl.h" /* for CURL_EXTERN */ + +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN int curl_mprintf(const char *format, ...); +CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...); +CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...); +CURL_EXTERN int curl_msnprintf(char *buffer, size_t maxlength, + const char *format, ...); +CURL_EXTERN int curl_mvprintf(const char *format, va_list args); +CURL_EXTERN int curl_mvfprintf(FILE *fd, const char *format, va_list args); +CURL_EXTERN int curl_mvsprintf(char *buffer, const char *format, va_list args); +CURL_EXTERN int curl_mvsnprintf(char *buffer, size_t maxlength, + const char *format, va_list args); +CURL_EXTERN char *curl_maprintf(const char *format, ...); +CURL_EXTERN char *curl_mvaprintf(const char *format, va_list args); + +#ifdef __cplusplus +} +#endif + +#endif /* __CURL_MPRINTF_H */ diff --git a/ext/curl-7.58.0/x64/include/curl/multi.h b/ext/curl-7.58.0/x64/include/curl/multi.h new file mode 100644 index 00000000..911c91dd --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/multi.h @@ -0,0 +1,439 @@ +#ifndef __CURL_MULTI_H +#define __CURL_MULTI_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +/* + This is an "external" header file. Don't give away any internals here! + + GOALS + + o Enable a "pull" interface. The application that uses libcurl decides where + and when to ask libcurl to get/send data. + + o Enable multiple simultaneous transfers in the same thread without making it + complicated for the application. + + o Enable the application to select() on its own file descriptors and curl's + file descriptors simultaneous easily. + +*/ + +/* + * This header file should not really need to include "curl.h" since curl.h + * itself includes this file and we expect user applications to do #include + * <curl/curl.h> without the need for especially including multi.h. + * + * For some reason we added this include here at one point, and rather than to + * break existing (wrongly written) libcurl applications, we leave it as-is + * but with this warning attached. + */ +#include "curl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(BUILDING_LIBCURL) || defined(CURL_STRICTER) +typedef struct Curl_multi CURLM; +#else +typedef void CURLM; +#endif + +typedef enum { + CURLM_CALL_MULTI_PERFORM = -1, /* please call curl_multi_perform() or + curl_multi_socket*() soon */ + CURLM_OK, + CURLM_BAD_HANDLE, /* the passed-in handle is not a valid CURLM handle */ + CURLM_BAD_EASY_HANDLE, /* an easy handle was not good/valid */ + CURLM_OUT_OF_MEMORY, /* if you ever get this, you're in deep sh*t */ + CURLM_INTERNAL_ERROR, /* this is a libcurl bug */ + CURLM_BAD_SOCKET, /* the passed in socket argument did not match */ + CURLM_UNKNOWN_OPTION, /* curl_multi_setopt() with unsupported option */ + CURLM_ADDED_ALREADY, /* an easy handle already added to a multi handle was + attempted to get added - again */ + CURLM_LAST +} CURLMcode; + +/* just to make code nicer when using curl_multi_socket() you can now check + for CURLM_CALL_MULTI_SOCKET too in the same style it works for + curl_multi_perform() and CURLM_CALL_MULTI_PERFORM */ +#define CURLM_CALL_MULTI_SOCKET CURLM_CALL_MULTI_PERFORM + +/* bitmask bits for CURLMOPT_PIPELINING */ +#define CURLPIPE_NOTHING 0L +#define CURLPIPE_HTTP1 1L +#define CURLPIPE_MULTIPLEX 2L + +typedef enum { + CURLMSG_NONE, /* first, not used */ + CURLMSG_DONE, /* This easy handle has completed. 'result' contains + the CURLcode of the transfer */ + CURLMSG_LAST /* last, not used */ +} CURLMSG; + +struct CURLMsg { + CURLMSG msg; /* what this message means */ + CURL *easy_handle; /* the handle it concerns */ + union { + void *whatever; /* message-specific data */ + CURLcode result; /* return code for transfer */ + } data; +}; +typedef struct CURLMsg CURLMsg; + +/* Based on poll(2) structure and values. + * We don't use pollfd and POLL* constants explicitly + * to cover platforms without poll(). */ +#define CURL_WAIT_POLLIN 0x0001 +#define CURL_WAIT_POLLPRI 0x0002 +#define CURL_WAIT_POLLOUT 0x0004 + +struct curl_waitfd { + curl_socket_t fd; + short events; + short revents; /* not supported yet */ +}; + +/* + * Name: curl_multi_init() + * + * Desc: inititalize multi-style curl usage + * + * Returns: a new CURLM handle to use in all 'curl_multi' functions. + */ +CURL_EXTERN CURLM *curl_multi_init(void); + +/* + * Name: curl_multi_add_handle() + * + * Desc: add a standard curl handle to the multi stack + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_add_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_remove_handle() + * + * Desc: removes a curl handle from the multi stack again + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_remove_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_fdset() + * + * Desc: Ask curl for its fd_set sets. The app can use these to select() or + * poll() on. We want curl_multi_perform() called as soon as one of + * them are ready. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_fdset(CURLM *multi_handle, + fd_set *read_fd_set, + fd_set *write_fd_set, + fd_set *exc_fd_set, + int *max_fd); + +/* + * Name: curl_multi_wait() + * + * Desc: Poll on all fds within a CURLM set as well as any + * additional fds passed to the function. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_wait(CURLM *multi_handle, + struct curl_waitfd extra_fds[], + unsigned int extra_nfds, + int timeout_ms, + int *ret); + + /* + * Name: curl_multi_perform() + * + * Desc: When the app thinks there's data available for curl it calls this + * function to read/write whatever there is right now. This returns + * as soon as the reads and writes are done. This function does not + * require that there actually is data available for reading or that + * data can be written, it can be called just in case. It returns + * the number of handles that still transfer data in the second + * argument's integer-pointer. + * + * Returns: CURLMcode type, general multi error code. *NOTE* that this only + * returns errors etc regarding the whole multi stack. There might + * still have occurred problems on invidual transfers even when this + * returns OK. + */ +CURL_EXTERN CURLMcode curl_multi_perform(CURLM *multi_handle, + int *running_handles); + + /* + * Name: curl_multi_cleanup() + * + * Desc: Cleans up and removes a whole multi stack. It does not free or + * touch any individual easy handles in any way. We need to define + * in what state those handles will be if this function is called + * in the middle of a transfer. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_cleanup(CURLM *multi_handle); + +/* + * Name: curl_multi_info_read() + * + * Desc: Ask the multi handle if there's any messages/informationals from + * the individual transfers. Messages include informationals such as + * error code from the transfer or just the fact that a transfer is + * completed. More details on these should be written down as well. + * + * Repeated calls to this function will return a new struct each + * time, until a special "end of msgs" struct is returned as a signal + * that there is no more to get at this point. + * + * The data the returned pointer points to will not survive calling + * curl_multi_cleanup(). + * + * The 'CURLMsg' struct is meant to be very simple and only contain + * very basic information. If more involved information is wanted, + * we will provide the particular "transfer handle" in that struct + * and that should/could/would be used in subsequent + * curl_easy_getinfo() calls (or similar). The point being that we + * must never expose complex structs to applications, as then we'll + * undoubtably get backwards compatibility problems in the future. + * + * Returns: A pointer to a filled-in struct, or NULL if it failed or ran out + * of structs. It also writes the number of messages left in the + * queue (after this read) in the integer the second argument points + * to. + */ +CURL_EXTERN CURLMsg *curl_multi_info_read(CURLM *multi_handle, + int *msgs_in_queue); + +/* + * Name: curl_multi_strerror() + * + * Desc: The curl_multi_strerror function may be used to turn a CURLMcode + * value into the equivalent human readable error string. This is + * useful for printing meaningful error messages. + * + * Returns: A pointer to a zero-terminated error message. + */ +CURL_EXTERN const char *curl_multi_strerror(CURLMcode); + +/* + * Name: curl_multi_socket() and + * curl_multi_socket_all() + * + * Desc: An alternative version of curl_multi_perform() that allows the + * application to pass in one of the file descriptors that have been + * detected to have "action" on them and let libcurl perform. + * See man page for details. + */ +#define CURL_POLL_NONE 0 +#define CURL_POLL_IN 1 +#define CURL_POLL_OUT 2 +#define CURL_POLL_INOUT 3 +#define CURL_POLL_REMOVE 4 + +#define CURL_SOCKET_TIMEOUT CURL_SOCKET_BAD + +#define CURL_CSELECT_IN 0x01 +#define CURL_CSELECT_OUT 0x02 +#define CURL_CSELECT_ERR 0x04 + +typedef int (*curl_socket_callback)(CURL *easy, /* easy handle */ + curl_socket_t s, /* socket */ + int what, /* see above */ + void *userp, /* private callback + pointer */ + void *socketp); /* private socket + pointer */ +/* + * Name: curl_multi_timer_callback + * + * Desc: Called by libcurl whenever the library detects a change in the + * maximum number of milliseconds the app is allowed to wait before + * curl_multi_socket() or curl_multi_perform() must be called + * (to allow libcurl's timed events to take place). + * + * Returns: The callback should return zero. + */ +typedef int (*curl_multi_timer_callback)(CURLM *multi, /* multi handle */ + long timeout_ms, /* see above */ + void *userp); /* private callback + pointer */ + +CURL_EXTERN CURLMcode curl_multi_socket(CURLM *multi_handle, curl_socket_t s, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_action(CURLM *multi_handle, + curl_socket_t s, + int ev_bitmask, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_all(CURLM *multi_handle, + int *running_handles); + +#ifndef CURL_ALLOW_OLD_MULTI_SOCKET +/* This macro below was added in 7.16.3 to push users who recompile to use + the new curl_multi_socket_action() instead of the old curl_multi_socket() +*/ +#define curl_multi_socket(x,y,z) curl_multi_socket_action(x,y,0,z) +#endif + +/* + * Name: curl_multi_timeout() + * + * Desc: Returns the maximum number of milliseconds the app is allowed to + * wait before curl_multi_socket() or curl_multi_perform() must be + * called (to allow libcurl's timed events to take place). + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_timeout(CURLM *multi_handle, + long *milliseconds); + +#undef CINIT /* re-using the same name as in curl.h */ + +#ifdef CURL_ISOCPP +#define CINIT(name,type,num) CURLMOPT_ ## name = CURLOPTTYPE_ ## type + num +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLMOPT_/**/name = type + number +#endif + +typedef enum { + /* This is the socket callback function pointer */ + CINIT(SOCKETFUNCTION, FUNCTIONPOINT, 1), + + /* This is the argument passed to the socket callback */ + CINIT(SOCKETDATA, OBJECTPOINT, 2), + + /* set to 1 to enable pipelining for this multi handle */ + CINIT(PIPELINING, LONG, 3), + + /* This is the timer callback function pointer */ + CINIT(TIMERFUNCTION, FUNCTIONPOINT, 4), + + /* This is the argument passed to the timer callback */ + CINIT(TIMERDATA, OBJECTPOINT, 5), + + /* maximum number of entries in the connection cache */ + CINIT(MAXCONNECTS, LONG, 6), + + /* maximum number of (pipelining) connections to one host */ + CINIT(MAX_HOST_CONNECTIONS, LONG, 7), + + /* maximum number of requests in a pipeline */ + CINIT(MAX_PIPELINE_LENGTH, LONG, 8), + + /* a connection with a content-length longer than this + will not be considered for pipelining */ + CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9), + + /* a connection with a chunk length longer than this + will not be considered for pipelining */ + CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10), + + /* a list of site names(+port) that are blacklisted from + pipelining */ + CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11), + + /* a list of server types that are blacklisted from + pipelining */ + CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12), + + /* maximum number of open connections in total */ + CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13), + + /* This is the server push callback function pointer */ + CINIT(PUSHFUNCTION, FUNCTIONPOINT, 14), + + /* This is the argument passed to the server push callback */ + CINIT(PUSHDATA, OBJECTPOINT, 15), + + CURLMOPT_LASTENTRY /* the last unused */ +} CURLMoption; + + +/* + * Name: curl_multi_setopt() + * + * Desc: Sets options for the multi handle. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_setopt(CURLM *multi_handle, + CURLMoption option, ...); + + +/* + * Name: curl_multi_assign() + * + * Desc: This function sets an association in the multi handle between the + * given socket and a private pointer of the application. This is + * (only) useful for curl_multi_socket uses. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle, + curl_socket_t sockfd, void *sockp); + + +/* + * Name: curl_push_callback + * + * Desc: This callback gets called when a new stream is being pushed by the + * server. It approves or denies the new stream. + * + * Returns: CURL_PUSH_OK or CURL_PUSH_DENY. + */ +#define CURL_PUSH_OK 0 +#define CURL_PUSH_DENY 1 + +struct curl_pushheaders; /* forward declaration only */ + +CURL_EXTERN char *curl_pushheader_bynum(struct curl_pushheaders *h, + size_t num); +CURL_EXTERN char *curl_pushheader_byname(struct curl_pushheaders *h, + const char *name); + +typedef int (*curl_push_callback)(CURL *parent, + CURL *easy, + size_t num_headers, + struct curl_pushheaders *headers, + void *userp); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif diff --git a/ext/curl-7.58.0/x64/include/curl/stdcheaders.h b/ext/curl-7.58.0/x64/include/curl/stdcheaders.h new file mode 100644 index 00000000..027b6f42 --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/stdcheaders.h @@ -0,0 +1,33 @@ +#ifndef __STDC_HEADERS_H +#define __STDC_HEADERS_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include <sys/types.h> + +size_t fread(void *, size_t, size_t, FILE *); +size_t fwrite(const void *, size_t, size_t, FILE *); + +int strcasecmp(const char *, const char *); +int strncasecmp(const char *, const char *, size_t); + +#endif /* __STDC_HEADERS_H */ diff --git a/ext/curl-7.58.0/x64/include/curl/system.h b/ext/curl-7.58.0/x64/include/curl/system.h new file mode 100644 index 00000000..07bbd9ca --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/system.h @@ -0,0 +1,473 @@ +#ifndef __CURL_SYSTEM_H +#define __CURL_SYSTEM_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* + * Try to keep one section per platform, compiler and architecture, otherwise, + * if an existing section is reused for a different one and later on the + * original is adjusted, probably the piggybacking one can be adversely + * changed. + * + * In order to differentiate between platforms/compilers/architectures use + * only compiler built in predefined preprocessor symbols. + * + * curl_off_t + * ---------- + * + * For any given platform/compiler curl_off_t must be typedef'ed to a 64-bit + * wide signed integral data type. The width of this data type must remain + * constant and independent of any possible large file support settings. + * + * As an exception to the above, curl_off_t shall be typedef'ed to a 32-bit + * wide signed integral data type if there is no 64-bit type. + * + * As a general rule, curl_off_t shall not be mapped to off_t. This rule shall + * only be violated if off_t is the only 64-bit data type available and the + * size of off_t is independent of large file support settings. Keep your + * build on the safe side avoiding an off_t gating. If you have a 64-bit + * off_t then take for sure that another 64-bit data type exists, dig deeper + * and you will find it. + * + */ + +#if defined(__DJGPP__) || defined(__GO32__) +# if defined(__DJGPP__) && (__DJGPP__ > 1) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__SALFORDC__) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__BORLANDC__) +# if (__BORLANDC__ < 0x520) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__TURBOC__) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__WATCOMC__) +# if defined(__386__) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__POCC__) +# if (__POCC__ < 280) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# elif defined(_MSC_VER) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# else +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__LCC__) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__SYMBIAN32__) +# if defined(__EABI__) /* Treat all ARM compilers equally */ +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__CW32__) +# pragma longlong on +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__VC32__) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T unsigned int + +#elif defined(__MWERKS__) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(_WIN32_WCE) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__MINGW32__) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_WS2TCPIP_H 1 + +#elif defined(__VMS) +# if defined(__VAX) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T unsigned int + +#elif defined(__OS400__) +# if defined(__ILEC400__) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 +# endif + +#elif defined(__MVS__) +# if defined(__IBMC__) || defined(__IBMCPP__) +# if defined(_ILP32) +# elif defined(_LP64) +# endif +# if defined(_LONG_LONG) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(_LP64) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 +# endif + +#elif defined(__370__) +# if defined(__IBMC__) || defined(__IBMCPP__) +# if defined(_ILP32) +# elif defined(_LP64) +# endif +# if defined(_LONG_LONG) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(_LP64) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 +# endif + +#elif defined(TPF) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +#elif defined(__TINYC__) /* also known as tcc */ + +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + +#elif defined(__SUNPRO_C) /* Oracle Solaris Studio */ +# if !defined(__LP64) && (defined(__ILP32) || \ + defined(__i386) || defined(__sparcv8)) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__LP64) || \ + defined(__amd64) || defined(__sparcv9) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + +/* ===================================== */ +/* KEEP MSVC THE PENULTIMATE ENTRY */ +/* ===================================== */ + +#elif defined(_MSC_VER) +# if (_MSC_VER >= 900) && (_INTEGRAL_MAX_BITS >= 64) +# define CURL_TYPEOF_CURL_OFF_T __int64 +# define CURL_FORMAT_CURL_OFF_T "I64d" +# define CURL_FORMAT_CURL_OFF_TU "I64u" +# define CURL_SUFFIX_CURL_OFF_T i64 +# define CURL_SUFFIX_CURL_OFF_TU ui64 +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T int + +/* ===================================== */ +/* KEEP GENERIC GCC THE LAST ENTRY */ +/* ===================================== */ + +#elif defined(__GNUC__) +# if !defined(__LP64__) && \ + (defined(__ILP32__) || defined(__i386__) || defined(__hppa__) || \ + defined(__ppc__) || defined(__powerpc__) || defined(__arm__) || \ + defined(__sparc__) || defined(__mips__) || defined(__sh__) || \ + defined(__XTENSA__) || \ + (defined(__SIZEOF_LONG__) && __SIZEOF_LONG__ == 4) || \ + (defined(__LONG_MAX__) && __LONG_MAX__ == 2147483647L)) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# elif defined(__LP64__) || \ + defined(__x86_64__) || defined(__ppc64__) || defined(__sparc64__) || \ + (defined(__SIZEOF_LONG__) && __SIZEOF_LONG__ == 8) || \ + (defined(__LONG_MAX__) && __LONG_MAX__ == 9223372036854775807L) +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + +#else +/* generic "safe guess" on old 32 bit style */ +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# define CURL_TYPEOF_CURL_SOCKLEN_T int +#endif + +#ifdef _AIX +/* AIX needs <sys/poll.h> */ +#define CURL_PULL_SYS_POLL_H +#endif + + +/* CURL_PULL_WS2TCPIP_H is defined above when inclusion of header file */ +/* ws2tcpip.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_WS2TCPIP_H +# include <winsock2.h> +# include <windows.h> +# include <ws2tcpip.h> +#endif + +/* CURL_PULL_SYS_TYPES_H is defined above when inclusion of header file */ +/* sys/types.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_SYS_TYPES_H +# include <sys/types.h> +#endif + +/* CURL_PULL_SYS_SOCKET_H is defined above when inclusion of header file */ +/* sys/socket.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_SYS_SOCKET_H +# include <sys/socket.h> +#endif + +/* CURL_PULL_SYS_POLL_H is defined above when inclusion of header file */ +/* sys/poll.h is required here to properly make type definitions below. */ +#ifdef CURL_PULL_SYS_POLL_H +# include <sys/poll.h> +#endif + +/* Data type definition of curl_socklen_t. */ +#ifdef CURL_TYPEOF_CURL_SOCKLEN_T + typedef CURL_TYPEOF_CURL_SOCKLEN_T curl_socklen_t; +#endif + +/* Data type definition of curl_off_t. */ + +#ifdef CURL_TYPEOF_CURL_OFF_T + typedef CURL_TYPEOF_CURL_OFF_T curl_off_t; +#endif + +/* + * CURL_ISOCPP and CURL_OFF_T_C definitions are done here in order to allow + * these to be visible and exported by the external libcurl interface API, + * while also making them visible to the library internals, simply including + * curl_setup.h, without actually needing to include curl.h internally. + * If some day this section would grow big enough, all this should be moved + * to its own header file. + */ + +/* + * Figure out if we can use the ## preprocessor operator, which is supported + * by ISO/ANSI C and C++. Some compilers support it without setting __STDC__ + * or __cplusplus so we need to carefully check for them too. + */ + +#if defined(__STDC__) || defined(_MSC_VER) || defined(__cplusplus) || \ + defined(__HP_aCC) || defined(__BORLANDC__) || defined(__LCC__) || \ + defined(__POCC__) || defined(__SALFORDC__) || defined(__HIGHC__) || \ + defined(__ILEC400__) + /* This compiler is believed to have an ISO compatible preprocessor */ +#define CURL_ISOCPP +#else + /* This compiler is believed NOT to have an ISO compatible preprocessor */ +#undef CURL_ISOCPP +#endif + +/* + * Macros for minimum-width signed and unsigned curl_off_t integer constants. + */ + +#if defined(__BORLANDC__) && (__BORLANDC__ == 0x0551) +# define __CURL_OFF_T_C_HLPR2(x) x +# define __CURL_OFF_T_C_HLPR1(x) __CURL_OFF_T_C_HLPR2(x) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_TU) +#else +# ifdef CURL_ISOCPP +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val ## Suffix +# else +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val/**/Suffix +# endif +# define __CURL_OFF_T_C_HLPR1(Val,Suffix) __CURL_OFF_T_C_HLPR2(Val,Suffix) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_TU) +#endif + +#endif /* __CURL_SYSTEM_H */ diff --git a/ext/curl-7.58.0/x64/include/curl/typecheck-gcc.h b/ext/curl-7.58.0/x64/include/curl/typecheck-gcc.h new file mode 100644 index 00000000..10c74c76 --- /dev/null +++ b/ext/curl-7.58.0/x64/include/curl/typecheck-gcc.h @@ -0,0 +1,683 @@ +#ifndef __CURL_TYPECHECK_GCC_H +#define __CURL_TYPECHECK_GCC_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* wraps curl_easy_setopt() with typechecking */ + +/* To add a new kind of warning, add an + * if(_curl_is_sometype_option(_curl_opt)) + * if(!_curl_is_sometype(value)) + * _curl_easy_setopt_err_sometype(); + * block and define _curl_is_sometype_option, _curl_is_sometype and + * _curl_easy_setopt_err_sometype below + * + * NOTE: We use two nested 'if' statements here instead of the && operator, in + * order to work around gcc bug #32061. It affects only gcc 4.3.x/4.4.x + * when compiling with -Wlogical-op. + * + * To add an option that uses the same type as an existing option, you'll just + * need to extend the appropriate _curl_*_option macro + */ +#define curl_easy_setopt(handle, option, value) \ +__extension__ ({ \ + __typeof__(option) _curl_opt = option; \ + if(__builtin_constant_p(_curl_opt)) { \ + if(_curl_is_long_option(_curl_opt)) \ + if(!_curl_is_long(value)) \ + _curl_easy_setopt_err_long(); \ + if(_curl_is_off_t_option(_curl_opt)) \ + if(!_curl_is_off_t(value)) \ + _curl_easy_setopt_err_curl_off_t(); \ + if(_curl_is_string_option(_curl_opt)) \ + if(!_curl_is_string(value)) \ + _curl_easy_setopt_err_string(); \ + if(_curl_is_write_cb_option(_curl_opt)) \ + if(!_curl_is_write_cb(value)) \ + _curl_easy_setopt_err_write_callback(); \ + if((_curl_opt) == CURLOPT_READFUNCTION) \ + if(!_curl_is_read_cb(value)) \ + _curl_easy_setopt_err_read_cb(); \ + if((_curl_opt) == CURLOPT_IOCTLFUNCTION) \ + if(!_curl_is_ioctl_cb(value)) \ + _curl_easy_setopt_err_ioctl_cb(); \ + if((_curl_opt) == CURLOPT_SOCKOPTFUNCTION) \ + if(!_curl_is_sockopt_cb(value)) \ + _curl_easy_setopt_err_sockopt_cb(); \ + if((_curl_opt) == CURLOPT_OPENSOCKETFUNCTION) \ + if(!_curl_is_opensocket_cb(value)) \ + _curl_easy_setopt_err_opensocket_cb(); \ + if((_curl_opt) == CURLOPT_PROGRESSFUNCTION) \ + if(!_curl_is_progress_cb(value)) \ + _curl_easy_setopt_err_progress_cb(); \ + if((_curl_opt) == CURLOPT_DEBUGFUNCTION) \ + if(!_curl_is_debug_cb(value)) \ + _curl_easy_setopt_err_debug_cb(); \ + if((_curl_opt) == CURLOPT_SSL_CTX_FUNCTION) \ + if(!_curl_is_ssl_ctx_cb(value)) \ + _curl_easy_setopt_err_ssl_ctx_cb(); \ + if(_curl_is_conv_cb_option(_curl_opt)) \ + if(!_curl_is_conv_cb(value)) \ + _curl_easy_setopt_err_conv_cb(); \ + if((_curl_opt) == CURLOPT_SEEKFUNCTION) \ + if(!_curl_is_seek_cb(value)) \ + _curl_easy_setopt_err_seek_cb(); \ + if(_curl_is_cb_data_option(_curl_opt)) \ + if(!_curl_is_cb_data(value)) \ + _curl_easy_setopt_err_cb_data(); \ + if((_curl_opt) == CURLOPT_ERRORBUFFER) \ + if(!_curl_is_error_buffer(value)) \ + _curl_easy_setopt_err_error_buffer(); \ + if((_curl_opt) == CURLOPT_STDERR) \ + if(!_curl_is_FILE(value)) \ + _curl_easy_setopt_err_FILE(); \ + if(_curl_is_postfields_option(_curl_opt)) \ + if(!_curl_is_postfields(value)) \ + _curl_easy_setopt_err_postfields(); \ + if((_curl_opt) == CURLOPT_HTTPPOST) \ + if(!_curl_is_arr((value), struct curl_httppost)) \ + _curl_easy_setopt_err_curl_httpost(); \ + if((_curl_opt) == CURLOPT_MIMEPOST) \ + if(!_curl_is_ptr((value), curl_mime)) \ + _curl_easy_setopt_err_curl_mimepost(); \ + if(_curl_is_slist_option(_curl_opt)) \ + if(!_curl_is_arr((value), struct curl_slist)) \ + _curl_easy_setopt_err_curl_slist(); \ + if((_curl_opt) == CURLOPT_SHARE) \ + if(!_curl_is_ptr((value), CURLSH)) \ + _curl_easy_setopt_err_CURLSH(); \ + } \ + curl_easy_setopt(handle, _curl_opt, value); \ +}) + +/* wraps curl_easy_getinfo() with typechecking */ +/* FIXME: don't allow const pointers */ +#define curl_easy_getinfo(handle, info, arg) \ +__extension__ ({ \ + __typeof__(info) _curl_info = info; \ + if(__builtin_constant_p(_curl_info)) { \ + if(_curl_is_string_info(_curl_info)) \ + if(!_curl_is_arr((arg), char *)) \ + _curl_easy_getinfo_err_string(); \ + if(_curl_is_long_info(_curl_info)) \ + if(!_curl_is_arr((arg), long)) \ + _curl_easy_getinfo_err_long(); \ + if(_curl_is_double_info(_curl_info)) \ + if(!_curl_is_arr((arg), double)) \ + _curl_easy_getinfo_err_double(); \ + if(_curl_is_slist_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_slist *)) \ + _curl_easy_getinfo_err_curl_slist(); \ + if(_curl_is_tlssessioninfo_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_tlssessioninfo *)) \ + _curl_easy_getinfo_err_curl_tlssesssioninfo(); \ + if(_curl_is_certinfo_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_certinfo *)) \ + _curl_easy_getinfo_err_curl_certinfo(); \ + if(_curl_is_socket_info(_curl_info)) \ + if(!_curl_is_arr((arg), curl_socket_t)) \ + _curl_easy_getinfo_err_curl_socket(); \ + if(_curl_is_off_t_info(_curl_info)) \ + if(!_curl_is_arr((arg), curl_off_t)) \ + _curl_easy_getinfo_err_curl_off_t(); \ + } \ + curl_easy_getinfo(handle, _curl_info, arg); \ +}) + +/* TODO: typechecking for curl_share_setopt() and curl_multi_setopt(), + * for now just make sure that the functions are called with three + * arguments + */ +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) + + +/* the actual warnings, triggered by calling the _curl_easy_setopt_err* + * functions */ + +/* To define a new warning, use _CURL_WARNING(identifier, "message") */ +#define _CURL_WARNING(id, message) \ + static void __attribute__((__warning__(message))) \ + __attribute__((__unused__)) __attribute__((__noinline__)) \ + id(void) { __asm__(""); } + +_CURL_WARNING(_curl_easy_setopt_err_long, + "curl_easy_setopt expects a long argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_off_t, + "curl_easy_setopt expects a curl_off_t argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_string, + "curl_easy_setopt expects a " + "string ('char *' or char[]) argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_write_callback, + "curl_easy_setopt expects a curl_write_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_read_cb, + "curl_easy_setopt expects a curl_read_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ioctl_cb, + "curl_easy_setopt expects a curl_ioctl_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_sockopt_cb, + "curl_easy_setopt expects a curl_sockopt_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_opensocket_cb, + "curl_easy_setopt expects a " + "curl_opensocket_callback argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_progress_cb, + "curl_easy_setopt expects a curl_progress_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_debug_cb, + "curl_easy_setopt expects a curl_debug_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ssl_ctx_cb, + "curl_easy_setopt expects a curl_ssl_ctx_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_conv_cb, + "curl_easy_setopt expects a curl_conv_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_seek_cb, + "curl_easy_setopt expects a curl_seek_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_cb_data, + "curl_easy_setopt expects a " + "private data pointer as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_error_buffer, + "curl_easy_setopt expects a " + "char buffer of CURL_ERROR_SIZE as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_FILE, + "curl_easy_setopt expects a 'FILE *' argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_postfields, + "curl_easy_setopt expects a 'void *' or 'char *' argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_httpost, + "curl_easy_setopt expects a 'struct curl_httppost *' " + "argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_mimepost, + "curl_easy_setopt expects a 'curl_mime *' " + "argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_slist, + "curl_easy_setopt expects a 'struct curl_slist *' argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_CURLSH, + "curl_easy_setopt expects a CURLSH* argument for this option") + +_CURL_WARNING(_curl_easy_getinfo_err_string, + "curl_easy_getinfo expects a pointer to 'char *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_long, + "curl_easy_getinfo expects a pointer to long for this info") +_CURL_WARNING(_curl_easy_getinfo_err_double, + "curl_easy_getinfo expects a pointer to double for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_slist, + "curl_easy_getinfo expects a pointer to 'struct curl_slist *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_tlssesssioninfo, + "curl_easy_getinfo expects a pointer to " + "'struct curl_tlssessioninfo *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_certinfo, + "curl_easy_getinfo expects a pointer to " + "'struct curl_certinfo *' for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_socket, + "curl_easy_getinfo expects a pointer to curl_socket_t for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_off_t, + "curl_easy_getinfo expects a pointer to curl_off_t for this info") + +/* groups of curl_easy_setops options that take the same type of argument */ + +/* To add a new option to one of the groups, just add + * (option) == CURLOPT_SOMETHING + * to the or-expression. If the option takes a long or curl_off_t, you don't + * have to do anything + */ + +/* evaluates to true if option takes a long argument */ +#define _curl_is_long_option(option) \ + (0 < (option) && (option) < CURLOPTTYPE_OBJECTPOINT) + +#define _curl_is_off_t_option(option) \ + ((option) > CURLOPTTYPE_OFF_T) + +/* evaluates to true if option takes a char* argument */ +#define _curl_is_string_option(option) \ + ((option) == CURLOPT_ABSTRACT_UNIX_SOCKET || \ + (option) == CURLOPT_ACCEPT_ENCODING || \ + (option) == CURLOPT_CAINFO || \ + (option) == CURLOPT_CAPATH || \ + (option) == CURLOPT_COOKIE || \ + (option) == CURLOPT_COOKIEFILE || \ + (option) == CURLOPT_COOKIEJAR || \ + (option) == CURLOPT_COOKIELIST || \ + (option) == CURLOPT_CRLFILE || \ + (option) == CURLOPT_CUSTOMREQUEST || \ + (option) == CURLOPT_DEFAULT_PROTOCOL || \ + (option) == CURLOPT_DNS_INTERFACE || \ + (option) == CURLOPT_DNS_LOCAL_IP4 || \ + (option) == CURLOPT_DNS_LOCAL_IP6 || \ + (option) == CURLOPT_DNS_SERVERS || \ + (option) == CURLOPT_EGDSOCKET || \ + (option) == CURLOPT_FTPPORT || \ + (option) == CURLOPT_FTP_ACCOUNT || \ + (option) == CURLOPT_FTP_ALTERNATIVE_TO_USER || \ + (option) == CURLOPT_INTERFACE || \ + (option) == CURLOPT_ISSUERCERT || \ + (option) == CURLOPT_KEYPASSWD || \ + (option) == CURLOPT_KRBLEVEL || \ + (option) == CURLOPT_LOGIN_OPTIONS || \ + (option) == CURLOPT_MAIL_AUTH || \ + (option) == CURLOPT_MAIL_FROM || \ + (option) == CURLOPT_NETRC_FILE || \ + (option) == CURLOPT_NOPROXY || \ + (option) == CURLOPT_PASSWORD || \ + (option) == CURLOPT_PINNEDPUBLICKEY || \ + (option) == CURLOPT_PRE_PROXY || \ + (option) == CURLOPT_PROXY || \ + (option) == CURLOPT_PROXYPASSWORD || \ + (option) == CURLOPT_PROXYUSERNAME || \ + (option) == CURLOPT_PROXYUSERPWD || \ + (option) == CURLOPT_PROXY_CAINFO || \ + (option) == CURLOPT_PROXY_CAPATH || \ + (option) == CURLOPT_PROXY_CRLFILE || \ + (option) == CURLOPT_PROXY_KEYPASSWD || \ + (option) == CURLOPT_PROXY_PINNEDPUBLICKEY || \ + (option) == CURLOPT_PROXY_SERVICE_NAME || \ + (option) == CURLOPT_PROXY_SSLCERT || \ + (option) == CURLOPT_PROXY_SSLCERTTYPE || \ + (option) == CURLOPT_PROXY_SSLKEY || \ + (option) == CURLOPT_PROXY_SSLKEYTYPE || \ + (option) == CURLOPT_PROXY_SSL_CIPHER_LIST || \ + (option) == CURLOPT_PROXY_TLSAUTH_PASSWORD || \ + (option) == CURLOPT_PROXY_TLSAUTH_USERNAME || \ + (option) == CURLOPT_PROXY_TLSAUTH_TYPE || \ + (option) == CURLOPT_RANDOM_FILE || \ + (option) == CURLOPT_RANGE || \ + (option) == CURLOPT_REFERER || \ + (option) == CURLOPT_RTSP_SESSION_ID || \ + (option) == CURLOPT_RTSP_STREAM_URI || \ + (option) == CURLOPT_RTSP_TRANSPORT || \ + (option) == CURLOPT_SERVICE_NAME || \ + (option) == CURLOPT_SOCKS5_GSSAPI_SERVICE || \ + (option) == CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 || \ + (option) == CURLOPT_SSH_KNOWNHOSTS || \ + (option) == CURLOPT_SSH_PRIVATE_KEYFILE || \ + (option) == CURLOPT_SSH_PUBLIC_KEYFILE || \ + (option) == CURLOPT_SSLCERT || \ + (option) == CURLOPT_SSLCERTTYPE || \ + (option) == CURLOPT_SSLENGINE || \ + (option) == CURLOPT_SSLKEY || \ + (option) == CURLOPT_SSLKEYTYPE || \ + (option) == CURLOPT_SSL_CIPHER_LIST || \ + (option) == CURLOPT_TLSAUTH_PASSWORD || \ + (option) == CURLOPT_TLSAUTH_TYPE || \ + (option) == CURLOPT_TLSAUTH_USERNAME || \ + (option) == CURLOPT_UNIX_SOCKET_PATH || \ + (option) == CURLOPT_URL || \ + (option) == CURLOPT_USERAGENT || \ + (option) == CURLOPT_USERNAME || \ + (option) == CURLOPT_USERPWD || \ + (option) == CURLOPT_XOAUTH2_BEARER || \ + 0) + +/* evaluates to true if option takes a curl_write_callback argument */ +#define _curl_is_write_cb_option(option) \ + ((option) == CURLOPT_HEADERFUNCTION || \ + (option) == CURLOPT_WRITEFUNCTION) + +/* evaluates to true if option takes a curl_conv_callback argument */ +#define _curl_is_conv_cb_option(option) \ + ((option) == CURLOPT_CONV_TO_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_UTF8_FUNCTION) + +/* evaluates to true if option takes a data argument to pass to a callback */ +#define _curl_is_cb_data_option(option) \ + ((option) == CURLOPT_CHUNK_DATA || \ + (option) == CURLOPT_CLOSESOCKETDATA || \ + (option) == CURLOPT_DEBUGDATA || \ + (option) == CURLOPT_FNMATCH_DATA || \ + (option) == CURLOPT_HEADERDATA || \ + (option) == CURLOPT_INTERLEAVEDATA || \ + (option) == CURLOPT_IOCTLDATA || \ + (option) == CURLOPT_OPENSOCKETDATA || \ + (option) == CURLOPT_PRIVATE || \ + (option) == CURLOPT_PROGRESSDATA || \ + (option) == CURLOPT_READDATA || \ + (option) == CURLOPT_SEEKDATA || \ + (option) == CURLOPT_SOCKOPTDATA || \ + (option) == CURLOPT_SSH_KEYDATA || \ + (option) == CURLOPT_SSL_CTX_DATA || \ + (option) == CURLOPT_WRITEDATA || \ + 0) + +/* evaluates to true if option takes a POST data argument (void* or char*) */ +#define _curl_is_postfields_option(option) \ + ((option) == CURLOPT_POSTFIELDS || \ + (option) == CURLOPT_COPYPOSTFIELDS || \ + 0) + +/* evaluates to true if option takes a struct curl_slist * argument */ +#define _curl_is_slist_option(option) \ + ((option) == CURLOPT_HTTP200ALIASES || \ + (option) == CURLOPT_HTTPHEADER || \ + (option) == CURLOPT_MAIL_RCPT || \ + (option) == CURLOPT_POSTQUOTE || \ + (option) == CURLOPT_PREQUOTE || \ + (option) == CURLOPT_PROXYHEADER || \ + (option) == CURLOPT_QUOTE || \ + (option) == CURLOPT_RESOLVE || \ + (option) == CURLOPT_TELNETOPTIONS || \ + 0) + +/* groups of curl_easy_getinfo infos that take the same type of argument */ + +/* evaluates to true if info expects a pointer to char * argument */ +#define _curl_is_string_info(info) \ + (CURLINFO_STRING < (info) && (info) < CURLINFO_LONG) + +/* evaluates to true if info expects a pointer to long argument */ +#define _curl_is_long_info(info) \ + (CURLINFO_LONG < (info) && (info) < CURLINFO_DOUBLE) + +/* evaluates to true if info expects a pointer to double argument */ +#define _curl_is_double_info(info) \ + (CURLINFO_DOUBLE < (info) && (info) < CURLINFO_SLIST) + +/* true if info expects a pointer to struct curl_slist * argument */ +#define _curl_is_slist_info(info) \ + (((info) == CURLINFO_SSL_ENGINES) || ((info) == CURLINFO_COOKIELIST)) + +/* true if info expects a pointer to struct curl_tlssessioninfo * argument */ +#define _curl_is_tlssessioninfo_info(info) \ + (((info) == CURLINFO_TLS_SSL_PTR) || ((info) == CURLINFO_TLS_SESSION)) + +/* true if info expects a pointer to struct curl_certinfo * argument */ +#define _curl_is_certinfo_info(info) ((info) == CURLINFO_CERTINFO) + +/* true if info expects a pointer to struct curl_socket_t argument */ +#define _curl_is_socket_info(info) \ + (CURLINFO_SOCKET < (info) && (info) < CURLINFO_OFF_T) + +/* true if info expects a pointer to curl_off_t argument */ +#define _curl_is_off_t_info(info) \ + (CURLINFO_OFF_T < (info)) + + +/* typecheck helpers -- check whether given expression has requested type*/ + +/* For pointers, you can use the _curl_is_ptr/_curl_is_arr macros, + * otherwise define a new macro. Search for __builtin_types_compatible_p + * in the GCC manual. + * NOTE: these macros MUST NOT EVALUATE their arguments! The argument is + * the actual expression passed to the curl_easy_setopt macro. This + * means that you can only apply the sizeof and __typeof__ operators, no + * == or whatsoever. + */ + +/* XXX: should evaluate to true iff expr is a pointer */ +#define _curl_is_any_ptr(expr) \ + (sizeof(expr) == sizeof(void *)) + +/* evaluates to true if expr is NULL */ +/* XXX: must not evaluate expr, so this check is not accurate */ +#define _curl_is_NULL(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), __typeof__(NULL))) + +/* evaluates to true if expr is type*, const type* or NULL */ +#define _curl_is_ptr(expr, type) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), type *) || \ + __builtin_types_compatible_p(__typeof__(expr), const type *)) + +/* evaluates to true if expr is one of type[], type*, NULL or const type* */ +#define _curl_is_arr(expr, type) \ + (_curl_is_ptr((expr), type) || \ + __builtin_types_compatible_p(__typeof__(expr), type [])) + +/* evaluates to true if expr is a string */ +#define _curl_is_string(expr) \ + (_curl_is_arr((expr), char) || \ + _curl_is_arr((expr), signed char) || \ + _curl_is_arr((expr), unsigned char)) + +/* evaluates to true if expr is a long (no matter the signedness) + * XXX: for now, int is also accepted (and therefore short and char, which + * are promoted to int when passed to a variadic function) */ +#define _curl_is_long(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), long) || \ + __builtin_types_compatible_p(__typeof__(expr), signed long) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned long) || \ + __builtin_types_compatible_p(__typeof__(expr), int) || \ + __builtin_types_compatible_p(__typeof__(expr), signed int) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned int) || \ + __builtin_types_compatible_p(__typeof__(expr), short) || \ + __builtin_types_compatible_p(__typeof__(expr), signed short) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned short) || \ + __builtin_types_compatible_p(__typeof__(expr), char) || \ + __builtin_types_compatible_p(__typeof__(expr), signed char) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned char)) + +/* evaluates to true if expr is of type curl_off_t */ +#define _curl_is_off_t(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), curl_off_t)) + +/* evaluates to true if expr is abuffer suitable for CURLOPT_ERRORBUFFER */ +/* XXX: also check size of an char[] array? */ +#define _curl_is_error_buffer(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), char *) || \ + __builtin_types_compatible_p(__typeof__(expr), char[])) + +/* evaluates to true if expr is of type (const) void* or (const) FILE* */ +#if 0 +#define _curl_is_cb_data(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_ptr((expr), FILE)) +#else /* be less strict */ +#define _curl_is_cb_data(expr) \ + _curl_is_any_ptr(expr) +#endif + +/* evaluates to true if expr is of type FILE* */ +#define _curl_is_FILE(expr) \ + (_curl_is_NULL(expr) || \ + (__builtin_types_compatible_p(__typeof__(expr), FILE *))) + +/* evaluates to true if expr can be passed as POST data (void* or char*) */ +#define _curl_is_postfields(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_arr((expr), char)) + +/* FIXME: the whole callback checking is messy... + * The idea is to tolerate char vs. void and const vs. not const + * pointers in arguments at least + */ +/* helper: __builtin_types_compatible_p distinguishes between functions and + * function pointers, hide it */ +#define _curl_callback_compatible(func, type) \ + (__builtin_types_compatible_p(__typeof__(func), type) || \ + __builtin_types_compatible_p(__typeof__(func) *, type)) + +/* evaluates to true if expr is of type curl_read_callback or "similar" */ +#define _curl_is_read_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), __typeof__(fread) *) || \ + _curl_callback_compatible((expr), curl_read_callback) || \ + _curl_callback_compatible((expr), _curl_read_callback1) || \ + _curl_callback_compatible((expr), _curl_read_callback2) || \ + _curl_callback_compatible((expr), _curl_read_callback3) || \ + _curl_callback_compatible((expr), _curl_read_callback4) || \ + _curl_callback_compatible((expr), _curl_read_callback5) || \ + _curl_callback_compatible((expr), _curl_read_callback6)) +typedef size_t (*_curl_read_callback1)(char *, size_t, size_t, void *); +typedef size_t (*_curl_read_callback2)(char *, size_t, size_t, const void *); +typedef size_t (*_curl_read_callback3)(char *, size_t, size_t, FILE *); +typedef size_t (*_curl_read_callback4)(void *, size_t, size_t, void *); +typedef size_t (*_curl_read_callback5)(void *, size_t, size_t, const void *); +typedef size_t (*_curl_read_callback6)(void *, size_t, size_t, FILE *); + +/* evaluates to true if expr is of type curl_write_callback or "similar" */ +#define _curl_is_write_cb(expr) \ + (_curl_is_read_cb(expr) || \ + _curl_callback_compatible((expr), __typeof__(fwrite) *) || \ + _curl_callback_compatible((expr), curl_write_callback) || \ + _curl_callback_compatible((expr), _curl_write_callback1) || \ + _curl_callback_compatible((expr), _curl_write_callback2) || \ + _curl_callback_compatible((expr), _curl_write_callback3) || \ + _curl_callback_compatible((expr), _curl_write_callback4) || \ + _curl_callback_compatible((expr), _curl_write_callback5) || \ + _curl_callback_compatible((expr), _curl_write_callback6)) +typedef size_t (*_curl_write_callback1)(const char *, size_t, size_t, void *); +typedef size_t (*_curl_write_callback2)(const char *, size_t, size_t, + const void *); +typedef size_t (*_curl_write_callback3)(const char *, size_t, size_t, FILE *); +typedef size_t (*_curl_write_callback4)(const void *, size_t, size_t, void *); +typedef size_t (*_curl_write_callback5)(const void *, size_t, size_t, + const void *); +typedef size_t (*_curl_write_callback6)(const void *, size_t, size_t, FILE *); + +/* evaluates to true if expr is of type curl_ioctl_callback or "similar" */ +#define _curl_is_ioctl_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_ioctl_callback) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback1) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback2) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback3) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback4)) +typedef curlioerr (*_curl_ioctl_callback1)(CURL *, int, void *); +typedef curlioerr (*_curl_ioctl_callback2)(CURL *, int, const void *); +typedef curlioerr (*_curl_ioctl_callback3)(CURL *, curliocmd, void *); +typedef curlioerr (*_curl_ioctl_callback4)(CURL *, curliocmd, const void *); + +/* evaluates to true if expr is of type curl_sockopt_callback or "similar" */ +#define _curl_is_sockopt_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_sockopt_callback) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback1) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback2)) +typedef int (*_curl_sockopt_callback1)(void *, curl_socket_t, curlsocktype); +typedef int (*_curl_sockopt_callback2)(const void *, curl_socket_t, + curlsocktype); + +/* evaluates to true if expr is of type curl_opensocket_callback or + "similar" */ +#define _curl_is_opensocket_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_opensocket_callback) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback1) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback2) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback3) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback4)) +typedef curl_socket_t (*_curl_opensocket_callback1) + (void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (*_curl_opensocket_callback2) + (void *, curlsocktype, const struct curl_sockaddr *); +typedef curl_socket_t (*_curl_opensocket_callback3) + (const void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (*_curl_opensocket_callback4) + (const void *, curlsocktype, const struct curl_sockaddr *); + +/* evaluates to true if expr is of type curl_progress_callback or "similar" */ +#define _curl_is_progress_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_progress_callback) || \ + _curl_callback_compatible((expr), _curl_progress_callback1) || \ + _curl_callback_compatible((expr), _curl_progress_callback2)) +typedef int (*_curl_progress_callback1)(void *, + double, double, double, double); +typedef int (*_curl_progress_callback2)(const void *, + double, double, double, double); + +/* evaluates to true if expr is of type curl_debug_callback or "similar" */ +#define _curl_is_debug_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_debug_callback) || \ + _curl_callback_compatible((expr), _curl_debug_callback1) || \ + _curl_callback_compatible((expr), _curl_debug_callback2) || \ + _curl_callback_compatible((expr), _curl_debug_callback3) || \ + _curl_callback_compatible((expr), _curl_debug_callback4) || \ + _curl_callback_compatible((expr), _curl_debug_callback5) || \ + _curl_callback_compatible((expr), _curl_debug_callback6) || \ + _curl_callback_compatible((expr), _curl_debug_callback7) || \ + _curl_callback_compatible((expr), _curl_debug_callback8)) +typedef int (*_curl_debug_callback1) (CURL *, + curl_infotype, char *, size_t, void *); +typedef int (*_curl_debug_callback2) (CURL *, + curl_infotype, char *, size_t, const void *); +typedef int (*_curl_debug_callback3) (CURL *, + curl_infotype, const char *, size_t, void *); +typedef int (*_curl_debug_callback4) (CURL *, + curl_infotype, const char *, size_t, const void *); +typedef int (*_curl_debug_callback5) (CURL *, + curl_infotype, unsigned char *, size_t, void *); +typedef int (*_curl_debug_callback6) (CURL *, + curl_infotype, unsigned char *, size_t, const void *); +typedef int (*_curl_debug_callback7) (CURL *, + curl_infotype, const unsigned char *, size_t, void *); +typedef int (*_curl_debug_callback8) (CURL *, + curl_infotype, const unsigned char *, size_t, const void *); + +/* evaluates to true if expr is of type curl_ssl_ctx_callback or "similar" */ +/* this is getting even messier... */ +#define _curl_is_ssl_ctx_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_ssl_ctx_callback) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback1) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback2) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback3) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback4) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback5) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback6) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback7) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback8)) +typedef CURLcode (*_curl_ssl_ctx_callback1)(CURL *, void *, void *); +typedef CURLcode (*_curl_ssl_ctx_callback2)(CURL *, void *, const void *); +typedef CURLcode (*_curl_ssl_ctx_callback3)(CURL *, const void *, void *); +typedef CURLcode (*_curl_ssl_ctx_callback4)(CURL *, const void *, + const void *); +#ifdef HEADER_SSL_H +/* hack: if we included OpenSSL's ssl.h, we know about SSL_CTX + * this will of course break if we're included before OpenSSL headers... + */ +typedef CURLcode (*_curl_ssl_ctx_callback5)(CURL *, SSL_CTX, void *); +typedef CURLcode (*_curl_ssl_ctx_callback6)(CURL *, SSL_CTX, const void *); +typedef CURLcode (*_curl_ssl_ctx_callback7)(CURL *, const SSL_CTX, void *); +typedef CURLcode (*_curl_ssl_ctx_callback8)(CURL *, const SSL_CTX, + const void *); +#else +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback5; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback6; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback7; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback8; +#endif + +/* evaluates to true if expr is of type curl_conv_callback or "similar" */ +#define _curl_is_conv_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_conv_callback) || \ + _curl_callback_compatible((expr), _curl_conv_callback1) || \ + _curl_callback_compatible((expr), _curl_conv_callback2) || \ + _curl_callback_compatible((expr), _curl_conv_callback3) || \ + _curl_callback_compatible((expr), _curl_conv_callback4)) +typedef CURLcode (*_curl_conv_callback1)(char *, size_t length); +typedef CURLcode (*_curl_conv_callback2)(const char *, size_t length); +typedef CURLcode (*_curl_conv_callback3)(void *, size_t length); +typedef CURLcode (*_curl_conv_callback4)(const void *, size_t length); + +/* evaluates to true if expr is of type curl_seek_callback or "similar" */ +#define _curl_is_seek_cb(expr) \ + (_curl_is_NULL(expr) || \ + _curl_callback_compatible((expr), curl_seek_callback) || \ + _curl_callback_compatible((expr), _curl_seek_callback1) || \ + _curl_callback_compatible((expr), _curl_seek_callback2)) +typedef CURLcode (*_curl_seek_callback1)(void *, curl_off_t, int); +typedef CURLcode (*_curl_seek_callback2)(const void *, curl_off_t, int); + + +#endif /* __CURL_TYPECHECK_GCC_H */ diff --git a/ext/curl-7.58.0/x64/lib/libcurl_a.lib b/ext/curl-7.58.0/x64/lib/libcurl_a.lib Binary files differnew file mode 100644 index 00000000..7efe65b1 --- /dev/null +++ b/ext/curl-7.58.0/x64/lib/libcurl_a.lib diff --git a/ext/curl-7.58.0/x64/lib/libcurl_a_debug.lib b/ext/curl-7.58.0/x64/lib/libcurl_a_debug.lib Binary files differnew file mode 100644 index 00000000..867964b2 --- /dev/null +++ b/ext/curl-7.58.0/x64/lib/libcurl_a_debug.lib diff --git a/ext/ed25519-amd64-asm/sign.c b/ext/ed25519-amd64-asm/sign.c index 958e4a14..882ae76f 100644 --- a/ext/ed25519-amd64-asm/sign.c +++ b/ext/ed25519-amd64-asm/sign.c @@ -124,18 +124,15 @@ void get_hram(unsigned char *hram, const unsigned char *sm, const unsigned char extern void ZT_sha512internal(void *digest,const void *data,unsigned int len); -extern void ed25519_amd64_asm_sign(const unsigned char *sk,const unsigned char *pk,const unsigned char *m,const unsigned int mlen,unsigned char *sig) +extern void ed25519_amd64_asm_sign(const unsigned char *sk,const unsigned char *pk,const unsigned char *digest,unsigned char *sig) { unsigned char az[64]; unsigned char nonce[64]; unsigned char hram[64]; sc25519 sck, scs, scsk; ge25519 ger; - unsigned char digest[64]; unsigned int i; - ZT_sha512internal(digest,m,mlen); - ZT_sha512internal(az,sk,32); az[0] &= 248; az[31] &= 127; diff --git a/ext/installfiles/mac/ZeroTier One.pkgproj b/ext/installfiles/mac/ZeroTier One.pkgproj index 0d3d0bda..71c2f2db 100755 --- a/ext/installfiles/mac/ZeroTier One.pkgproj +++ b/ext/installfiles/mac/ZeroTier One.pkgproj @@ -109,9 +109,9 @@ <key>CHILDREN</key> <array/> <key>GID</key> - <integer>0</integer> + <integer>80</integer> <key>PATH</key> - <string>../../bin/tap-mac/tap.kext</string> + <string>../../../MacEthernetTapAgent</string> <key>PATH_TYPE</key> <integer>1</integer> <key>PERMISSIONS</key> @@ -664,7 +664,7 @@ <key>USE_HFS+_COMPRESSION</key> <false/> <key>VERSION</key> - <string>1.2.12</string> + <string>1.2.99</string> </dict> <key>PROJECT_COMMENTS</key> <dict> diff --git a/ext/installfiles/mac/uninstall.sh b/ext/installfiles/mac/uninstall.sh index 9bf5d6fc..52c09b63 100755 --- a/ext/installfiles/mac/uninstall.sh +++ b/ext/installfiles/mac/uninstall.sh @@ -27,7 +27,7 @@ kextunload '/Library/Application Support/ZeroTier/One/tap.kext' >>/dev/null 2>&1 echo "Removing ZeroTier One files..." rm -rf '/Applications/ZeroTier One.app' -rm -f '/usr/bin/zerotier-one' '/usr/bin/zerotier-idtool' '/usr/bin/zerotier-cli' '/Library/LaunchDaemons/com.zerotier.one.plist' +rm -f '/usr/local/bin/zerotier-one' '/usr/local/bin/zerotier-idtool' '/usr/local/bin/zerotier-cli' '/Library/LaunchDaemons/com.zerotier.one.plist' cd '/Library/Application Support/ZeroTier/One' if [ "`pwd`" = '/Library/Application Support/ZeroTier/One' ]; then diff --git a/ext/installfiles/windows/ZeroTier One.aip b/ext/installfiles/windows/ZeroTier One.aip index de973bf1..df9a5064 100644 --- a/ext/installfiles/windows/ZeroTier One.aip +++ b/ext/installfiles/windows/ZeroTier One.aip @@ -27,10 +27,10 @@ <ROW Property="CTRLS" Value="2"/>
<ROW Property="MSIFASTINSTALL" MultiBuildValue="DefaultBuild:2"/>
<ROW Property="Manufacturer" Value="ZeroTier, Inc."/>
- <ROW Property="ProductCode" Value="1033:{855E8629-580C-4BDF-8B59-B9290C7E7BA5} " Type="16"/>
+ <ROW Property="ProductCode" Value="1033:{FF2B4EDA-7B30-41A3-8A3F-BF81E1085234} " Type="16"/>
<ROW Property="ProductLanguage" Value="1033"/>
<ROW Property="ProductName" Value="ZeroTier One"/>
- <ROW Property="ProductVersion" Value="1.2.12" Type="32"/>
+ <ROW Property="ProductVersion" Value="1.2.99" Type="32"/>
<ROW Property="REBOOT" MultiBuildValue="DefaultBuild:ReallySuppress"/>
<ROW Property="RUNAPPLICATION" Value="1" Type="4"/>
<ROW Property="SecureCustomProperties" Value="OLDPRODUCTS;AI_NEWERPRODUCTFOUND;AI_SETUPEXEPATH;SETUPEXEDIR"/>
@@ -64,7 +64,7 @@ <ROW Directory="x86_Dir" Directory_Parent="tapwindows_Dir" DefaultDir="x86"/>
</COMPONENT>
<COMPONENT cid="caphyon.advinst.msicomp.MsiCompsComponent">
- <ROW Component="AI_CustomARPName" ComponentId="{92D9A995-E340-41B2-98F5-F2DB3F6E8AD8}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
+ <ROW Component="AI_CustomARPName" ComponentId="{6CAB31CD-CEDA-4B74-A52D-3D35D4A426D8}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
<ROW Component="AI_DisableModify" ComponentId="{020DCABD-5D56-49B9-AF48-F07F0B55E590}" Directory_="APPDIR" Attributes="4" KeyPath="NoModify" Options="1"/>
<ROW Component="AI_ExePath" ComponentId="{8E02B36C-7A19-429B-A93E-77A9261AC918}" Directory_="APPDIR" Attributes="4" KeyPath="AI_ExePath"/>
<ROW Component="Hardcodet.Wpf.TaskbarNotification.dll" ComponentId="{BEA825AF-2555-44AF-BE40-47FFC16DCBA6}" Directory_="APPDIR" Attributes="0" KeyPath="Hardcodet.Wpf.TaskbarNotification.dll"/>
@@ -454,7 +454,7 @@ <ROW XmlAttribute="xsischemaLocation" XmlElement="swidsoftware_identification_tag" Name="xsi:schemaLocation" Flags="14" Order="3" Value="http://standards.iso.org/iso/19770/-2/2008/schema.xsd software_identification_tag.xsd"/>
</COMPONENT>
<COMPONENT cid="caphyon.advinst.msicomp.XmlElementComponent">
- <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="12"/>
+ <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="99"/>
<ROW XmlElement="swidentitlement_required_indicator" ParentElement="swidsoftware_identification_tag" Name="swid:entitlement_required_indicator" Condition="1" Order="0" Flags="14" Text="false"/>
<ROW XmlElement="swidmajor" ParentElement="swidnumeric" Name="swid:major" Condition="1" Order="0" Flags="14" Text="1"/>
<ROW XmlElement="swidminor" ParentElement="swidnumeric" Name="swid:minor" Condition="1" Order="1" Flags="14" Text="2"/>
diff --git a/ext/installfiles/windows/chocolatey/zerotier-one/tools/LICENSE.txt b/ext/installfiles/windows/chocolatey/zerotier-one/tools/LICENSE.txt deleted file mode 100644 index ce0564ac..00000000 --- a/ext/installfiles/windows/chocolatey/zerotier-one/tools/LICENSE.txt +++ /dev/null @@ -1,11 +0,0 @@ -From: https://raw.githubusercontent.com/zerotier/ZeroTierOne/master/COPYING - -LICENSE - -ZeroTier One is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or (at -your option) any later version. - -See the file ‘LICENSE.GPL-3’ for the text of the GNU GPL version 3. -If that file is not present, see <http://www.gnu.org/licenses/>. diff --git a/ext/installfiles/windows/chocolatey/zerotier-one/tools/VERIFICATION.txt b/ext/installfiles/windows/chocolatey/zerotier-one/tools/VERIFICATION.txt deleted file mode 100644 index 0a5bc760..00000000 --- a/ext/installfiles/windows/chocolatey/zerotier-one/tools/VERIFICATION.txt +++ /dev/null @@ -1,5 +0,0 @@ -VERIFICATION -Verification is intended to assist the Chocolatey moderators and community -in verifying that this package's contents are trustworthy. - -Our MSI installer should be signed by ZeroTier, Inc. using a certificate from DigiCert. diff --git a/ext/installfiles/windows/chocolatey/zerotier-one/tools/chocolateyinstall.ps1 b/ext/installfiles/windows/chocolatey/zerotier-one/tools/chocolateyinstall.ps1 deleted file mode 100644 index 9a0f6bac..00000000 --- a/ext/installfiles/windows/chocolatey/zerotier-one/tools/chocolateyinstall.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -$packageName = 'zerotier-one' -$installerType = 'msi' -$url = 'https://download.zerotier.com/RELEASES/1.2.4/dist/ZeroTier%20One.msi' -$url64 = 'https://download.zerotier.com/RELEASES/1.2.4/dist/ZeroTier%20One.msi' -$silentArgs = '/quiet' -$validExitCodes = @(0,3010) - -Install-ChocolateyPackage $packageName $installerType $silentArgs $url $url64 -validExitCodes $validExitCodes diff --git a/ext/installfiles/windows/chocolatey/zerotier-one/tools/chocolateyuninstall.ps1 b/ext/installfiles/windows/chocolatey/zerotier-one/tools/chocolateyuninstall.ps1 deleted file mode 100644 index 81f7a5ac..00000000 --- a/ext/installfiles/windows/chocolatey/zerotier-one/tools/chocolateyuninstall.ps1 +++ /dev/null @@ -1,30 +0,0 @@ -$ErrorActionPreference = 'Stop'; - -$packageName = 'zerotier-one' -$softwareName = 'ZeroTier One*' -$installerType = 'MSI' - -$silentArgs = '/qn /norestart' -$validExitCodes = @(0, 3010, 1605, 1614, 1641) -$uninstalled = $false - -[array]$key = Get-UninstallRegistryKey -SoftwareName $softwareName - -if ($key.Count -eq 1) { - $key | % { - $silentArgs = "$($_.PSChildName) $silentArgs" - $file = '' - Uninstall-ChocolateyPackage -PackageName $packageName ` - -FileType $installerType ` - -SilentArgs "$silentArgs" ` - -ValidExitCodes $validExitCodes ` - -File "$file" - } -} elseif ($key.Count -eq 0) { - Write-Warning "$packageName has already been uninstalled by other means." -} elseif ($key.Count -gt 1) { - Write-Warning "$key.Count matches found!" - Write-Warning "To prevent accidental data loss, no programs will be uninstalled." - Write-Warning "Please alert package maintainer the following keys were matched:" - $key | % {Write-Warning "- $_.DisplayName"} -} diff --git a/ext/installfiles/windows/chocolatey/zerotier-one/zerotier-one.nuspec b/ext/installfiles/windows/chocolatey/zerotier-one/zerotier-one.nuspec deleted file mode 100644 index 2fb4fe57..00000000 --- a/ext/installfiles/windows/chocolatey/zerotier-one/zerotier-one.nuspec +++ /dev/null @@ -1,76 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Read this before creating packages: https://github.com/chocolatey/chocolatey/wiki/CreatePackages --> -<!-- It is especially important to read the above link to understand additional requirements when publishing packages to the community feed aka dot org (https://chocolatey.org/packages). --> - -<!-- Test your packages in a test environment: https://github.com/chocolatey/chocolatey-test-environment --> - -<!-- -This is a nuspec. It mostly adheres to https://docs.nuget.org/create/Nuspec-Reference. Chocolatey uses a special version of NuGet.Core that allows us to do more than was initially possible. As such there are certain things to be aware of: - -* the package xmlns schema url may cause issues with nuget.exe -* Any of the following elements can ONLY be used by choco tools - projectSourceUrl, docsUrl, mailingListUrl, bugTrackerUrl, packageSourceUrl, provides, conflicts, replaces -* nuget.exe can still install packages with those elements but they are ignored. Any authoring tools or commands will error on those elements ---> - -<!-- You can embed software files directly into packages, as long as you are not bound by distribution rights. --> -<!-- * If you are an organization making private packages, you probably have no issues here --> -<!-- * If you are releasing to the community feed, you need to consider distribution rights. --> -<!-- Do not remove this test for UTF-8: if “Ω” doesn’t appear as greek uppercase omega letter enclosed in quotation marks, you should use an editor that supports UTF-8, not this one. --> -<package xmlns="http://schemas.microsoft.com/packaging/2015/06/nuspec.xsd"> - <metadata> - <!-- == PACKAGE SPECIFIC SECTION == --> - <!-- This section is about this package, although id and version have ties back to the software --> - <!-- id is lowercase and if you want a good separator for words, use '-', not '.'. Dots are only acceptable as suffixes for certain types of packages, e.g. .install, .portable, .extension, .template --> - <!-- If the software is cross-platform, attempt to use the same id as the debian/rpm package(s) if possible. --> - <id>zerotier-one</id> - <!-- version should MATCH as closely as possible with the underlying software --> - <!-- Is the version a prerelease of a version? https://docs.nuget.org/create/versioning#creating-prerelease-packages --> - <!-- Note that unstable versions like 0.0.1 can be considered a released version, but it's possible that one can release a 0.0.1-beta before you release a 0.0.1 version. If the version number is final, that is considered a released version and not a prerelease. --> - <version>1.2.12</version> - <!-- <packageSourceUrl>Where is this Chocolatey package located (think GitHub)? packageSourceUrl is highly recommended for the community feed</packageSourceUrl>--> - <!-- owners is a poor name for maintainers of the package. It sticks around by this name for compatibility reasons. It basically means you. --> - <!--<owners>ZeroTier, Inc.</owners>--> - <!-- ============================== --> - - <!-- == SOFTWARE SPECIFIC SECTION == --> - <!-- This section is about the software itself --> - <title>zerotier-one (Install)</title> - <authors>ZeroTier, Inc.</authors> - <!-- projectUrl is required for the community feed --> - <projectUrl>https://www.zerotier.com/</projectUrl> - <!--<iconUrl>https://www.zerotier.com/img/ZeroTierIcon.png</iconUrl>--> - <!-- <copyright>2011-2016 ZeroTier, Inc.</copyright> --> - <!-- If there is a license Url available, it is is required for the community feed --> - <!-- <licenseUrl>https://raw.githubusercontent.com/zerotier/ZeroTierOne/master/COPYING</licenseUrl> - <requireLicenseAcceptance>true</requireLicenseAcceptance>--> - <!--<projectSourceUrl>https://github.com/zerotier/ZeroTierOne</projectSourceUrl>--> - <!--<docsUrl>https://www.zerotier.com/</docsUrl>--> - <!--<mailingListUrl></mailingListUrl>--> - <!--<bugTrackerUrl>https://github.com/zerotier/ZeroTierOne/issues</bugTrackerUrl>--> - <tags>zerotier-one admin</tags> - <summary>ZeroTier One Virtual Network Endpoint for Windows</summary> - <description>ZeroTier is a smart switch for Earth with VLAN capability. See https://www.zerotier.com/ for more information.</description> - <!-- <releaseNotes>__REPLACE_OR_REMOVE__MarkDown_Okay</releaseNotes> --> - <!-- =============================== --> - - <!-- Specifying dependencies and version ranges? https://docs.nuget.org/create/versioning#specifying-version-ranges-in-.nuspec-files --> - <!--<dependencies> - <dependency id="" version="__MINIMUM_VERSION__" /> - <dependency id="" version="[__EXACT_VERSION__]" /> - <dependency id="" version="[_MIN_VERSION_INCLUSIVE, MAX_VERSION_INCLUSIVE]" /> - <dependency id="" version="[_MIN_VERSION_INCLUSIVE, MAX_VERSION_EXCLUSIVE)" /> - <dependency id="" /> - <dependency id="chocolatey-uninstall.extension" /> - </dependencies>--> - <!-- chocolatey-uninstall.extension - If supporting 0.9.9.x (or below) and including a chocolateyUninstall.ps1 file to uninstall an EXE/MSI, you probably want to include chocolatey-uninstall.extension as a dependency. Please verify whether you are using a helper function from that package. --> - - <!--<provides>NOT YET IMPLEMENTED</provides>--> - <!--<conflicts>NOT YET IMPLEMENTED</conflicts>--> - <!--<replaces>NOT YET IMPLEMENTED</replaces>--> - </metadata> - <files> - <!-- this section controls what actually gets packaged into the Chocolatey package --> - <file src="tools\**" target="tools" /> - <!--Building from Linux? You may need this instead: <file src="tools/**" target="tools" />--> - </files> -</package> diff --git a/ext/json/README.md b/ext/json/README.md index 256409e3..f3bbf847 100644 --- a/ext/json/README.md +++ b/ext/json/README.md @@ -5,11 +5,11 @@ [](https://coveralls.io/r/nlohmann/json) [](https://scan.coverity.com/projects/nlohmann-json) [](https://www.codacy.com/app/nlohmann/json?utm_source=github.com&utm_medium=referral&utm_content=nlohmann/json&utm_campaign=Badge_Grade) -[](https://wandbox.org/permlink/Op57X0V7fTf2tdwl) +[](https://wandbox.org/permlink/TarF5pPn9NtHQjhf) [](http://nlohmann.github.io/json) [](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) -[](https://github.com/nlohmann/json/releases) -[](http://github.com/nlohmann/json/issues) +[](https://github.com/nlohmann/json/releases) +[](http://github.com/nlohmann/json/issues) [](http://isitmaintained.com/project/nlohmann/json "Average time to resolve an issue") [](https://bestpractices.coreinfrastructure.org/projects/289) @@ -21,9 +21,10 @@ - [STL-like access](#stl-like-access) - [Conversion from STL containers](#conversion-from-stl-containers) - [JSON Pointer and JSON Patch](#json-pointer-and-json-patch) + - [JSON Merge Patch](#json-merge-patch) - [Implicit conversions](#implicit-conversions) - [Conversions to/from arbitrary types](#arbitrary-types-conversions) - - [Binary formats (CBOR and MessagePack)](#binary-formats-cbor-and-messagepack) + - [Binary formats (CBOR, MessagePack, and UBJSON)](#binary-formats-cbor-messagepack-and-ubjson) - [Supported compilers](#supported-compilers) - [License](#license) - [Contact](#contact) @@ -39,9 +40,9 @@ There are myriads of [JSON](http://json.org) libraries out there, and each may e - **Intuitive syntax**. In languages such as Python, JSON feels like a first class data type. We used all the operator magic of modern C++ to achieve the same feeling in your code. Check out the [examples below](#examples) and you'll know what I mean. -- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/src/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. +- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/single_include/nlohmann/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. -- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/blob/master/test/src/unit.cpp) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) that there are no memory leaks. To maintain high quality, the project is following the [Core Infrastructure Initiative (CII) best practices](https://bestpractices.coreinfrastructure.org/projects/289). +- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/tree/develop/test/src) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) and the [Clang Sanitizers](https://clang.llvm.org/docs/index.html) that there are no memory leaks. [Google OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/json) additionally runs fuzz tests agains all parsers 24/7, effectively executing billions of tests so far. To maintain high quality, the project is following the [Core Infrastructure Initiative (CII) best practices](https://bestpractices.coreinfrastructure.org/projects/289). Other aspects were not so important to us: @@ -54,31 +55,42 @@ See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/. ## Integration -The single required source, file `json.hpp` is in the `src` directory or [released here](https://github.com/nlohmann/json/releases). All you need to do is add +[`json.hpp`](https://github.com/nlohmann/json/blob/develop/single_include/nlohmann/json.hpp) is the single required file in `single_include/nlohmann` or [released here](https://github.com/nlohmann/json/releases). You need to add ```cpp -#include "json.hpp" +#include <nlohmann/json.hpp> // for convenience using json = nlohmann::json; ``` -to the files you want to use JSON objects. That's it. Do not forget to set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). +to the files you want to process JSON and set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). + +You can further use file [`include/nlohmann/json_fwd.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/json_fwd.hpp) for forward-declarations. The installation of json_fwd.hpp (as part of cmake's install step), can be achieved by setting `-DJSON_MultipleHeaders=ON`. + +### Package Managers :beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. -If you are using the [Meson Build System](http://mesonbuild.com), then you can wrap this repo as a subproject. +If you are using the [Meson Build System](http://mesonbuild.com), then you can wrap this repository as a subproject. If you are using [Conan](https://www.conan.io/) to manage your dependencies, merely add `jsonformoderncpp/x.y.z@vthiery/stable` to your `conanfile.py`'s requires, where `x.y.z` is the release version you want to use. Please file issues [here](https://github.com/vthiery/conan-jsonformoderncpp/issues) if you experience problems with the packages. +If you are using [Spack](https://www.spack.io/) to manage your dependencies, you can use the `nlohmann_json` package. Please see the [spack project](https://github.com/spack/spack) for any issues regarding the packaging. + If you are using [hunter](https://github.com/ruslo/hunter/) on your project for external dependencies, then you can use the [nlohmann_json package](https://docs.hunter.sh/en/latest/packages/pkg/nlohmann_json.html). Please see the hunter project for any issues regarding the packaging. +If you are using [Buckaroo](https://buckaroo.pm), you can install this library's module with `buckaroo install nlohmann/json`. Please file issues [here](https://github.com/LoopPerfect/buckaroo-recipes/issues/new?title=nlohmann/nlohmann/json). + If you are using [vcpkg](https://github.com/Microsoft/vcpkg/) on your project for external dependencies, then you can use the [nlohmann-json package](https://github.com/Microsoft/vcpkg/tree/master/ports/nlohmann-json). Please see the vcpkg project for any issues regarding the packaging. +If you are using [cget](http://cget.readthedocs.io/en/latest/), you can install the latest development version with `cget install nlohmann/json`. A specific version can be installed with `cget install nlohmann/json@v3.1.0`. Also, the multiple header version can be installed by adding the `-DJSON_MultipleHeaders=ON` flag (i.e., `cget install nlohmann/json -DJSON_MultipleHeaders=ON`). + +If you are using [CocoaPods](https://cocoapods.org), you can use the library by adding pod `"nlohmann_json", '~>3.1.2'` to your podfile (see [an example](https://bitbucket.org/benman/nlohmann_json-cocoapod/src/master/)). Please file issues [here](https://bitbucket.org/benman/nlohmann_json-cocoapod/issues?status=new&status=open). ## Examples -Beside the examples below, you may want to check the [documentation](https://nlohmann.github.io/json/) where each function contains a separate code example (e.g., check out [`emplace()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a602f275f0359ab181221384989810604.html#a602f275f0359ab181221384989810604)). All [example files](https://github.com/nlohmann/json/tree/develop/doc/examples) can be compiled and executed on their own (e.g., file [emplace.cpp](https://github.com/nlohmann/json/blob/develop/doc/examples/emplace.cpp)). +Beside the examples below, you may want to check the [documentation](https://nlohmann.github.io/json/) where each function contains a separate code example (e.g., check out [`emplace()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a5338e282d1d02bed389d852dd670d98d.html#a5338e282d1d02bed389d852dd670d98d)). All [example files](https://github.com/nlohmann/json/tree/develop/doc/examples) can be compiled and executed on their own (e.g., file [emplace.cpp](https://github.com/nlohmann/json/blob/develop/doc/examples/emplace.cpp)). ### JSON as first-class data type @@ -103,7 +115,7 @@ Assume you want to create the JSON object } ``` -With the JSON class, you could write: +With this library, you could write: ```cpp // create an empty structure (null) @@ -147,7 +159,7 @@ json j2 = { }; ``` -Note that in all these cases, you never need to "tell" the compiler which JSON value you want to use. If you want to be explicit or express some edge cases, the functions `json::array` and `json::object` will help: +Note that in all these cases, you never need to "tell" the compiler which JSON value type you want to use. If you want to be explicit or express some edge cases, the functions [`json::array`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_aa80485befaffcadaa39965494e0b4d2e.html#aa80485befaffcadaa39965494e0b4d2e) and [`json::object`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_aa13f7c0615867542ce80337cbcf13ada.html#aa13f7c0615867542ce80337cbcf13ada) will help: ```cpp // a way to express the empty array [] @@ -161,12 +173,11 @@ json empty_object_explicit = json::object(); json array_not_object = json::array({ {"currency", "USD"}, {"value", 42.99} }); ``` - ### Serialization / Deserialization #### To/from strings -You can create an object (deserialization) by appending `_json` to a string literal: +You can create a JSON value (deserialization) by appending `_json` to a string literal: ```cpp // create object from string literal @@ -183,14 +194,14 @@ auto j2 = R"( Note that without appending the `_json` suffix, the passed string literal is not parsed, but just used as JSON string value. That is, `json j = "{ \"happy\": true, \"pi\": 3.141 }"` would just store the string `"{ "happy": true, "pi": 3.141 }"` rather than parsing the actual object. -The above example can also be expressed explicitly using `json::parse()`: +The above example can also be expressed explicitly using [`json::parse()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_aa9676414f2e36383c4b181fe856aa3c0.html#aa9676414f2e36383c4b181fe856aa3c0): ```cpp // parse explicitly auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); ``` -You can also get a string representation (serialize): +You can also get a string representation of a JSON value (serialize): ```cpp // explicit conversion to string @@ -225,8 +236,9 @@ std::cout << cpp_string << " == " << cpp_string2 << " == " << j_string.get<std:: std::cout << j_string << " == " << serialized_string << std::endl; ``` -`.dump()` always returns the serialized value, and `.get<std::string>()` returns the originally stored string value. +[`.dump()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a5adea76fedba9898d404fef8598aa663.html#a5adea76fedba9898d404fef8598aa663) always returns the serialized value, and [`.get<std::string>()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a16f9445f7629f634221a42b967cdcd43.html#a16f9445f7629f634221a42b967cdcd43) returns the originally stored string value. +Note the library only supports UTF-8. When you store strings with different encodings in the library, calling [`dump()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a5adea76fedba9898d404fef8598aa663.html#a5adea76fedba9898d404fef8598aa663) may throw an exception. #### To/from streams (e.g. files, string streams) @@ -261,7 +273,7 @@ Please note that setting the exception bit for `failbit` is inappropriate for th #### Read from iterator range -You can also read JSON from an iterator range; that is, from any container accessible by iterators whose content is stored as contiguous byte sequence, for instance a `std::vector<std::uint8_t>`: +You can also parse JSON from an iterator range; that is, from any container accessible by iterators whose content is stored as contiguous byte sequence, for instance a `std::vector<std::uint8_t>`: ```cpp std::vector<std::uint8_t> v = {'t', 'r', 'u', 'e'}; @@ -275,10 +287,53 @@ std::vector<std::uint8_t> v = {'t', 'r', 'u', 'e'}; json j = json::parse(v); ``` +#### SAX interface + +The library uses a SAX-like interface with the following functions: + +```cpp +// called when null is parsed +bool null(); + +// called when a boolean is parsed; value is passed +bool boolean(bool val); + +// called when a signed or unsigned integer number is parsed; value is passed +bool number_integer(number_integer_t val); +bool number_unsigned(number_unsigned_t val); + +// called when a floating-point number is parsed; value and original string is passed +bool number_float(number_float_t val, const string_t& s); + +// called when a string is parsed; value is passed and can be safely moved away +bool string(string_t& val); + +// called when an object or array begins or ends, resp. The number of elements is passed (or -1 if not known) +bool start_object(std::size_t elements); +bool end_object(); +bool start_array(std::size_t elements); +bool end_array(); +// called when an object key is parsed; value is passed and can be safely moved away +bool key(string_t& val); + +// called when a parse error occurs; byte position, the last token, and an exception is passed +bool parse_error(std::size_t position, const std::string& last_token, const detail::exception& ex); +``` + +The return value of each function determines whether parsing should proceed. + +To implement your own SAX handler, proceed as follows: + +1. Implement the SAX interface in a class. You can use class `nlohmann::json_sax<json>` as base class, but you can also use any class where the functions described above are implemented and public. +2. Create an object of your SAX interface class, e.g. `my_sax`. +3. Call `bool json::sax_parse(input, &my_sax)`; where the first parameter can be any input like a string or an input stream and the second parameter is a pointer to your SAX interface. + +Note the `sax_parse` function only returns a `bool` indicating the result of the last executed SAX event. It does not return a `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file [`json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp). + ### STL-like access -We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) requirement. +We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirement. ```cpp // create an array using push_back @@ -352,7 +407,7 @@ o.erase("foo"); ### Conversion from STL containers -Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON types (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends on how the elements are ordered in the respective STL container. +Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON values (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends on how the elements are ordered in the respective STL container. ```cpp std::vector<int> c_vector {1, 2, 3, 4}; @@ -392,7 +447,7 @@ json j_umset(c_umset); // both entries for "one" are used // maybe ["one", "two", "one", "four"] ``` -Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON types (see examples above) can be used to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. +Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON values (see examples above) can be used to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. ```cpp std::map<std::string, int> c_map { {"one", 1}, {"two", 2}, {"three", 3} }; @@ -450,6 +505,37 @@ json::diff(j_result, j_original); // ] ``` +### JSON Merge Patch + +The library supports **JSON Merge Patch** ([RFC 7386](https://tools.ietf.org/html/rfc7386)) as a patch format. Instead of using JSON Pointer (see above) to specify values to be manipulated, it describes the changes using a syntax that closely mimics the document being modified. + +```cpp +// a JSON value +json j_document = R"({ + "a": "b", + "c": { + "d": "e", + "f": "g" + } +})"_json; + +// a patch +json j_patch = R"({ + "a":"z", + "c": { + "f": null + } +})"_json; + +// apply the patch +j_original.merge_patch(j_patch); +// { +// "a": "z", +// "c": { +// "d": "e" +// } +// } +``` ### Implicit conversions @@ -484,9 +570,17 @@ int vi = jn.get<int>(); // etc. ``` +Note that `char` types are not automatically converted to JSON strings, but to integer numbers. A conversion to a string must be specified explicitly: + +```cpp +char ch = 'A'; // ASCII value 65 +json j_default = ch; // stores integer number 65 +json j_string = std::string(1, ch); // stores string "A" +``` + ### Arbitrary types conversions -Every type can be serialized in JSON, not just STL-containers and scalar types. Usually, you would do something along those lines: +Every type can be serialized in JSON, not just STL containers and scalar types. Usually, you would do something along those lines: ```cpp namespace ns { @@ -561,7 +655,8 @@ Likewise, when calling `get<your_type>()`, the `from_json` method will be called Some important things: * Those methods **MUST** be in your type's namespace (which can be the global namespace), or the library will not be able to locate them (in this example, they are in namespace `ns`, where `person` is defined). -* When using `get<your_type>()`, `your_type` **MUST** be [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). (There is a way to bypass this requirement described later.) +* Those methods **MUST** be available (e.g., properly headers must be included) everywhere you use the implicit conversions. Look at [issue 1108](https://github.com/nlohmann/json/issues/1108) for errors that may occur otherwise. +* When using `get<your_type>()`, `your_type` **MUST** be [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). (There is a way to bypass this requirement described later.) * In function `from_json`, use function [`at()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a93403e803947b86f4da2d1fb3345cf2c.html#a93403e803947b86f4da2d1fb3345cf2c) to access the object values rather than `operator[]`. In case a key does not exist, `at` throws an exception that you can handle, whereas `operator[]` exhibits undefined behavior. * In case your type contains several `operator=` definitions, code like `your_variable = your_json;` [may not compile](https://github.com/nlohmann/json/issues/667). You need to write `your_variable = your_json.get<decltype your_variable>();` instead. * You do not need to add serializers or deserializers for STL types like `std::vector`: the library already implements these. @@ -573,7 +668,7 @@ Some important things: This requires a bit more advanced technique. But first, let's see how this conversion mechanism works: The library uses **JSON Serializers** to convert types to json. -The default serializer for `nlohmann::json` is `nlohmann::adl_serializer` (ADL means [Argument-Dependent Lookup](http://en.cppreference.com/w/cpp/language/adl)). +The default serializer for `nlohmann::json` is `nlohmann::adl_serializer` (ADL means [Argument-Dependent Lookup](https://en.cppreference.com/w/cpp/language/adl)). It is implemented like this (simplified): @@ -590,7 +685,7 @@ struct adl_serializer { }; ``` -This serializer works fine when you have control over the type's namespace. However, what about `boost::optional`, or `std::filesystem::path` (C++17)? Hijacking the `boost` namespace is pretty bad, and it's illegal to add something other than template specializations to `std`... +This serializer works fine when you have control over the type's namespace. However, what about `boost::optional` or `std::filesystem::path` (C++17)? Hijacking the `boost` namespace is pretty bad, and it's illegal to add something other than template specializations to `std`... To solve this, you need to add a specialization of `adl_serializer` to the `nlohmann` namespace, here's an example: @@ -612,7 +707,7 @@ namespace nlohmann { if (j.is_null()) { opt = boost::none; } else { - opt = j.get<T>(); // same as above, but with + opt = j.get<T>(); // same as above, but with // adl_serializer<T>::from_json } } @@ -622,7 +717,7 @@ namespace nlohmann { #### How can I use `get()` for non-default constructible/non-copyable types? -There is a way, if your type is [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible). You will need to specialize the `adl_serializer` as well, but with a special `from_json` overload: +There is a way, if your type is [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible). You will need to specialize the `adl_serializer` as well, but with a special `from_json` overload: ```cpp struct move_only_type { @@ -630,7 +725,7 @@ struct move_only_type { move_only_type(int ii): i(ii) {} move_only_type(const move_only_type&) = delete; move_only_type(move_only_type&&) = default; - + int i; }; @@ -642,7 +737,7 @@ namespace nlohmann { static move_only_type from_json(const json& j) { return {j.get<int>()}; } - + // Here's the catch! You must provide a to_json method! Otherwise you // will not be able to convert move_only_type to json, since you fully // specialized adl_serializer on that type @@ -659,9 +754,9 @@ Yes. You might want to take a look at [`unit-udt.cpp`](https://github.com/nlohma If you write your own serializer, you'll need to do a few things: -* use a different `basic_json` alias than `nlohmann::json` (the last template parameter of `basic_json` is the `JSONSerializer`) -* use your `basic_json` alias (or a template parameter) in all your `to_json`/`from_json` methods -* use `nlohmann::to_json` and `nlohmann::from_json` when you need ADL +- use a different `basic_json` alias than `nlohmann::json` (the last template parameter of `basic_json` is the `JSONSerializer`) +- use your `basic_json` alias (or a template parameter) in all your `to_json`/`from_json` methods +- use `nlohmann::to_json` and `nlohmann::from_json` when you need ADL Here is an example, without simplifications, that only accepts types with a size <= 32, and uses ADL. @@ -677,7 +772,7 @@ struct less_than_32_serializer { // this is where the magic happens to_json(j, value); } - + template <typename BasicJsonType> static void from_json(const BasicJsonType& j, T& value) { // same thing here @@ -699,7 +794,7 @@ struct bad_serializer // if BasicJsonType::json_serializer == bad_serializer ... oops! j = value; } - + template <typename BasicJsonType> static void to_json(const BasicJsonType& j, T& value) { // this calls BasicJsonType::json_serializer<T>::from_json(j, value); @@ -709,9 +804,9 @@ struct bad_serializer }; ``` -### Binary formats (CBOR and MessagePack) +### Binary formats (CBOR, MessagePack, and UBJSON) -Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [CBOR](http://cbor.io) (Concise Binary Object Representation) and [MessagePack](http://msgpack.org) to efficiently encode JSON values to byte vectors and to decode such vectors. +Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [CBOR](http://cbor.io) (Concise Binary Object Representation), [MessagePack](http://msgpack.org), and [UBJSON](http://ubjson.org) (Universal Binary JSON Specification) to efficiently encode JSON values to byte vectors and to decode such vectors. ```cpp // create a JSON value @@ -720,7 +815,7 @@ json j = R"({"compact": true, "schema": 0})"_json; // serialize to CBOR std::vector<std::uint8_t> v_cbor = json::to_cbor(j); -// 0xa2, 0x67, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xf5, 0x66, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 +// 0xA2, 0x67, 0x63, 0x6F, 0x6D, 0x70, 0x61, 0x63, 0x74, 0xF5, 0x66, 0x73, 0x63, 0x68, 0x65, 0x6D, 0x61, 0x00 // roundtrip json j_from_cbor = json::from_cbor(v_cbor); @@ -728,19 +823,27 @@ json j_from_cbor = json::from_cbor(v_cbor); // serialize to MessagePack std::vector<std::uint8_t> v_msgpack = json::to_msgpack(j); -// 0x82, 0xa7, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xc3, 0xa6, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 +// 0x82, 0xA7, 0x63, 0x6F, 0x6D, 0x70, 0x61, 0x63, 0x74, 0xC3, 0xA6, 0x73, 0x63, 0x68, 0x65, 0x6D, 0x61, 0x00 // roundtrip json j_from_msgpack = json::from_msgpack(v_msgpack); + +// serialize to UBJSON +std::vector<std::uint8_t> v_ubjson = json::to_ubjson(j); + +// 0x7B, 0x69, 0x07, 0x63, 0x6F, 0x6D, 0x70, 0x61, 0x63, 0x74, 0x54, 0x69, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6D, 0x61, 0x69, 0x00, 0x7D + +// roundtrip +json j_from_ubjson = json::from_ubjson(v_ubjson); ``` ## Supported compilers -Though it's 2017 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: +Though it's 2018 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: -- GCC 4.9 - 7.2 (and possibly later) -- Clang 3.4 - 5.0 (and possibly later) +- GCC 4.9 - 8.2 (and possibly later) +- Clang 3.4 - 6.1 (and possibly later) - Intel C++ Compiler 17.0.2 (and possibly later) - Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) - Microsoft Visual C++ 2017 / Build Tools 15.5.180.51428 (and possibly later) @@ -751,43 +854,49 @@ Please note: - GCC 4.8 does not work because of two bugs ([55817](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55817) and [57824](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57824)) in the C++11 support. Note there is a [pull request](https://github.com/nlohmann/json/pull/212) to fix some of the issues. - Android defaults to using very old compilers and C++ libraries. To fix this, add the following to your `Application.mk`. This will switch to the LLVM C++ library, the Clang compiler, and enable C++11 and other features disabled by default. - + ``` APP_STL := c++_shared NDK_TOOLCHAIN_VERSION := clang3.6 APP_CPPFLAGS += -frtti -fexceptions ``` - + The code compiles successfully with [Android NDK](https://developer.android.com/ndk/index.html?hl=ml), Revision 9 - 11 (and possibly later) and [CrystaX's Android NDK](https://www.crystax.net/en/android/ndk) version 10. - For GCC running on MinGW or Android SDK, the error `'to_string' is not a member of 'std'` (or similarly, for `strtod`) may occur. Note this is not an issue with the code, but rather with the compiler itself. On Android, see above to build with a newer environment. For MinGW, please refer to [this site](http://tehsausage.com/mingw-to-string) and [this discussion](https://github.com/nlohmann/json/issues/136) for information on how to fix this bug. For Android NDK using `APP_STL := gnustl_static`, please refer to [this discussion](https://github.com/nlohmann/json/issues/219). +- Unsupported versions of GCC and Clang are rejected by `#error` directives. This can be switched off by defining `JSON_SKIP_UNSUPPORTED_COMPILER_CHECK`. Note that you can expect no support in this case. + The following compilers are currently used in continuous integration at [Travis](https://travis-ci.org/nlohmann/json) and [AppVeyor](https://ci.appveyor.com/project/nlohmann/json): | Compiler | Operating System | Version String | |-----------------|------------------------------|----------------| -| GCC 4.9.4 | Ubuntu 14.04.5 LTS | g++-4.9 (Ubuntu 4.9.4-2ubuntu1~14.04.1) 4.9.4 | -| GCC 5.4.1 | Ubuntu 14.04.5 LTS | g++-5 (Ubuntu 5.4.1-2ubuntu1~14.04) 5.4.1 20160904 | -| GCC 6.3.0 | Ubuntu 14.04.5 LTS | g++-6 (Ubuntu/Linaro 6.3.0-18ubuntu2~14.04) 6.3.0 20170519 | -| GCC 7.1.0 | Ubuntu 14.04.5 LTS | g++-7 (Ubuntu 7.1.0-5ubuntu2~14.04) 7.1.0 -| Clang 3.5.0 | Ubuntu 14.04.5 LTS | clang version 3.5.0-4ubuntu2~trusty2 (tags/RELEASE_350/final) | -| Clang 3.6.2 | Ubuntu 14.04.5 LTS | clang version 3.6.2-svn240577-1~exp1 (branches/release_36) | -| Clang 3.7.1 | Ubuntu 14.04.5 LTS | clang version 3.7.1-svn253571-1~exp1 (branches/release_37) | -| Clang 3.8.0 | Ubuntu 14.04.5 LTS | clang version 3.8.0-2ubuntu3~trusty5 (tags/RELEASE_380/final) | -| Clang 3.9.1 | Ubuntu 14.04.5 LTS | clang version 3.9.1-4ubuntu3~14.04.2 (tags/RELEASE_391/rc2) | -| Clang 4.0.1 | Ubuntu 14.04.5 LTS | clang version 4.0.1-svn305264-1~exp1 (branches/release_40) | -| Clang 5.0.0 | Ubuntu 14.04.5 LTS | clang version 5.0.0-svn310902-1~exp1 (branches/release_50) | -| Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | -| Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | -| Clang Xcode 8.0 | Darwin Kernel Version 15.6.0 | Apple LLVM version 8.0.0 (clang-800.0.38) | -| Clang Xcode 8.1 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | -| Clang Xcode 8.2 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | -| Clang Xcode 8.3 | Darwin Kernel Version 16.5.0 (macOS 10.12.4) | Apple LLVM version 8.1.0 (clang-802.0.38) | -| Clang Xcode 9.0 | Darwin Kernel Version 16.7.0 (macOS 10.12.6) | Apple LLVM version 9.0.0 (clang-900.0.37) | -| Clang Xcode 9.1 | Darwin Kernel Version 16.7.0 (macOS 10.12.6) | Apple LLVM version 9.0.0 (clang-900.0.38) | -| Clang Xcode 9.2 | Darwin Kernel Version 16.7.0 (macOS 10.12.6) | Apple LLVM version 8.1.0 (clang-900.0.39.2) | -| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25420.1, MSVC 19.0.24215.1 | -| Visual Studio 2017 | Windows Server 2016 | Microsoft (R) Build Engine version 15.5.180.51428, MSVC 19.12.25830.2 | +| GCC 4.9.4 | Ubuntu 14.04.1 LTS | g++-4.9 (Ubuntu 4.9.4-2ubuntu1~14.04.1) 4.9.4 | +| GCC 5.5.0 | Ubuntu 14.04.1 LTS | g++-5 (Ubuntu 5.5.0-12ubuntu1~14.04) 5.5.0 20171010 | +| GCC 6.4.0 | Ubuntu 14.04.1 LTS | g++-6 (Ubuntu 6.4.0-17ubuntu1~14.04) 6.4.0 20180424 | +| GCC 7.3.0 | Ubuntu 14.04.1 LTS | g++-7 (Ubuntu 7.3.0-21ubuntu1~14.04) 7.3.0 | +| GCC 7.3.0 | Windows Server 2012 R2 (x64) | g++ (x86_64-posix-seh-rev0, Built by MinGW-W64 project) 7.3.0 | +| GCC 8.1.0 | Ubuntu 14.04.1 LTS | g++-8 (Ubuntu 8.1.0-5ubuntu1~14.04) 8.1.0 | +| Clang 3.5.0 | Ubuntu 14.04.1 LTS | clang version 3.5.0-4ubuntu2~trusty2 (tags/RELEASE_350/final) (based on LLVM 3.5.0) | +| Clang 3.6.2 | Ubuntu 14.04.1 LTS | clang version 3.6.2-svn240577-1~exp1 (branches/release_36) (based on LLVM 3.6.2) | +| Clang 3.7.1 | Ubuntu 14.04.1 LTS | clang version 3.7.1-svn253571-1~exp1 (branches/release_37) (based on LLVM 3.7.1) | +| Clang 3.8.0 | Ubuntu 14.04.1 LTS | clang version 3.8.0-2ubuntu3~trusty5 (tags/RELEASE_380/final) | +| Clang 3.9.1 | Ubuntu 14.04.1 LTS | clang version 3.9.1-4ubuntu3~14.04.3 (tags/RELEASE_391/rc2) | +| Clang 4.0.1 | Ubuntu 14.04.1 LTS | clang version 4.0.1-svn305264-1~exp1 (branches/release_40) | +| Clang 5.0.2 | Ubuntu 14.04.1 LTS | clang version 5.0.2-svn328729-1~exp1~20180509123505.100 (branches/release_50) | +| Clang 6.0.1 | Ubuntu 14.04.1 LTS | clang version 6.0.1-svn334776-1~exp1~20180726133705.85 (branches/release_60) | +| Clang Xcode 6.4 | OSX 10.10.5 | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | +| Clang Xcode 7.3 | OSX 10.11.6 | Apple LLVM version 7.3.0 (clang-703.0.31) | +| Clang Xcode 8.0 | OSX 10.11.6 | Apple LLVM version 8.0.0 (clang-800.0.38) | +| Clang Xcode 8.1 | OSX 10.12.6 | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.2 | OSX 10.12.6 | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.3 | OSX 10.11.6 | Apple LLVM version 8.1.0 (clang-802.0.38) | +| Clang Xcode 9.0 | OSX 10.12.6 | Apple LLVM version 9.0.0 (clang-900.0.37) | +| Clang Xcode 9.1 | OSX 10.12.6 | Apple LLVM version 9.0.0 (clang-900.0.38) | +| Clang Xcode 9.2 | OSX 10.13.3 | Apple LLVM version 9.1.0 (clang-902.0.39.1) | +| Clang Xcode 9.3 | OSX 10.13.3 | Apple LLVM version 9.1.0 (clang-902.0.39.2) | +| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25420.1, MSVC 19.0.24215.1 | +| Visual Studio 2017 | Windows Server 2016 | Microsoft (R) Build Engine version 15.7.180.61344, MSVC 19.14.26433.0 | ## License @@ -795,7 +904,7 @@ The following compilers are currently used in continuous integration at [Travis] The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): -Copyright © 2013-2017 [Niels Lohmann](http://nlohmann.me) +Copyright © 2013-2018 [Niels Lohmann](http://nlohmann.me) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: @@ -807,12 +916,17 @@ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR I The class contains the UTF-8 Decoder from Bjoern Hoehrmann which is licensed under the [MIT License](http://opensource.org/licenses/MIT) (see above). Copyright © 2008-2009 [Björn Hoehrmann](http://bjoern.hoehrmann.de/) <bjoern@hoehrmann.de> +The class contains a slightly modified version of the Grisu2 algorithm from Florian Loitsch which is licensed under the [MIT License](http://opensource.org/licenses/MIT) (see above). Copyright © 2009 [Florian Loitsch](http://florian.loitsch.com/) + ## Contact -If you have questions regarding the library, I would like to invite you to [open an issue at Github](https://github.com/nlohmann/json/issues/new). Please describe your request, problem, or question as detailed as possible, and also mention the version of the library you are using as well as the version of your compiler and operating system. Opening an issue at Github allows other users and contributors to this library to collaborate. For instance, I have little experience with MSVC, and most issues in this regard have been solved by a growing community. If you have a look at the [closed issues](https://github.com/nlohmann/json/issues?q=is%3Aissue+is%3Aclosed), you will see that we react quite timely in most cases. +If you have questions regarding the library, I would like to invite you to [open an issue at GitHub](https://github.com/nlohmann/json/issues/new). Please describe your request, problem, or question as detailed as possible, and also mention the version of the library you are using as well as the version of your compiler and operating system. Opening an issue at GitHub allows other users and contributors to this library to collaborate. For instance, I have little experience with MSVC, and most issues in this regard have been solved by a growing community. If you have a look at the [closed issues](https://github.com/nlohmann/json/issues?q=is%3Aissue+is%3Aclosed), you will see that we react quite timely in most cases. Only if your request would contain confidential information, please [send me an email](mailto:mail@nlohmann.me). For encrypted messages, please use [this key](https://keybase.io/nlohmann/pgp_keys.asc). +## Security + +[Commits by Niels Lohmann](https://github.com/nlohmann/json/commits) and [releases](https://github.com/nlohmann/json/releases) are signed with this [PGP Key](https://keybase.io/nlohmann/pgp_keys.asc?fingerprint=797167ae41c0a6d9232e48457f3cea63ae251b69). ## Thanks @@ -847,7 +961,7 @@ I deeply appreciate the help of the following people. - [Corbin Hughes](https://github.com/nibroc) fixed some typos in the contribution guidelines. - [twelsby](https://github.com/twelsby) fixed the array subscript operator, an issue that failed the MSVC build, and floating-point parsing/dumping. He further added support for unsigned integer numbers and implemented better roundtrip support for parsed numbers. - [Volker Diels-Grabsch](https://github.com/vog) fixed a link in the README file. -- [msm-](https://github.com/msm-) added support for american fuzzy lop. +- [msm-](https://github.com/msm-) added support for American Fuzzy Lop. - [Annihil](https://github.com/Annihil) fixed an example in the README file. - [Themercee](https://github.com/Themercee) noted a wrong URL in the README file. - [Lv Zheng](https://github.com/lv-zheng) fixed a namespace issue with `int64_t` and `uint64_t`. @@ -860,14 +974,14 @@ I deeply appreciate the help of the following people. - [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. - [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. - [Thomas Braun](https://github.com/t-b) fixed a warning in a test case. -- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). He also implemented the magic behind the serialization/deserialization of user-defined types. +- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). He also implemented the magic behind the serialization/deserialization of user-defined types and split the single header file into smaller chunks. - [Stefan](https://github.com/5tefan) fixed a minor issue in the documentation. - [Vasil Dimov](https://github.com/vasild) fixed the documentation regarding conversions from `std::multiset`. - [ChristophJud](https://github.com/ChristophJud) overworked the CMake files to ease project inclusion. - [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable and added Visual Studio 17 to the build matrix. - [Denis Andrejew](https://github.com/seeekr) fixed a grammar issue in the README file. - [Pierre-Antoine Lacaze](https://github.com/palacaze) found a subtle bug in the `dump()` function. -- [TurpentineDistillery](https://github.com/TurpentineDistillery) pointed to [`std::locale::classic()`](http://en.cppreference.com/w/cpp/locale/locale/classic) to avoid too much locale joggling, found some nice performance improvements in the parser, improved the benchmarking code, and realized locale-independent number parsing and printing. +- [TurpentineDistillery](https://github.com/TurpentineDistillery) pointed to [`std::locale::classic()`](https://en.cppreference.com/w/cpp/locale/locale/classic) to avoid too much locale joggling, found some nice performance improvements in the parser, improved the benchmarking code, and realized locale-independent number parsing and printing. - [cgzones](https://github.com/cgzones) had an idea how to fix the Coverity scan. - [Jared Grubb](https://github.com/jaredgrubb) silenced a nasty documentation warning. - [Yixin Zhang](https://github.com/qwename) fixed an integer overflow check. @@ -893,7 +1007,7 @@ I deeply appreciate the help of the following people. - [Vincent Thiery](https://github.com/vthiery) maintains a package for the Conan package manager. - [Steffen](https://github.com/koemeet) fixed a potential issue with MSVC and `std::min`. - [Mike Tzou](https://github.com/Chocobo1) fixed some typos. -- [amrcode](https://github.com/amrcode) noted a missleading documentation about comparison of floats. +- [amrcode](https://github.com/amrcode) noted a misleading documentation about comparison of floats. - [Oleg Endo](https://github.com/olegendo) reduced the memory consumption by replacing `<iostream>` with `<iosfwd>`. - [dan-42](https://github.com/dan-42) cleaned up the CMake files to simplify including/reusing of the library. - [Nikita Ofitserov](https://github.com/himikof) allowed for moving values from initializer lists. @@ -910,25 +1024,50 @@ I deeply appreciate the help of the following people. - [Nate Vargas](https://github.com/eld00d) added a Doxygen tag file. - [pvleuven](https://github.com/pvleuven) helped fixing a warning in ICC. - [Pavel](https://github.com/crea7or) helped fixing some warnings in MSVC. -- [Jamie Seward](https://github.com/jseward) avoided unneccessary string copies in `find()` and `count()`. +- [Jamie Seward](https://github.com/jseward) avoided unnecessary string copies in `find()` and `count()`. - [Mitja](https://github.com/Itja) fixed some typos. - [Jorrit Wronski](https://github.com/jowr) updated the Hunter package links. - [Matthias Möller](https://github.com/TinyTinni) added a `.natvis` for the MSVC debug view. - [bogemic](https://github.com/bogemic) fixed some C++17 deprecation warnings. - [Eren Okka](https://github.com/erengy) fixed some MSVC warnings. - +- [abolz](https://github.com/abolz) integrated the Grisu2 algorithm for proper floating-point formatting, allowing more roundtrip checks to succeed. +- [Vadim Evard](https://github.com/Pipeliner) fixed a Markdown issue in the README. +- [zerodefect](https://github.com/zerodefect) fixed a compiler warning. +- [Kert](https://github.com/kaidokert) allowed to template the string type in the serialization and added the possibility to override the exceptional behavior. +- [mark-99](https://github.com/mark-99) helped fixing an ICC error. +- [Patrik Huber](https://github.com/patrikhuber) fixed links in the README file. +- [johnfb](https://github.com/johnfb) found a bug in the implementation of CBOR's indefinite length strings. +- [Paul Fultz II](https://github.com/pfultz2) added a note on the cget package manager. +- [Wilson Lin](https://github.com/wla80) made the integration section of the README more concise. +- [RalfBielig](https://github.com/ralfbielig) detected and fixed a memory leak in the parser callback. +- [agrianius](https://github.com/agrianius) allowed to dump JSON to an alternative string type. +- [Kevin Tonon](https://github.com/ktonon) overworked the C++11 compiler checks in CMake. +- [Axel Huebl](https://github.com/ax3l) simplified a CMake check and added support for the [Spack package manager](https://spack.io). +- [Carlos O'Ryan](https://github.com/coryan) fixed a typo. +- [James Upjohn](https://github.com/jammehcow) fixed a version number in the compilers section. +- [Chuck Atkins](https://github.com/chuckatkins) adjusted the CMake files to the CMake packaging guidelines +- [Jan Schöppach](https://github.com/dns13) fixed a typo. +- [martin-mfg](https://github.com/martin-mfg) fixed a typo. +- [Matthias Möller](https://github.com/TinyTinni) removed the dependency from `std::stringstream`. +- [agrianius](https://github.com/agrianius) added code to use alternative string implementations. +- [Daniel599](https://github.com/Daniel599) allowed to use more algorithms with the `items()` function. +- [Julius Rakow](https://github.com/jrakow) fixed the Meson include directory and fixed the links to [cppreference.com](cppreference.com). +- [Sonu Lohani](https://github.com/sonulohani) fixed the compilation with MSVC 2015 in debug mode. +- [grembo](https://github.com/grembo) fixed the test suite and re-enabled several test cases. +- [Hyeon Kim](https://github.com/simnalamburt) introduced the macro `JSON_INTERNAL_CATCH` to control the exception handling inside the library. +- [thyu](https://github.com/thyu) fixed a compiler warning. Thanks a lot for helping out! Please [let me know](mailto:mail@nlohmann.me) if I forgot someone. ## Used third-party tools -The library itself contains of a single header file licensed under the MIT license. However, it is built, tested, documented, and whatnot using a lot of third-party tools and services. Thanks a lot! +The library itself consists of a single header file licensed under the MIT license. However, it is built, tested, documented, and whatnot using a lot of third-party tools and services. Thanks a lot! +- [**amalgamate.py - Amalgamate C source and header files**](https://github.com/edlund/amalgamate) to create a single header file - [**American fuzzy lop**](http://lcamtuf.coredump.cx/afl/) for fuzz testing - [**AppVeyor**](https://www.appveyor.com) for [continuous integration](https://ci.appveyor.com/project/nlohmann/json) on Windows - [**Artistic Style**](http://astyle.sourceforge.net) for automatic source code identation -- [**benchpress**](https://github.com/sbs-ableton/benchpress) to benchmark the code - [**Catch**](https://github.com/philsquared/Catch) for the unit tests - [**Clang**](http://clang.llvm.org) for compilation with code sanitizers - [**Cmake**](https://cmake.org) for build automation @@ -936,17 +1075,17 @@ The library itself contains of a single header file licensed under the MIT licen - [**Coveralls**](https://coveralls.io) to measure [code coverage](https://coveralls.io/github/nlohmann/json) - [**Coverity Scan**](https://scan.coverity.com) for [static analysis](https://scan.coverity.com/projects/nlohmann-json) - [**cppcheck**](http://cppcheck.sourceforge.net) for static analysis -- [**cxxopts**](https://github.com/jarro2783/cxxopts) to let benchpress parse command-line parameters - [**Doxygen**](http://www.stack.nl/~dimitri/doxygen/) to generate [documentation](https://nlohmann.github.io/json/) - [**git-update-ghpages**](https://github.com/rstacruz/git-update-ghpages) to upload the documentation to gh-pages -- [**Github Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md) +- [**GitHub Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md) +- [**Google Benchmark**](https://github.com/google/benchmark) to implement the benchmarks - [**libFuzzer**](http://llvm.org/docs/LibFuzzer.html) to implement fuzz testing for OSS-Fuzz -- [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library +- [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library ([project repository](https://github.com/google/oss-fuzz/tree/master/projects/json)) - [**Probot**](https://probot.github.io) for automating maintainer tasks such as closing stale issues, requesting missing information, or detecting toxic comments. - [**send_to_wandbox**](https://github.com/nlohmann/json/blob/develop/doc/scripts/send_to_wandbox.py) to send code examples to [Wandbox](http://melpon.org/wandbox) - [**Travis**](https://travis-ci.org) for [continuous integration](https://travis-ci.org/nlohmann/json) on Linux and macOS - [**Valgrind**](http://valgrind.org) to check for correct memory management -- [**Wandbox**](http://melpon.org/wandbox) for [online examples](https://wandbox.org/permlink/Op57X0V7fTf2tdwl) +- [**Wandbox**](http://melpon.org/wandbox) for [online examples](https://wandbox.org/permlink/TarF5pPn9NtHQjhf) ## Projects using JSON for Modern C++ @@ -956,7 +1095,7 @@ The library is currently used in Apple macOS Sierra and iOS 10. I am not sure wh ## Notes -- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). In particular, note [`operator[]`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a2e26bd0b0168abb61f67ad5bcd5b9fa1.html#a2e26bd0b0168abb61f67ad5bcd5b9fa1) implements **unchecked access** for const objects: If the given key is not present, the behavior is undefined (think of a dereferenced null pointer) and yields an [assertion failure](https://github.com/nlohmann/json/issues/289) if assertions are switched on. If you are not sure whether an element in an object exists, use checked access with the [`at()` function](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a674de1ee73e6bf4843fc5dc1351fb726.html#a674de1ee73e6bf4843fc5dc1351fb726). +- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](https://en.cppreference.com/w/cpp/error/assert). In particular, note [`operator[]`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a2e26bd0b0168abb61f67ad5bcd5b9fa1.html#a2e26bd0b0168abb61f67ad5bcd5b9fa1) implements **unchecked access** for const objects: If the given key is not present, the behavior is undefined (think of a dereferenced null pointer) and yields an [assertion failure](https://github.com/nlohmann/json/issues/289) if assertions are switched on. If you are not sure whether an element in an object exists, use checked access with the [`at()` function](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a674de1ee73e6bf4843fc5dc1351fb726.html#a674de1ee73e6bf4843fc5dc1351fb726). - As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions. - The library supports **Unicode input** as follows: - Only **UTF-8** encoded input is supported which is the default encoding for JSON according to [RFC 7159](http://rfc7159.net/rfc7159#rfc.section.8.1). diff --git a/ext/json/json.hpp b/ext/json/json.hpp index 5b0b0ea5..b80386f3 100644 --- a/ext/json/json.hpp +++ b/ext/json/json.hpp @@ -1,11 +1,12 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.0.1 +| | |__ | | | | | | version 3.2.0 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License <http://opensource.org/licenses/MIT>. -Copyright (c) 2013-2017 Niels Lohmann <http://nlohmann.me>. +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -29,42 +30,104 @@ SOFTWARE. #ifndef NLOHMANN_JSON_HPP #define NLOHMANN_JSON_HPP -#include <algorithm> // all_of, copy, fill, find, for_each, generate_n, none_of, remove, reverse, transform -#include <array> // array +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 2 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include <algorithm> // all_of, find, for_each #include <cassert> // assert #include <ciso646> // and, not, or -#include <clocale> // lconv, localeconv -#include <cmath> // isfinite, labs, ldexp, signbit #include <cstddef> // nullptr_t, ptrdiff_t, size_t -#include <cstdint> // int64_t, uint64_t -#include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull -#include <cstring> // memcpy, strlen -#include <forward_list> // forward_list -#include <functional> // function, hash, less +#include <functional> // hash, less #include <initializer_list> // initializer_list -#include <iomanip> // hex -#include <iosfwd> // istream, ostream -#include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator -#include <limits> // numeric_limits -#include <locale> // locale -#include <map> // map -#include <memory> // addressof, allocator, allocator_traits, unique_ptr +#include <iosfwd> // istream, ostream +#include <iterator> // iterator_traits, random_access_iterator_tag #include <numeric> // accumulate -#include <sstream> // stringstream -#include <string> // getline, stoi, string, to_string -#include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type -#include <utility> // declval, forward, make_pair, move, pair, swap -#include <valarray> // valarray +#include <string> // string, stoi, to_string +#include <utility> // declval, forward, move, pair, swap + +// #include <nlohmann/json_fwd.hpp> +#ifndef NLOHMANN_JSON_FWD_HPP +#define NLOHMANN_JSON_FWD_HPP + +#include <cstdint> // int64_t, uint64_t +#include <map> // map +#include <memory> // allocator +#include <string> // string #include <vector> // vector +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template<typename T = void, typename SFINAE = void> +struct adl_serializer; + +template<template<typename U, typename V, typename... Args> class ObjectType = + std::map, + template<typename U, typename... Args> class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template<typename U> class AllocatorType = std::allocator, + template<typename T, typename SFINAE = void> class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template<typename BasicJsonType> +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} + +#endif + +// #include <nlohmann/detail/macro_scope.hpp> + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + // exclude unsupported compilers -#if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif #endif #endif @@ -90,14 +153,35 @@ SOFTWARE. #endif // allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION) +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception #define JSON_TRY try #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) #else #define JSON_THROW(exception) std::abort() #define JSON_TRY if(true) #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER #endif // manual branch prediction @@ -117,27 +201,6 @@ SOFTWARE. #define JSON_HAS_CPP_14 #endif -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ -template<typename = void, typename = void> -struct adl_serializer; - -// forward declaration of basic_json (required to split the class) -template<template<typename, typename, typename...> class ObjectType = std::map, - template<typename, typename...> class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template<typename> class AllocatorType = std::allocator, - template<typename, typename = void> class JSONSerializer = adl_serializer> -class basic_json; - // Ugly macros to avoid uglier copy-paste when specializing basic_json. They // may be removed in the future once the class is split. @@ -154,17 +217,334 @@ class basic_json; NumberIntegerType, NumberUnsignedType, NumberFloatType, \ AllocatorType, JSONSerializer> +/*! +@brief Helper to determine whether there's a key_type for T. + +This helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. + +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0, overworked in version 2.0.6 +*/ +#define NLOHMANN_JSON_HAS_HELPER(type) \ + template<typename T> struct has_##type { \ + private: \ + template<typename U, typename = typename U::type> \ + static int detect(U &&); \ + static void detect(...); \ + public: \ + static constexpr bool value = \ + std::is_integral<decltype(detect(std::declval<T>()))>::value; \ + } + +// #include <nlohmann/detail/meta/cpp_future.hpp> + + +#include <ciso646> // not +#include <cstddef> // size_t +#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template<bool B, typename T = void> +using enable_if_t = typename std::enable_if<B, T>::type; + +template<typename T> +using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template<std::size_t... Ints> +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template<class Sequence1, class Sequence2> +struct merge_and_renumber; + +template<std::size_t... I1, std::size_t... I2> +struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template<std::size_t N> +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template<typename... Ts> +using index_sequence_for = make_index_sequence<sizeof...(Ts)>; + +/* +Implementation of two C++17 constructs: conjunction, negation. This is needed +to avoid evaluating all the traits in a condition + +For example: not std::is_same<void, T>::value and has_value_type<T>::value +will not compile when T = void (on MSVC at least). Whereas +conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will +stop evaluating if negation<...>::value == false + +Please note that those constructs must be used with caution, since symbols can +become very long quickly (which can slow down compilation and cause MSVC +internal compiler errors). Only use it when you have to (see example ahead). +*/ +template<class...> struct conjunction : std::true_type {}; +template<class B1> struct conjunction<B1> : B1 {}; +template<class B1, class... Bn> +struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {}; + +template<class B> struct negation : std::integral_constant<bool, not B::value> {}; + +// dispatch utility (taken from ranges-v3) +template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; +// taken from ranges-v3 +template<typename T> +struct static_const +{ + static constexpr T value{}; +}; + +template<typename T> +constexpr T static_const<T>::value; +} +} + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +#include <ciso646> // not +#include <limits> // numeric_limits +#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type +#include <utility> // declval + +// #include <nlohmann/json_fwd.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + + +namespace nlohmann +{ /*! -@brief unnamed namespace with internal helper functions +@brief detail namespace with internal helper functions -This namespace collects some functions that could not be defined inside the -@ref basic_json class. +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. @since version 2.1.0 */ namespace detail { +///////////// +// helpers // +///////////// + +template<typename> struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {}; + +//////////////////////// +// has_/is_ functions // +//////////////////////// + +// source: https://stackoverflow.com/a/37193089/4116453 + +template <typename T, typename = void> +struct is_complete_type : std::false_type {}; + +template <typename T> +struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {}; + +NLOHMANN_JSON_HAS_HELPER(mapped_type); +NLOHMANN_JSON_HAS_HELPER(key_type); +NLOHMANN_JSON_HAS_HELPER(value_type); +NLOHMANN_JSON_HAS_HELPER(iterator); + +template<bool B, class RealType, class CompatibleObjectType> +struct is_compatible_object_type_impl : std::false_type {}; + +template<class RealType, class CompatibleObjectType> +struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType> +{ + static constexpr auto value = + std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and + std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value; +}; + +template<bool B, class RealType, class CompatibleStringType> +struct is_compatible_string_type_impl : std::false_type {}; + +template<class RealType, class CompatibleStringType> +struct is_compatible_string_type_impl<true, RealType, CompatibleStringType> +{ + static constexpr auto value = + std::is_same<typename RealType::value_type, typename CompatibleStringType::value_type>::value and + std::is_constructible<RealType, CompatibleStringType>::value; +}; + +template<class BasicJsonType, class CompatibleObjectType> +struct is_compatible_object_type +{ + static auto constexpr value = is_compatible_object_type_impl < + conjunction<negation<std::is_same<void, CompatibleObjectType>>, + has_mapped_type<CompatibleObjectType>, + has_key_type<CompatibleObjectType>>::value, + typename BasicJsonType::object_t, CompatibleObjectType >::value; +}; + +template<class BasicJsonType, class CompatibleStringType> +struct is_compatible_string_type +{ + static auto constexpr value = is_compatible_string_type_impl < + conjunction<negation<std::is_same<void, CompatibleStringType>>, + has_value_type<CompatibleStringType>>::value, + typename BasicJsonType::string_t, CompatibleStringType >::value; +}; + +template<typename BasicJsonType, typename T> +struct is_basic_json_nested_type +{ + static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or + std::is_same<T, typename BasicJsonType::const_iterator>::value or + std::is_same<T, typename BasicJsonType::reverse_iterator>::value or + std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value; +}; + +template<class BasicJsonType, class CompatibleArrayType> +struct is_compatible_array_type +{ + static auto constexpr value = + conjunction<negation<std::is_same<void, CompatibleArrayType>>, + negation<is_compatible_object_type< + BasicJsonType, CompatibleArrayType>>, + negation<std::is_constructible<typename BasicJsonType::string_t, + CompatibleArrayType>>, + negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>, + has_value_type<CompatibleArrayType>, + has_iterator<CompatibleArrayType>>::value; +}; + +template<bool, typename, typename> +struct is_compatible_integer_type_impl : std::false_type {}; + +template<typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits<RealIntegerType>; + using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>; + + static constexpr auto value = + std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template<typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type +{ + static constexpr auto value = + is_compatible_integer_type_impl < + std::is_integral<CompatibleNumberIntegerType>::value and + not std::is_same<bool, CompatibleNumberIntegerType>::value, + RealIntegerType, CompatibleNumberIntegerType > ::value; +}; + +// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists +template<typename BasicJsonType, typename T> +struct has_from_json +{ + private: + // also check the return type of from_json + template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json( + std::declval<BasicJsonType>(), std::declval<T&>()))>::value>> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral<decltype( + detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; +}; + +// This trait checks if JSONSerializer<T>::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template<typename BasicJsonType, typename T> +struct has_non_default_from_json +{ + private: + template < + typename U, + typename = enable_if_t<std::is_same< + T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value >> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral<decltype(detect( + std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; +}; + +// This trait checks if BasicJsonType::json_serializer<T>::to_json exists +template<typename BasicJsonType, typename T> +struct has_to_json +{ + private: + template<typename U, typename = decltype(uncvref_t<U>::to_json( + std::declval<BasicJsonType&>(), std::declval<T>()))> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral<decltype(detect( + std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; +}; + +template <typename BasicJsonType, typename CompatibleCompleteType> +struct is_compatible_complete_type +{ + static constexpr bool value = + not std::is_base_of<std::istream, CompatibleCompleteType>::value and + not is_basic_json<CompatibleCompleteType>::value and + not is_basic_json_nested_type<BasicJsonType, CompatibleCompleteType>::value and + has_to_json<BasicJsonType, CompatibleCompleteType>::value; +}; + +template <typename BasicJsonType, typename CompatibleType> +struct is_compatible_type + : conjunction<is_complete_type<CompatibleType>, + is_compatible_complete_type<BasicJsonType, CompatibleType>> +{ +}; +} +} + +// #include <nlohmann/detail/exceptions.hpp> + + +#include <exception> // exception +#include <stdexcept> // runtime_error +#include <string> // to_string + +namespace nlohmann +{ +namespace detail +{ //////////////// // exceptions // //////////////// @@ -419,6 +799,8 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON only supports integers numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | @liveexample{The following code shows how an `out_of_range` exception can be caught.,out_of_range} @@ -481,9 +863,21 @@ class other_error : public exception private: other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; +} +} + +// #include <nlohmann/detail/value_t.hpp> +#include <array> // array +#include <ciso646> // and +#include <cstddef> // size_t +#include <cstdint> // uint8_t +namespace nlohmann +{ +namespace detail +{ /////////////////////////// // JSON type enumeration // /////////////////////////// @@ -512,7 +906,7 @@ value with the default value for a given type @since version 1.0.0 */ -enum class value_t : uint8_t +enum class value_t : std::uint8_t { null, ///< null value object, ///< object (unordered set of name/value pairs) @@ -537,7 +931,7 @@ Returns an ordering that is similar to Python: */ inline bool operator<(const value_t lhs, const value_t rhs) noexcept { - static constexpr std::array<uint8_t, 8> order = {{ + static constexpr std::array<std::uint8_t, 8> order = {{ 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */ } @@ -547,80 +941,558 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept const auto r_index = static_cast<std::size_t>(rhs); return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]; } +} +} +// #include <nlohmann/detail/conversions/from_json.hpp> -///////////// -// helpers // -///////////// -template<typename> struct is_basic_json : std::false_type {}; +#include <algorithm> // transform +#include <array> // array +#include <ciso646> // and, not +#include <forward_list> // forward_list +#include <iterator> // inserter, front_inserter, end +#include <map> // map +#include <string> // string +#include <tuple> // tuple, make_tuple +#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include <unordered_map> // unordered_map +#include <utility> // pair, declval +#include <valarray> // valarray -NLOHMANN_BASIC_JSON_TPL_DECLARATION -struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {}; +// #include <nlohmann/detail/exceptions.hpp> -// alias templates to reduce boilerplate -template<bool B, typename T = void> -using enable_if_t = typename std::enable_if<B, T>::type; +// #include <nlohmann/detail/macro_scope.hpp> -template<typename T> -using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type; +// #include <nlohmann/detail/meta/cpp_future.hpp> -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template<std::size_t... Ints> -struct index_sequence +// #include <nlohmann/detail/meta/type_traits.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann { - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept +namespace detail +{ +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename std::nullptr_t& n) +{ + if (JSON_UNLIKELY(not j.is_null())) { - return sizeof...(Ints); + JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name()))); + } + n = nullptr; +} + +// overloads for basic_json template parameters +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t<std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (JSON_UNLIKELY(not j.is_boolean())) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()))); + } + b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template < + typename BasicJsonType, typename CompatibleStringType, + enable_if_t < + is_compatible_string_type<BasicJsonType, CompatibleStringType>::value and + not std::is_same<typename BasicJsonType::string_t, + CompatibleStringType>::value, + int > = 0 > +void from_json(const BasicJsonType& j, CompatibleStringType& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType, typename EnumType, + enable_if_t<std::is_enum<EnumType>::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type<EnumType>::type val; + get_arithmetic_value(j, val); + e = static_cast<EnumType>(val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + arr = *j.template get_ptr<const typename BasicJsonType::array_t*>(); +} + +// forward_list doesn't have an insert method +template<typename BasicJsonType, typename T, typename Allocator, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + std::transform(j.rbegin(), j.rend(), + std::front_inserter(l), [](const BasicJsonType & i) + { + return i.template get<T>(); + }); +} + +// valarray doesn't have an insert method +template<typename BasicJsonType, typename T, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::valarray<T>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + l.resize(j.size()); + std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l)); +} + +template<typename BasicJsonType, typename CompatibleArrayType> +void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/) +{ + using std::end; + + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename CompatibleArrayType::value_type>(); + }); +} + +template<typename BasicJsonType, typename CompatibleArrayType> +auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/) +-> decltype( + arr.reserve(std::declval<typename CompatibleArrayType::size_type>()), + void()) +{ + using std::end; + + arr.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename CompatibleArrayType::value_type>(); + }); +} + +template<typename BasicJsonType, typename T, std::size_t N> +void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get<T>(); + } +} + +template < + typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < + is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and + not std::is_same<typename BasicJsonType::array_t, + CompatibleArrayType>::value and + std::is_constructible < + BasicJsonType, typename CompatibleArrayType::value_type >::value, + int > = 0 > +void from_json(const BasicJsonType& j, CompatibleArrayType& arr) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + + std::string(j.type_name()))); + } + + from_json_array_impl(j, arr, priority_tag<2> {}); +} + +template<typename BasicJsonType, typename CompatibleObjectType, + enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleObjectType& obj) +{ + if (JSON_UNLIKELY(not j.is_object())) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()))); + } + + auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>(); + using value_type = typename CompatibleObjectType::value_type; + std::transform( + inner_object->begin(), inner_object->end(), + std::inserter(obj, obj.begin()), + [](typename BasicJsonType::object_t::value_type const & p) + { + return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>()); + }); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t < + std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + case value_t::boolean: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType, typename A1, typename A2> +void from_json(const BasicJsonType& j, std::pair<A1, A2>& p) +{ + p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()}; +} + +template<typename BasicJsonType, typename Tuple, std::size_t... Idx> +void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>) +{ + t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...); +} + +template<typename BasicJsonType, typename... Args> +void from_json(const BasicJsonType& j, std::tuple<Args...>& t) +{ + from_json_tuple_impl(j, t, index_sequence_for<Args...> {}); +} + +template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +struct from_json_fn +{ + private: + template<typename BasicJsonType, typename T> + auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); + } + + template<typename BasicJsonType, typename T> + void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find from_json() method in T's namespace"); +#ifdef _MSC_VER + // MSVC does not show a stacktrace for the above assert + using decayed = uncvref_t<T>; + static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, + "forcing MSVC stacktrace to show which T we're talking about."); +#endif + } + + public: + template<typename BasicJsonType, typename T> + void operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {}))) + { + return call(j, val, priority_tag<1> {}); } }; +} -template<class Sequence1, class Sequence2> -struct merge_and_renumber; +/// namespace to hold default `from_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace +{ +constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value; +} +} -template<std::size_t... I1, std::size_t... I2> -struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; +// #include <nlohmann/detail/conversions/to_json.hpp> -template<std::size_t N> -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; +#include <ciso646> // or, and, not +#include <iterator> // begin, end +#include <tuple> // tuple, get +#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type +#include <utility> // move, forward, declval, pair +#include <valarray> // valarray +#include <vector> // vector -template<typename... Ts> -using index_sequence_for = make_index_sequence<sizeof...(Ts)>; +// #include <nlohmann/detail/meta/cpp_future.hpp> -/* -Implementation of two C++17 constructs: conjunction, negation. This is needed -to avoid evaluating all the traits in a condition +// #include <nlohmann/detail/meta/type_traits.hpp> -For example: not std::is_same<void, T>::value and has_value_type<T>::value -will not compile when T = void (on MSVC at least). Whereas -conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will -stop evaluating if negation<...>::value == false +// #include <nlohmann/detail/value_t.hpp> -Please note that those constructs must be used with caution, since symbols can -become very long quickly (which can slow down compilation and cause MSVC -internal compiler errors). Only use it when you have to (see example ahead). -*/ -template<class...> struct conjunction : std::true_type {}; -template<class B1> struct conjunction<B1> : B1 {}; -template<class B1, class... Bn> -struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {}; +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> -template<class B> struct negation : std::integral_constant<bool, not B::value> {}; -// dispatch utility (taken from ranges-v3) -template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; +#include <cstddef> // size_t +#include <string> // string, to_string +#include <iterator> // input_iterator_tag + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/// proxy class for the items() function +template<typename IteratorType> class iteration_proxy +{ + private: + /// helper class for iteration + class iteration_proxy_internal + { + public: + using difference_type = std::ptrdiff_t; + using value_type = iteration_proxy_internal; + using pointer = iteration_proxy_internal*; + using reference = iteration_proxy_internal&; + using iterator_category = std::input_iterator_tag; + + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + std::size_t array_index = 0; + /// last stringified array index + mutable std::size_t array_index_last = 0; + /// a string representation of the array index + mutable std::string array_index_str = "0"; + /// an empty string (to return a reference for primitive values) + const std::string empty_str = ""; + + public: + explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {} + + iteration_proxy_internal(const iteration_proxy_internal&) = default; + iteration_proxy_internal& operator=(const iteration_proxy_internal&) = default; + + /// dereference operator (needed for range-based for) + iteration_proxy_internal& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_internal& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// equality operator (needed for InputIterator) + bool operator==(const iteration_proxy_internal& o) const noexcept + { + return anchor == o.anchor; + } + + /// inequality operator (needed for range-based for) + bool operator!=(const iteration_proxy_internal& o) const noexcept + { + return anchor != o.anchor; + } + + /// return key of the iterator + const std::string& key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + if (array_index != array_index_last) + { + array_index_str = std::to_string(array_index); + array_index_last = array_index; + } + return array_index_str; + } + + // use key from the object + case value_t::object: + return anchor.key(); + + // use an empty key for all primitive types + default: + return empty_str; + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } + }; + + /// the container to iterate + typename IteratorType::reference container; + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) noexcept + : container(cont) {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_internal begin() noexcept + { + return iteration_proxy_internal(container.begin()); + } + /// return iterator end (needed for range-based for) + iteration_proxy_internal end() noexcept + { + return iteration_proxy_internal(container.end()); + } +}; +} +} + + +namespace nlohmann +{ +namespace detail +{ ////////////////// // constructors // ////////////////// @@ -657,6 +1529,16 @@ struct external_constructor<value_t::string> j.m_value = std::move(s); j.assert_invariant(); } + + template<typename BasicJsonType, typename CompatibleStringType, + enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleStringType& str) + { + j.m_type = value_t::string; + j.m_value.string = j.template create<typename BasicJsonType::string_t>(str); + j.assert_invariant(); + } }; template<> @@ -783,159 +1665,6 @@ struct external_constructor<value_t::object> } }; - -//////////////////////// -// has_/is_ functions // -//////////////////////// - -/*! -@brief Helper to determine whether there's a key_type for T. - -This helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. - -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0, overworked in version 2.0.6 -*/ -#define NLOHMANN_JSON_HAS_HELPER(type) \ - template<typename T> struct has_##type { \ - private: \ - template<typename U, typename = typename U::type> \ - static int detect(U &&); \ - static void detect(...); \ - public: \ - static constexpr bool value = \ - std::is_integral<decltype(detect(std::declval<T>()))>::value; \ - } - -NLOHMANN_JSON_HAS_HELPER(mapped_type); -NLOHMANN_JSON_HAS_HELPER(key_type); -NLOHMANN_JSON_HAS_HELPER(value_type); -NLOHMANN_JSON_HAS_HELPER(iterator); - -#undef NLOHMANN_JSON_HAS_HELPER - - -template<bool B, class RealType, class CompatibleObjectType> -struct is_compatible_object_type_impl : std::false_type {}; - -template<class RealType, class CompatibleObjectType> -struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType> -{ - static constexpr auto value = - std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and - std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value; -}; - -template<class BasicJsonType, class CompatibleObjectType> -struct is_compatible_object_type -{ - static auto constexpr value = is_compatible_object_type_impl < - conjunction<negation<std::is_same<void, CompatibleObjectType>>, - has_mapped_type<CompatibleObjectType>, - has_key_type<CompatibleObjectType>>::value, - typename BasicJsonType::object_t, CompatibleObjectType >::value; -}; - -template<typename BasicJsonType, typename T> -struct is_basic_json_nested_type -{ - static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or - std::is_same<T, typename BasicJsonType::const_iterator>::value or - std::is_same<T, typename BasicJsonType::reverse_iterator>::value or - std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value; -}; - -template<class BasicJsonType, class CompatibleArrayType> -struct is_compatible_array_type -{ - static auto constexpr value = - conjunction<negation<std::is_same<void, CompatibleArrayType>>, - negation<is_compatible_object_type< - BasicJsonType, CompatibleArrayType>>, - negation<std::is_constructible<typename BasicJsonType::string_t, - CompatibleArrayType>>, - negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>, - has_value_type<CompatibleArrayType>, - has_iterator<CompatibleArrayType>>::value; -}; - -template<bool, typename, typename> -struct is_compatible_integer_type_impl : std::false_type {}; - -template<typename RealIntegerType, typename CompatibleNumberIntegerType> -struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType> -{ - // is there an assert somewhere on overflows? - using RealLimits = std::numeric_limits<RealIntegerType>; - using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>; - - static constexpr auto value = - std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and - CompatibleLimits::is_integer and - RealLimits::is_signed == CompatibleLimits::is_signed; -}; - -template<typename RealIntegerType, typename CompatibleNumberIntegerType> -struct is_compatible_integer_type -{ - static constexpr auto value = - is_compatible_integer_type_impl < - std::is_integral<CompatibleNumberIntegerType>::value and - not std::is_same<bool, CompatibleNumberIntegerType>::value, - RealIntegerType, CompatibleNumberIntegerType >::value; -}; - - -// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists -template<typename BasicJsonType, typename T> -struct has_from_json -{ - private: - // also check the return type of from_json - template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json( - std::declval<BasicJsonType>(), std::declval<T&>()))>::value>> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype( - detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - -// This trait checks if JSONSerializer<T>::from_json(json const&) exists -// this overload is used for non-default-constructible user-defined-types -template<typename BasicJsonType, typename T> -struct has_non_default_from_json -{ - private: - template<typename U, typename = - enable_if_t<std::is_same<T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value>> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype(detect( - std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - -// This trait checks if BasicJsonType::json_serializer<T>::to_json exists -template<typename BasicJsonType, typename T> -struct has_to_json -{ - private: - template<typename U, typename = decltype(uncvref_t<U>::to_json( - std::declval<BasicJsonType&>(), std::declval<T>()))> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype(detect( - std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - - ///////////// // to_json // ///////////// @@ -1006,7 +1735,7 @@ void to_json(BasicJsonType& j, const CompatibleArrayType& arr) template<typename BasicJsonType, typename T, enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0> -void to_json(BasicJsonType& j, std::valarray<T> arr) +void to_json(BasicJsonType& j, const std::valarray<T>& arr) { external_constructor<value_t::array>::construct(j, std::move(arr)); } @@ -1043,271 +1772,24 @@ void to_json(BasicJsonType& j, const std::pair<Args...>& p) j = {p.first, p.second}; } -template<typename BasicJsonType, typename Tuple, std::size_t... Idx> -void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>) -{ - j = {std::get<Idx>(t)...}; -} - -template<typename BasicJsonType, typename... Args> -void to_json(BasicJsonType& j, const std::tuple<Args...>& t) -{ - to_json_tuple_impl(j, t, index_sequence_for<Args...> {}); -} - -/////////////// -// from_json // -/////////////// - -// overloads for basic_json template parameters -template<typename BasicJsonType, typename ArithmeticType, - enable_if_t<std::is_arithmetic<ArithmeticType>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, - int> = 0> -void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) -{ - switch (static_cast<value_t>(j)) - { - case value_t::number_unsigned: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); - break; - } - case value_t::number_integer: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); - break; - } - case value_t::number_float: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); - break; - } - - default: - JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); - } -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) -{ - if (JSON_UNLIKELY(not j.is_boolean())) - { - JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()))); - } - b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>(); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) -{ - if (JSON_UNLIKELY(not j.is_string())) - { - JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); - } - s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType, typename EnumType, - enable_if_t<std::is_enum<EnumType>::value, int> = 0> -void from_json(const BasicJsonType& j, EnumType& e) -{ - typename std::underlying_type<EnumType>::type val; - get_arithmetic_value(j, val); - e = static_cast<EnumType>(val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - arr = *j.template get_ptr<const typename BasicJsonType::array_t*>(); -} - -// forward_list doesn't have an insert method -template<typename BasicJsonType, typename T, typename Allocator, - enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> -void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - std::transform(j.rbegin(), j.rend(), - std::front_inserter(l), [](const BasicJsonType & i) - { - return i.template get<T>(); - }); -} - -// valarray doesn't have an insert method +// for https://github.com/nlohmann/json/pull/1134 template<typename BasicJsonType, typename T, - enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> -void from_json(const BasicJsonType& j, std::valarray<T>& l) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - l.resize(j.size()); - std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l)); -} - -template<typename BasicJsonType, typename CompatibleArrayType> -void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/) -{ - using std::end; - - std::transform(j.begin(), j.end(), - std::inserter(arr, end(arr)), [](const BasicJsonType & i) - { - // get<BasicJsonType>() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get<typename CompatibleArrayType::value_type>(); - }); -} - -template<typename BasicJsonType, typename CompatibleArrayType> -auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/) --> decltype( - arr.reserve(std::declval<typename CompatibleArrayType::size_type>()), - void()) -{ - using std::end; - - arr.reserve(j.size()); - std::transform(j.begin(), j.end(), - std::inserter(arr, end(arr)), [](const BasicJsonType & i) - { - // get<BasicJsonType>() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get<typename CompatibleArrayType::value_type>(); - }); -} - -template<typename BasicJsonType, typename T, std::size_t N> -void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/) -{ - for (std::size_t i = 0; i < N; ++i) - { - arr[i] = j.at(i).template get<T>(); - } -} - -template<typename BasicJsonType, typename CompatibleArrayType, - enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and - std::is_convertible<BasicJsonType, typename CompatibleArrayType::value_type>::value and - not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0> -void from_json(const BasicJsonType& j, CompatibleArrayType& arr) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - - from_json_array_impl(j, arr, priority_tag<2> {}); -} - -template<typename BasicJsonType, typename CompatibleObjectType, - enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0> -void from_json(const BasicJsonType& j, CompatibleObjectType& obj) -{ - if (JSON_UNLIKELY(not j.is_object())) - { - JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()))); - } - - auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>(); - using value_type = typename CompatibleObjectType::value_type; - std::transform( - inner_object->begin(), inner_object->end(), - std::inserter(obj, obj.begin()), - [](typename BasicJsonType::object_t::value_type const & p) - { - return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>()); - }); -} - -// overload for arithmetic types, not chosen for basic_json template arguments -// (BooleanType, etc..); note: Is it really necessary to provide explicit -// overloads for boolean_t etc. in case of a custom BooleanType which is not -// an arithmetic type? -template<typename BasicJsonType, typename ArithmeticType, - enable_if_t < - std::is_arithmetic<ArithmeticType>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, - int> = 0> -void from_json(const BasicJsonType& j, ArithmeticType& val) -{ - switch (static_cast<value_t>(j)) - { - case value_t::number_unsigned: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); - break; - } - case value_t::number_integer: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); - break; - } - case value_t::number_float: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); - break; - } - case value_t::boolean: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>()); - break; - } - - default: - JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); - } -} - -template<typename BasicJsonType, typename A1, typename A2> -void from_json(const BasicJsonType& j, std::pair<A1, A2>& p) + enable_if_t<std::is_same<T, typename iteration_proxy<typename BasicJsonType::iterator>::iteration_proxy_internal>::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept { - p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()}; + j = {{b.key(), b.value()}}; } template<typename BasicJsonType, typename Tuple, std::size_t... Idx> -void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>) +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>) { - t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...); + j = {std::get<Idx>(t)...}; } template<typename BasicJsonType, typename... Args> -void from_json(const BasicJsonType& j, std::tuple<Args...>& t) +void to_json(BasicJsonType& j, const std::tuple<Args...>& t) { - from_json_tuple_impl(j, t, index_sequence_for<Args...> {}); + to_json_tuple_impl(j, t, index_sequence_for<Args...> {}); } struct to_json_fn @@ -1342,49 +1824,38 @@ struct to_json_fn return call(j, std::forward<T>(val), priority_tag<1> {}); } }; +} -struct from_json_fn +/// namespace to hold default `to_json` function +namespace { - private: - template<typename BasicJsonType, typename T> - auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const - noexcept(noexcept(from_json(j, val))) - -> decltype(from_json(j, val), void()) - { - return from_json(j, val); - } +constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; +} +} - template<typename BasicJsonType, typename T> - void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept - { - static_assert(sizeof(BasicJsonType) == 0, - "could not find from_json() method in T's namespace"); -#ifdef _MSC_VER - // MSVC does not show a stacktrace for the above assert - using decayed = uncvref_t<T>; - static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, - "forcing MSVC stacktrace to show which T we're talking about."); -#endif - } +// #include <nlohmann/detail/input/input_adapters.hpp> - public: - template<typename BasicJsonType, typename T> - void operator()(const BasicJsonType& j, T& val) const - noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {}))) - { - return call(j, val, priority_tag<1> {}); - } -}; -// taken from ranges-v3 -template<typename T> -struct static_const -{ - static constexpr T value{}; -}; +#include <cassert> // assert +#include <cstddef> // size_t +#include <cstring> // strlen +#include <istream> // istream +#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include <memory> // shared_ptr, make_shared, addressof +#include <numeric> // accumulate +#include <string> // string, char_traits +#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer +#include <utility> // pair, declval + +// #include <nlohmann/detail/macro_scope.hpp> -template<typename T> -constexpr T static_const<T>::value; + +namespace nlohmann +{ +namespace detail +{ +/// the supported input formats +enum class input_format_t { json, cbor, msgpack, ubjson }; //////////////////// // input adapters // @@ -1394,19 +1865,17 @@ constexpr T static_const<T>::value; @brief abstract input adapter interface Produces a stream of std::char_traits<char>::int_type characters from a -std::istream, a buffer, or some other input type. Accepts the return of exactly -one non-EOF character for future input. The int_type characters returned -consist of all valid char values as positive values (typically unsigned char), -plus an EOF value outside that range, specified by the value of the function -std::char_traits<char>::eof(). This value is typically -1, but could be any -arbitrary value which is not a valid char value. +std::istream, a buffer, or some other input type. Accepts the return of +exactly one non-EOF character for future input. The int_type characters +returned consist of all valid char values as positive values (typically +unsigned char), plus an EOF value outside that range, specified by the value +of the function std::char_traits<char>::eof(). This value is typically -1, but +could be any arbitrary value which is not a valid char value. */ struct input_adapter_protocol { /// get a character [0,255] or std::char_traits<char>::eof(). virtual std::char_traits<char>::int_type get_character() = 0; - /// restore the last non-eof() character to input - virtual void unget_character() = 0; virtual ~input_adapter_protocol() = default; }; @@ -1434,34 +1903,7 @@ class input_stream_adapter : public input_adapter_protocol explicit input_stream_adapter(std::istream& i) : is(i), sb(*i.rdbuf()) - { - // skip byte order mark - std::char_traits<char>::int_type c; - if ((c = get_character()) == 0xEF) - { - if ((c = get_character()) == 0xBB) - { - if ((c = get_character()) == 0xBF) - { - return; // Ignore BOM - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); - } - is.putback('\xBB'); - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); - } - is.putback('\xEF'); - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); // no byte order mark; process as usual - } - } + {} // delete because of pointer members input_stream_adapter(const input_stream_adapter&) = delete; @@ -1475,11 +1917,6 @@ class input_stream_adapter : public input_adapter_protocol return sb.sbumpc(); } - void unget_character() override - { - sb.sungetc(); // is.unget() avoided for performance - } - private: /// the associated input stream std::istream& is; @@ -1491,14 +1928,8 @@ class input_buffer_adapter : public input_adapter_protocol { public: input_buffer_adapter(const char* b, const std::size_t l) - : cursor(b), limit(b + l), start(b) - { - // skip byte order mark - if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF') - { - cursor += 3; - } - } + : cursor(b), limit(b + l) + {} // delete because of pointer members input_buffer_adapter(const input_buffer_adapter&) = delete; @@ -1514,21 +1945,164 @@ class input_buffer_adapter : public input_adapter_protocol return std::char_traits<char>::eof(); } - void unget_character() noexcept override + private: + /// pointer to the current character + const char* cursor; + /// pointer past the last character + const char* const limit; +}; + +template<typename WideStringType> +class wide_string_input_adapter : public input_adapter_protocol +{ + public: + explicit wide_string_input_adapter(const WideStringType& w) : str(w) {} + + std::char_traits<char>::int_type get_character() noexcept override { - if (JSON_LIKELY(cursor > start)) + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) { - --cursor; + if (sizeof(typename WideStringType::value_type) == 2) + { + fill_buffer_utf16(); + } + else + { + fill_buffer_utf32(); + } + + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index == 0); } + + // use buffer + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; } private: - /// pointer to the current character - const char* cursor; - /// pointer past the last character - const char* limit; - /// pointer to the first character - const char* start; + void fill_buffer_utf16() + { + utf8_bytes_index = 0; + + if (current_wchar == str.size()) + { + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const int wc = static_cast<int>(str[current_wchar++]); + + // UTF-16 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6)); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (0xD800 > wc or wc >= 0xE000) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12)); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else + { + if (current_wchar < str.size()) + { + const int wc2 = static_cast<int>(str[current_wchar++]); + const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF)); + utf8_bytes[0] = 0xf0 | (charcode >> 18); + utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (charcode & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + ++current_wchar; + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + } + } + } + + void fill_buffer_utf32() + { + utf8_bytes_index = 0; + + if (current_wchar == str.size()) + { + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const int wc = static_cast<int>(str[current_wchar++]); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = 0xF0 | ((wc >> 18 ) & 0x07); + utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + } + } + + private: + /// the wstring to process + const WideStringType& str; + + /// index of the current wchar in str + std::size_t current_wchar = 0; + + /// a buffer for UTF-8 bytes + std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}}; + + /// index to the utf8_codes array for the next valid byte + std::size_t utf8_bytes_index = 0; + /// number of valid bytes in the utf8_codes array + std::size_t utf8_bytes_filled = 0; }; class input_adapter @@ -1544,6 +2118,15 @@ class input_adapter input_adapter(std::istream&& i) : ia(std::make_shared<input_stream_adapter>(i)) {} + input_adapter(const std::wstring& ws) + : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {} + + input_adapter(const std::u16string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {} + + input_adapter(const std::u32string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {} + /// input adapter for buffer template<typename CharT, typename std::enable_if< @@ -1624,10 +2207,32 @@ class input_adapter /// the actual adapter input_adapter_t ia = nullptr; }; +} +} -////////////////////// -// lexer and parser // -////////////////////// +// #include <nlohmann/detail/input/lexer.hpp> + + +#include <clocale> // localeconv +#include <cstddef> // size_t +#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull +#include <cstdio> // snprintf +#include <initializer_list> // initializer_list +#include <string> // char_traits, string +#include <vector> // vector + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/input/input_adapters.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////// +// lexer // +/////////// /*! @brief lexical analysis @@ -1640,6 +2245,7 @@ class lexer using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; public: /// token types for the parser @@ -1701,12 +2307,14 @@ class lexer return "end of input"; case token_type::literal_or_value: return "'[', '{', or a literal"; + // LCOV_EXCL_START default: // catch non-enum values - return "unknown token"; // LCOV_EXCL_LINE + return "unknown token"; + // LCOV_EXCL_STOP } } - explicit lexer(detail::input_adapter_t adapter) + explicit lexer(detail::input_adapter_t&& adapter) : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {} // delete because of pointer members @@ -1819,9 +2427,10 @@ class lexer @brief scan a string literal This function scans a string according to Sect. 7 of RFC 7159. While - scanning, bytes are escaped and copied into buffer yytext. Then the function - returns successfully, yytext is *not* null-terminated (as it may contain \0 - bytes), and yytext.size() is the number of bytes in the string. + scanning, bytes are escaped and copied into buffer token_buffer. Then the + function returns successfully, token_buffer is *not* null-terminated (as it + may contain \0 bytes), and token_buffer.size() is the number of bytes in the + string. @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @@ -1831,7 +2440,7 @@ class lexer */ token_type scan_string() { - // reset yytext (ignore opening quote) + // reset token_buffer (ignore opening quote) reset(); // we entered the function by reading an open quote @@ -2303,7 +2912,7 @@ class lexer contains cycles, but any cycle can be left when EOF is read. Therefore, the function is guaranteed to terminate. - During scanning, the read bytes are stored in yytext. This string is + During scanning, the read bytes are stored in token_buffer. This string is then converted to a signed integer, an unsigned integer, or a floating-point number. @@ -2317,7 +2926,7 @@ class lexer */ token_type scan_number() { - // reset yytext to store the number's bytes + // reset token_buffer to store the number's bytes reset(); // the type of the parsed number; initially set to unsigned; will be @@ -2353,11 +2962,13 @@ class lexer goto scan_number_any1; } + // LCOV_EXCL_START default: { // all other characters are rejected outside scan_number() - assert(false); // LCOV_EXCL_LINE + assert(false); } + // LCOV_EXCL_STOP } scan_number_minus: @@ -2601,10 +3212,10 @@ scan_number_done: // try to parse integers first and fall back to floats if (number_type == token_type::value_unsigned) { - const auto x = std::strtoull(yytext.data(), &endptr, 10); + const auto x = std::strtoull(token_buffer.data(), &endptr, 10); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); if (errno == 0) { @@ -2617,10 +3228,10 @@ scan_number_done: } else if (number_type == token_type::value_integer) { - const auto x = std::strtoll(yytext.data(), &endptr, 10); + const auto x = std::strtoll(token_buffer.data(), &endptr, 10); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); if (errno == 0) { @@ -2634,10 +3245,10 @@ scan_number_done: // this code is reached if we parse a floating-point number or if an // integer conversion above failed - strtof(value_float, yytext.data(), &endptr); + strtof(value_float, token_buffer.data(), &endptr); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); return token_type::value_float; } @@ -2666,10 +3277,10 @@ scan_number_done: // input management ///////////////////// - /// reset yytext; current character is beginning of token + /// reset token_buffer; current character is beginning of token void reset() noexcept { - yytext.clear(); + token_buffer.clear(); token_string.clear(); token_string.push_back(std::char_traits<char>::to_char_type(current)); } @@ -2687,7 +3298,16 @@ scan_number_done: std::char_traits<char>::int_type get() { ++chars_read; - current = ia->get_character(); + if (next_unget) + { + // just reset the next_unget variable and work with current + next_unget = false; + } + else + { + current = ia->get_character(); + } + if (JSON_LIKELY(current != std::char_traits<char>::eof())) { token_string.push_back(std::char_traits<char>::to_char_type(current)); @@ -2695,22 +3315,29 @@ scan_number_done: return current; } - /// unget current character (return it again on next get) + /*! + @brief unget current character (read it again on next get) + + We implement unget by setting variable next_unget to true. The input is not + changed - we just simulate ungetting by modifying chars_read and + token_string. The next call to get() will behave as if the unget character + is read again. + */ void unget() { + next_unget = true; --chars_read; if (JSON_LIKELY(current != std::char_traits<char>::eof())) { - ia->unget_character(); assert(token_string.size() != 0); token_string.pop_back(); } } - /// add a character to yytext + /// add a character to token_buffer void add(int c) { - yytext.push_back(std::char_traits<char>::to_char_type(c)); + token_buffer.push_back(std::char_traits<char>::to_char_type(c)); } public: @@ -2737,9 +3364,9 @@ scan_number_done: } /// return current string value (implicitly resets the token; useful only once) - std::string move_string() + string_t& get_string() { - return std::move(yytext); + return token_buffer; } ///////////////////// @@ -2764,10 +3391,9 @@ scan_number_done: if ('\x00' <= c and c <= '\x1F') { // escape control characters - std::stringstream ss; - ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0') - << std::hex << static_cast<int>(c) << ">"; - result += ss.str(); + char cs[9]; + snprintf(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c)); + result += cs; } else { @@ -2789,8 +3415,43 @@ scan_number_done: // actual scanner ///////////////////// + /*! + @brief skip the UTF-8 byte order mark + @return true iff there is no BOM or the correct BOM has been skipped + */ + bool skip_bom() + { + if (get() == 0xEF) + { + if (get() == 0xBB and get() == 0xBF) + { + // we completely parsed the BOM + return true; + } + else + { + // after reading 0xEF, an unexpected character followed + return false; + } + } + else + { + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; + } + } + token_type scan() { + // initially, skip the BOM + if (chars_read == 0 and not skip_bom()) + { + error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; + return token_type::parse_error; + } + // read next character and ignore whitespace do { @@ -2860,6 +3521,9 @@ scan_number_done: /// the current character std::char_traits<char>::int_type current = std::char_traits<char>::eof(); + /// whether the next get() call should just return current + bool next_unget = false; + /// the number of characters read std::size_t chars_read = 0; @@ -2867,7 +3531,7 @@ scan_number_done: std::vector<char> token_string {}; /// buffer for variable-length tokens (numbers, strings) - std::string yytext {}; + string_t token_buffer {}; /// a description of occurred lexer errors const char* error_message = ""; @@ -2880,6 +3544,955 @@ scan_number_done: /// the decimal point const char decimal_point_char = '.'; }; +} +} + +// #include <nlohmann/detail/input/parser.hpp> + + +#include <cassert> // assert +#include <cmath> // isfinite +#include <cstdint> // uint8_t +#include <functional> // function +#include <string> // string +#include <utility> // move + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + + +#include <cstdint> // size_t +#include <utility> // declval + +// #include <nlohmann/detail/meta/detected.hpp> + + +#include <type_traits> + +// #include <nlohmann/detail/meta/void_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename...> +using void_t = void; +} +} + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + void operator=(nonesuch const&) = delete; +}; + +template <class Default, + class AlwaysVoid, + template <class...> class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template <class Default, template <class...> class Op, class... Args> +struct detector<Default, void_t<Op<Args...>>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op<Args...>; +}; + +template <template <class...> class Op, class... Args> +using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t; + +template <template <class...> class Op, class... Args> +using detected_t = typename detector<nonesuch, void, Op, Args...>::type; + +template <class Default, template <class...> class Op, class... Args> +using detected_or = detector<Default, void, Op, Args...>; + +template <class Default, template <class...> class Op, class... Args> +using detected_or_t = typename detected_or<Default, Op, Args...>::type; + +template <class Expected, template <class...> class Op, class... Args> +using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>; + +template <class To, template <class...> class Op, class... Args> +using is_detected_convertible = + std::is_convertible<detected_t<Op, Args...>, To>; +} +} + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename T> +using null_function_t = decltype(std::declval<T&>().null()); + +template <typename T> +using boolean_function_t = + decltype(std::declval<T&>().boolean(std::declval<bool>())); + +template <typename T, typename Integer> +using number_integer_function_t = + decltype(std::declval<T&>().number_integer(std::declval<Integer>())); + +template <typename T, typename Unsigned> +using number_unsigned_function_t = + decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>())); + +template <typename T, typename Float, typename String> +using number_float_function_t = decltype(std::declval<T&>().number_float( + std::declval<Float>(), std::declval<const String&>())); + +template <typename T, typename String> +using string_function_t = + decltype(std::declval<T&>().string(std::declval<String&>())); + +template <typename T> +using start_object_function_t = + decltype(std::declval<T&>().start_object(std::declval<std::size_t>())); + +template <typename T, typename String> +using key_function_t = + decltype(std::declval<T&>().key(std::declval<String&>())); + +template <typename T> +using end_object_function_t = decltype(std::declval<T&>().end_object()); + +template <typename T> +using start_array_function_t = + decltype(std::declval<T&>().start_array(std::declval<std::size_t>())); + +template <typename T> +using end_array_function_t = decltype(std::declval<T&>().end_array()); + +template <typename T, typename Exception> +using parse_error_function_t = decltype(std::declval<T&>().parse_error( + std::declval<std::size_t>(), std::declval<const std::string&>(), + std::declval<const Exception&>())); + +template <typename SAX, typename BasicJsonType> +struct is_sax +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact<bool, null_function_t, SAX>::value && + is_detected_exact<bool, boolean_function_t, SAX>::value && + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value && + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value && + is_detected_exact<bool, number_float_function_t, SAX, number_float_t, + string_t>::value && + is_detected_exact<bool, string_function_t, SAX, string_t>::value && + is_detected_exact<bool, start_object_function_t, SAX>::value && + is_detected_exact<bool, key_function_t, SAX, string_t>::value && + is_detected_exact<bool, end_object_function_t, SAX>::value && + is_detected_exact<bool, start_array_function_t, SAX>::value && + is_detected_exact<bool, end_array_function_t, SAX>::value && + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value; +}; + +template <typename SAX, typename BasicJsonType> +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact<bool, null_function_t, SAX>::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact<bool, number_float_function_t, SAX, + number_float_t, string_t>::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact<bool, string_function_t, SAX, string_t>::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} +} + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + + +#include <cstddef> +#include <string> +#include <vector> + +// #include <nlohmann/detail/input/parser.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template<typename BasicJsonType> +struct json_sax +{ + /// type for (signed) integers + using number_integer_t = typename BasicJsonType::number_integer_t; + /// type for unsigned integers + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type for floating-point numbers + using number_float_t = typename BasicJsonType::number_float_t; + /// type for strings + using string_t = typename BasicJsonType::string_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief an floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] error_msg a detailed error message + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template<typename BasicJsonType> +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + /*! + @param[in, out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t&) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + ref_stack.pop_back(); + return true; + } + + bool parse_error(std::size_t, const std::string&, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template<typename Value> + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward<Value>(v)); + return &root; + } + else + { + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v)); + return &(ref_stack.back()->m_value.array->back()); + } + else + { + assert(object_element); + *object_element = BasicJsonType(std::forward<Value>(v)); + return object_element; + } + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template<typename BasicJsonType> +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t&) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep and ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back()) + { + if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (not ref_stack.empty() and ref_stack.back()) + { + // remove discarded value + if (ref_stack.back()->is_object()) + { + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + // discard array + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (not keep and not ref_stack.empty()) + { + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + } + + return true; + } + + bool parse_error(std::size_t, const std::string&, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template<typename Value> + std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false) + { + assert(not keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (not keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward<Value>(v)); + + // check callback + const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (not keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + else + { + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (not ref_stack.back()) + { + return {false, nullptr}; + } + + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->push_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + else + { + // check if we should store an element for the current key + assert(not key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (not store_element) + { + return {false, nullptr}; + } + + assert(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// stack to manage which values to keep + std::vector<bool> keep_stack; + /// stack to manage which object keys to keep + std::vector<bool> key_keep_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template<typename BasicJsonType> +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + bool null() + { + return true; + } + + bool boolean(bool) + { + return true; + } + + bool number_integer(number_integer_t) + { + return true; + } + + bool number_unsigned(number_unsigned_t) + { + return true; + } + + bool number_float(number_float_t, const string_t&) + { + return true; + } + + bool string(string_t&) + { + return true; + } + + bool start_object(std::size_t = std::size_t(-1)) + { + return true; + } + + bool key(string_t&) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t = std::size_t(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t, const std::string&, const detail::exception&) + { + return false; + } +}; +} + +} + +// #include <nlohmann/detail/input/lexer.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +//////////// +// parser // +//////////// /*! @brief syntax analysis @@ -2892,6 +4505,7 @@ class parser using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; using lexer_t = lexer<BasicJsonType>; using token_type = typename lexer_t::token_type; @@ -2916,11 +4530,14 @@ class parser std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>; /// a parser reading from an input adapter - explicit parser(detail::input_adapter_t adapter, + explicit parser(detail::input_adapter_t&& adapter, const parser_callback_t cb = nullptr, const bool allow_exceptions_ = true) - : callback(cb), m_lexer(adapter), allow_exceptions(allow_exceptions_) - {} + : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_) + { + // read first token + get_token(); + } /*! @brief public parser interface @@ -2934,31 +4551,54 @@ class parser */ void parse(const bool strict, BasicJsonType& result) { - // read first token - get_token(); + if (callback) + { + json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); - parse_internal(true, result); - result.assert_invariant(); + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); + } - // in strict mode, input must be completely read - if (strict) - { - get_token(); - expect(token_type::end_of_input); - } + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } - // in case of an error, return discarded value - if (errored) - { - result = value_t::discarded; - return; + // set top-level value to null if it was discarded by the callback + // function + if (result.is_discarded()) + { + result = nullptr; + } } - - // set top-level value to null if it was discarded by the callback - // function - if (result.is_discarded()) + else { - result = nullptr; + json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); + + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } } } @@ -2970,413 +4610,311 @@ class parser */ bool accept(const bool strict = true) { - // read first token - get_token(); + json_sax_acceptor<BasicJsonType> sax_acceptor; + return sax_parse(&sax_acceptor, strict); + } - if (not accept_internal()) + template <typename SAX> + bool sax_parse(SAX* sax, const bool strict = true) + { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; + const bool result = sax_parse_internal(sax); + + // strict mode: next byte must be EOF + if (result and strict and (get_token() != token_type::end_of_input)) { - return false; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); } - // strict => last token must be EOF - return not strict or (get_token() == token_type::end_of_input); + return result; } private: - /*! - @brief the actual parser - @throw parse_error.101 in case of an unexpected token - @throw parse_error.102 if to_unicode fails or surrogate error - @throw parse_error.103 if to_unicode fails - */ - void parse_internal(bool keep, BasicJsonType& result) + template <typename SAX> + bool sax_parse_internal(SAX* sax) { - // never parse after a parse error was detected - assert(not errored); - - // start with a discarded value - if (not result.is_discarded()) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } + // stack to remember the hieararchy of structured values we are parsing + // true = array; false = object + std::vector<bool> states; + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; - switch (last_token) + while (true) { - case token_type::begin_object: + if (not skip_to_state_evaluation) { - if (keep) + // invariant: get_token() was called before each iteration + switch (last_token) { - if (callback) - { - keep = callback(depth++, parse_event_t::object_start, result); - } - - if (not callback or keep) + case token_type::begin_object: { - // explicitly set result to object to cope with {} - result.m_type = value_t::object; - result.m_value = value_t::object; - } - } - - // read next token - get_token(); - - // closing } -> we are done - if (last_token == token_type::end_object) - { - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - break; - } + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } - // parse values - std::string key; - BasicJsonType value; - while (true) - { - // store key - if (not expect(token_type::value_string)) - { - return; - } - key = m_lexer.move_string(); + // closing } -> we are done + if (get_token() == token_type::end_object) + { + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } + break; + } - bool keep_tag = false; - if (keep) - { - if (callback) + // parse key + if (JSON_UNLIKELY(last_token != token_type::value_string)) { - BasicJsonType k(key); - keep_tag = callback(depth, parse_event_t::key, k); + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string))); } else { - keep_tag = true; + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } } - } - // parse separator (:) - get_token(); - if (not expect(token_type::name_separator)) - { - return; - } + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator))); + } - // parse and add value - get_token(); - value.m_value.destroy(value.m_type); - value.m_type = value_t::discarded; - parse_internal(keep, value); + // remember we are now inside an object + states.push_back(false); - if (JSON_UNLIKELY(errored)) - { - return; + // parse values + get_token(); + continue; } - if (keep and keep_tag and not value.is_discarded()) + case token_type::begin_array: { - result.m_value.object->emplace(std::move(key), std::move(value)); - } + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) - { - get_token(); + // closing ] -> we are done + if (get_token() == token_type::end_array) + { + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } + break; + } + + // remember we are now inside an array + states.push_back(true); + + // parse values (no need to call get_token) continue; } - // closing } - if (not expect(token_type::end_object)) + case token_type::value_float: { - return; - } - break; - } + const auto res = m_lexer.get_number_float(); - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - break; - } + if (JSON_UNLIKELY(not std::isfinite(res))) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'")); + } + else + { + if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) + { + return false; + } + break; + } + } - case token_type::begin_array: - { - if (keep) - { - if (callback) + case token_type::literal_false: { - keep = callback(depth++, parse_event_t::array_start, result); + if (JSON_UNLIKELY(not sax->boolean(false))) + { + return false; + } + break; } - if (not callback or keep) + case token_type::literal_null: { - // explicitly set result to array to cope with [] - result.m_type = value_t::array; - result.m_value = value_t::array; + if (JSON_UNLIKELY(not sax->null())) + { + return false; + } + break; } - } - // read next token - get_token(); - - // closing ] -> we are done - if (last_token == token_type::end_array) - { - if (callback and not callback(--depth, parse_event_t::array_end, result)) + case token_type::literal_true: { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; + if (JSON_UNLIKELY(not sax->boolean(true))) + { + return false; + } + break; } - break; - } - - // parse values - BasicJsonType value; - while (true) - { - // parse value - value.m_value.destroy(value.m_type); - value.m_type = value_t::discarded; - parse_internal(keep, value); - if (JSON_UNLIKELY(errored)) + case token_type::value_integer: { - return; + if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer()))) + { + return false; + } + break; } - if (keep and not value.is_discarded()) + case token_type::value_string: { - result.m_value.array->push_back(std::move(value)); + if (JSON_UNLIKELY(not sax->string(m_lexer.get_string()))) + { + return false; + } + break; } - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) + case token_type::value_unsigned: { - get_token(); - continue; + if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned()))) + { + return false; + } + break; } - // closing ] - if (not expect(token_type::end_array)) + case token_type::parse_error: { - return; + // using "uninitialized" to avoid "expected" message + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized))); } - break; - } - - if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - break; - } - - case token_type::literal_null: - { - result.m_type = value_t::null; - break; - } - - case token_type::value_string: - { - result.m_type = value_t::string; - result.m_value = m_lexer.move_string(); - break; - } - - case token_type::literal_true: - { - result.m_type = value_t::boolean; - result.m_value = true; - break; - } - case token_type::literal_false: - { - result.m_type = value_t::boolean; - result.m_value = false; - break; - } - - case token_type::value_unsigned: - { - result.m_type = value_t::number_unsigned; - result.m_value = m_lexer.get_number_unsigned(); - break; - } - - case token_type::value_integer: - { - result.m_type = value_t::number_integer; - result.m_value = m_lexer.get_number_integer(); - break; - } - - case token_type::value_float: - { - result.m_type = value_t::number_float; - result.m_value = m_lexer.get_number_float(); - - // throw in case of infinity or NAN - if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float))) - { - if (allow_exceptions) + default: // the last token was unexpected { - JSON_THROW(out_of_range::create(406, "number overflow parsing '" + - m_lexer.get_token_string() + "'")); + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value))); } - expect(token_type::uninitialized); } - break; } - - case token_type::parse_error: + else { - // using "uninitialized" to avoid "expected" message - if (not expect(token_type::uninitialized)) - { - return; - } - break; // LCOV_EXCL_LINE + skip_to_state_evaluation = false; } - default: + // we reached this line after we successfully parsed a value + if (states.empty()) { - // the last token was unexpected; we expected a value - if (not expect(token_type::literal_or_value)) - { - return; - } - break; // LCOV_EXCL_LINE + // empty stack: we reached the end of the hieararchy: done + return true; } - } - - if (keep and callback and not callback(depth, parse_event_t::value, result)) - { - result.m_type = value_t::discarded; - } - } - - /*! - @brief the actual acceptor - - @invariant 1. The last token is not yet processed. Therefore, the caller - of this function must make sure a token has been read. - 2. When this function returns, the last token is processed. - That is, the last read character was already considered. - - This invariant makes sure that no token needs to be "unput". - */ - bool accept_internal() - { - switch (last_token) - { - case token_type::begin_object: + else { - // read next token - get_token(); - - // closing } -> we are done - if (last_token == token_type::end_object) + if (states.back()) // array { - return true; - } - - // parse values - while (true) - { - // parse key - if (last_token != token_type::value_string) + // comma -> next value + if (get_token() == token_type::value_separator) { - return false; + // parse a new value + get_token(); + continue; } - // parse separator (:) - get_token(); - if (last_token != token_type::name_separator) + // closing ] + if (JSON_LIKELY(last_token == token_type::end_array)) { - return false; - } + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } - // parse value - get_token(); - if (not accept_internal()) + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + else { - return false; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array))); } - + } + else // object + { // comma -> next value - get_token(); - if (last_token == token_type::value_separator) + if (get_token() == token_type::value_separator) { + // parse key + if (JSON_UNLIKELY(get_token() != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string))); + } + else + { + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } + } + + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator))); + } + + // parse values get_token(); continue; } // closing } - return (last_token == token_type::end_object); - } - } - - case token_type::begin_array: - { - // read next token - get_token(); - - // closing ] -> we are done - if (last_token == token_type::end_array) - { - return true; - } - - // parse values - while (true) - { - // parse value - if (not accept_internal()) + if (JSON_LIKELY(last_token == token_type::end_object)) { - return false; - } + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) - { - get_token(); + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; continue; } - - // closing ] - return (last_token == token_type::end_array); + else + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object))); + } } } - - case token_type::value_float: - { - // reject infinity or NAN - return std::isfinite(m_lexer.get_number_float()); - } - - case token_type::literal_false: - case token_type::literal_null: - case token_type::literal_true: - case token_type::value_integer: - case token_type::value_string: - case token_type::value_unsigned: - return true; - - default: // the last token was unexpected - return false; } } @@ -3386,29 +4924,7 @@ class parser return (last_token = m_lexer.scan()); } - /*! - @throw parse_error.101 if expected token did not occur - */ - bool expect(token_type t) - { - if (JSON_UNLIKELY(t != last_token)) - { - errored = true; - expected = t; - if (allow_exceptions) - { - throw_exception(); - } - else - { - return false; - } - } - - return true; - } - - [[noreturn]] void throw_exception() const + std::string exception_message(const token_type expected) { std::string error_msg = "syntax error - "; if (last_token == token_type::parse_error) @@ -3426,31 +4942,33 @@ class parser error_msg += "; expected " + std::string(lexer_t::token_type_name(expected)); } - JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); + return error_msg; } private: - /// current level of recursion - int depth = 0; /// callback function const parser_callback_t callback = nullptr; /// the type of the last read token token_type last_token = token_type::uninitialized; /// the lexer lexer_t m_lexer; - /// whether a syntax error occurred - bool errored = false; - /// possible reason for the syntax error - token_type expected = token_type::uninitialized; /// whether to throw exceptions in case of errors const bool allow_exceptions = true; }; +} +} -/////////////// -// iterators // -/////////////// +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> -/*! + +#include <cstddef> // ptrdiff_t +#include <limits> // numeric_limits + +namespace nlohmann +{ +namespace detail +{ +/* @brief an iterator for primitive JSON types This class models an iterator for primitive JSON types (boolean, number, @@ -3461,9 +4979,15 @@ end_value (`1`) models past the end. */ class primitive_iterator_t { - public: + private: using difference_type = std::ptrdiff_t; + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + /// iterator as signed integer type + difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)(); + public: constexpr difference_type get_value() const noexcept { return m_it; @@ -3503,10 +5027,10 @@ class primitive_iterator_t return lhs.m_it < rhs.m_it; } - primitive_iterator_t operator+(difference_type i) + primitive_iterator_t operator+(difference_type n) noexcept { auto result = *this; - result += i; + result += n; return result; } @@ -3515,57 +5039,57 @@ class primitive_iterator_t return lhs.m_it - rhs.m_it; } - friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it) - { - return os << it.m_it; - } - - primitive_iterator_t& operator++() + primitive_iterator_t& operator++() noexcept { ++m_it; return *this; } - primitive_iterator_t const operator++(int) + primitive_iterator_t const operator++(int) noexcept { auto result = *this; - m_it++; + ++m_it; return result; } - primitive_iterator_t& operator--() + primitive_iterator_t& operator--() noexcept { --m_it; return *this; } - primitive_iterator_t const operator--(int) + primitive_iterator_t const operator--(int) noexcept { auto result = *this; - m_it--; + --m_it; return result; } - primitive_iterator_t& operator+=(difference_type n) + primitive_iterator_t& operator+=(difference_type n) noexcept { m_it += n; return *this; } - primitive_iterator_t& operator-=(difference_type n) + primitive_iterator_t& operator-=(difference_type n) noexcept { m_it -= n; return *this; } +}; +} +} - private: - static constexpr difference_type begin_value = 0; - static constexpr difference_type end_value = begin_value + 1; +// #include <nlohmann/detail/iterators/internal_iterator.hpp> - /// iterator as signed integer type - difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)(); -}; +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> + + +namespace nlohmann +{ +namespace detail +{ /*! @brief an iterator value @@ -3581,7 +5105,34 @@ template<typename BasicJsonType> struct internal_iterator /// generic iterator for all other types primitive_iterator_t primitive_iterator {}; }; +} +} + +// #include <nlohmann/detail/iterators/iter_impl.hpp> + + +#include <ciso646> // not +#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next +#include <type_traits> // conditional, is_const, remove_const + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/iterators/internal_iterator.hpp> +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +// forward declare, to be able to friend it later on template<typename IteratorType> class iteration_proxy; /*! @@ -3597,7 +5148,7 @@ This class implements a both iterators (iterator and const_iterator) for the @requirement The class satisfies the following concept requirements: - -[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator): +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): The iterator that can be moved can be moved in both directions (i.e. incremented and decremented). @@ -4149,7 +5700,7 @@ class iter_impl @brief return the key of an object iterator @pre The iterator is initialized; i.e. `m_object != nullptr`. */ - typename object_t::key_type key() const + const typename object_t::key_type& key() const { assert(m_object != nullptr); @@ -4174,94 +5725,27 @@ class iter_impl /// associated JSON instance pointer m_object = nullptr; /// the actual iterator of the associated instance - internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it = {}; + internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it; }; +} +} -/// proxy class for the iterator_wrapper functions -template<typename IteratorType> class iteration_proxy -{ - private: - /// helper class for iteration - class iteration_proxy_internal - { - private: - /// the iterator - IteratorType anchor; - /// an index for arrays (used to create key names) - std::size_t array_index = 0; - - public: - explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {} - - /// dereference operator (needed for range-based for) - iteration_proxy_internal& operator*() - { - return *this; - } - - /// increment operator (needed for range-based for) - iteration_proxy_internal& operator++() - { - ++anchor; - ++array_index; - - return *this; - } - - /// inequality operator (needed for range-based for) - bool operator!=(const iteration_proxy_internal& o) const noexcept - { - return anchor != o.anchor; - } - - /// return key of the iterator - std::string key() const - { - assert(anchor.m_object != nullptr); - - switch (anchor.m_object->type()) - { - // use integer array index as key - case value_t::array: - return std::to_string(array_index); - - // use key from the object - case value_t::object: - return anchor.key(); - - // use an empty key for all primitive types - default: - return ""; - } - } - - /// return value of the iterator - typename IteratorType::reference value() const - { - return anchor.value(); - } - }; +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> - /// the container to iterate - typename IteratorType::reference container; +// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp> - public: - /// construct iteration proxy from a container - explicit iteration_proxy(typename IteratorType::reference cont) - : container(cont) {} - /// return iterator begin (needed for range-based for) - iteration_proxy_internal begin() noexcept - { - return iteration_proxy_internal(container.begin()); - } +#include <cstddef> // ptrdiff_t +#include <iterator> // reverse_iterator +#include <utility> // declval - /// return iterator end (needed for range-based for) - iteration_proxy_internal end() noexcept - { - return iteration_proxy_internal(container.end()); - } -}; +namespace nlohmann +{ +namespace detail +{ +////////////////////// +// reverse_iterator // +////////////////////// /*! @brief a template for a reverse iterator class @@ -4272,10 +5756,10 @@ create @ref const_reverse_iterator). @requirement The class satisfies the following concept requirements: - -[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator): +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): The iterator that can be moved can be moved in both directions (i.e. incremented and decremented). -- [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): +- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): It is possible to write to the pointed-to element (only if @a Base is @ref iterator). @@ -4292,11 +5776,11 @@ class json_reverse_iterator : public std::reverse_iterator<Base> using reference = typename Base::reference; /// create reverse iterator from iterator - json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept : base_iterator(it) {} /// create reverse iterator from base class - json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} + explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} /// post-increment (it++) json_reverse_iterator const operator++(int) @@ -4366,11 +5850,25 @@ class json_reverse_iterator : public std::reverse_iterator<Base> return it.operator * (); } }; +} +} -///////////////////// -// output adapters // -///////////////////// +// #include <nlohmann/detail/output/output_adapters.hpp> + +#include <algorithm> // copy +#include <cstddef> // size_t +#include <ios> // streamsize +#include <iterator> // back_inserter +#include <memory> // shared_ptr, make_shared +#include <ostream> // basic_ostream +#include <string> // basic_string +#include <vector> // vector + +namespace nlohmann +{ +namespace detail +{ /// abstract output adapter interface template<typename CharType> struct output_adapter_protocol { @@ -4426,11 +5924,11 @@ class output_stream_adapter : public output_adapter_protocol<CharType> }; /// output adapter for basic_string -template<typename CharType> +template<typename CharType, typename StringType = std::basic_string<CharType>> class output_string_adapter : public output_adapter_protocol<CharType> { public: - explicit output_string_adapter(std::basic_string<CharType>& s) : str(s) {} + explicit output_string_adapter(StringType& s) : str(s) {} void write_character(CharType c) override { @@ -4443,10 +5941,10 @@ class output_string_adapter : public output_adapter_protocol<CharType> } private: - std::basic_string<CharType>& str; + StringType& str; }; -template<typename CharType> +template<typename CharType, typename StringType = std::basic_string<CharType>> class output_adapter { public: @@ -4456,8 +5954,8 @@ class output_adapter output_adapter(std::basic_ostream<CharType>& s) : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {} - output_adapter(std::basic_string<CharType>& s) - : oa(std::make_shared<output_string_adapter<CharType>>(s)) {} + output_adapter(StringType& s) + : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {} operator output_adapter_t<CharType>() { @@ -4467,19 +5965,57 @@ class output_adapter private: output_adapter_t<CharType> oa = nullptr; }; +} +} + +// #include <nlohmann/detail/input/binary_reader.hpp> + + +#include <algorithm> // generate_n +#include <array> // array +#include <cassert> // assert +#include <cmath> // ldexp +#include <cstddef> // size_t +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstdio> // snprintf +#include <cstring> // memcpy +#include <iterator> // back_inserter +#include <limits> // numeric_limits +#include <string> // char_traits, string +#include <utility> // make_pair, move -////////////////////////////// -// binary reader and writer // -////////////////////////////// +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary reader // +/////////////////// /*! -@brief deserialization of CBOR and MessagePack values +@brief deserialization of CBOR, MessagePack, and UBJSON values */ -template<typename BasicJsonType> +template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>> class binary_reader { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using json_sax_t = SAX; public: /*! @@ -4489,49 +6025,63 @@ class binary_reader */ explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; assert(ia); } /*! - @brief create a JSON value from CBOR input - + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor @param[in] strict whether to expect the input to be consumed completed - @return JSON value created from CBOR input - @throw parse_error.110 if input ended unexpectedly or the end of file was - not reached when @a strict was set to true - @throw parse_error.112 if unsupported byte was read + @return */ - BasicJsonType parse_cbor(const bool strict) + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true) { - const auto res = parse_cbor_internal(); - if (strict) + sax = sax_; + bool result = false; + + switch (format) { - get(); - check_eof(true); - } - return res; - } + case input_format_t::cbor: + result = parse_cbor_internal(); + break; - /*! - @brief create a JSON value from MessagePack input + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; - @param[in] strict whether to expect the input to be consumed completed - @return JSON value created from MessagePack input + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; - @throw parse_error.110 if input ended unexpectedly or the end of file was - not reached when @a strict was set to true - @throw parse_error.112 if unsupported byte was read - */ - BasicJsonType parse_msgpack(const bool strict) - { - const auto res = parse_msgpack_internal(); - if (strict) + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP + } + + // strict mode: next byte must be EOF + if (result and strict) { - get(); - check_eof(true); + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read, "expected end of input")); + } } - return res; + + return result; } /*! @@ -4551,14 +6101,16 @@ class binary_reader @param[in] get_char whether a new character should be retrieved from the input (true, default) or whether the last read character should be considered instead + + @return whether a valid CBOR value was passed to the SAX parser */ - BasicJsonType parse_cbor_internal(const bool get_char = true) + bool parse_cbor_internal(const bool get_char = true) { switch (get_char ? get() : current) { // EOF case std::char_traits<char>::eof(): - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + return unexpect_eof(); // Integer 0x00..0x17 (0..23) case 0x00: @@ -4585,19 +6137,31 @@ class binary_reader case 0x15: case 0x16: case 0x17: - return static_cast<number_unsigned_t>(current); + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); case 0x18: // Unsigned integer (one-byte uint8_t follows) - return get_number<uint8_t>(); + { + uint8_t number; + return get_number(number) and sax->number_unsigned(number); + } case 0x19: // Unsigned integer (two-byte uint16_t follows) - return get_number<uint16_t>(); + { + uint16_t number; + return get_number(number) and sax->number_unsigned(number); + } case 0x1A: // Unsigned integer (four-byte uint32_t follows) - return get_number<uint32_t>(); + { + uint32_t number; + return get_number(number) and sax->number_unsigned(number); + } case 0x1B: // Unsigned integer (eight-byte uint64_t follows) - return get_number<uint64_t>(); + { + uint64_t number; + return get_number(number) and sax->number_unsigned(number); + } // Negative integer -1-0x00..-1-0x17 (-1..-24) case 0x20: @@ -4624,28 +6188,31 @@ class binary_reader case 0x35: case 0x36: case 0x37: - return static_cast<int8_t>(0x20 - 1 - current); + return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current)); case 0x38: // Negative integer (one-byte uint8_t follows) { - // must be uint8_t ! - return static_cast<number_integer_t>(-1) - get_number<uint8_t>(); + uint8_t number; + return get_number(number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x39: // Negative integer -1-n (two-byte uint16_t follows) { - return static_cast<number_integer_t>(-1) - get_number<uint16_t>(); + uint16_t number; + return get_number(number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) { - return static_cast<number_integer_t>(-1) - get_number<uint32_t>(); + uint32_t number; + return get_number(number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) { - return static_cast<number_integer_t>(-1) - - static_cast<number_integer_t>(get_number<uint64_t>()); + uint64_t number; + return get_number(number) and sax->number_integer(static_cast<number_integer_t>(-1) + - static_cast<number_integer_t>(number)); } // UTF-8 string (0x00..0x17 bytes follow) @@ -4679,7 +6246,8 @@ class binary_reader case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) case 0x7F: // UTF-8 string (indefinite length) { - return get_cbor_string(); + string_t s; + return get_cbor_string(s) and sax->string(s); } // array (0x00..0x17 data items follow) @@ -4707,39 +6275,34 @@ class binary_reader case 0x95: case 0x96: case 0x97: - { - return get_cbor_array(current & 0x1F); - } + return get_cbor_array(static_cast<std::size_t>(current & 0x1F)); case 0x98: // array (one-byte uint8_t for n follows) { - return get_cbor_array(get_number<uint8_t>()); + uint8_t len; + return get_number(len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x99: // array (two-byte uint16_t for n follow) { - return get_cbor_array(get_number<uint16_t>()); + uint16_t len; + return get_number(len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { - return get_cbor_array(get_number<uint32_t>()); + uint32_t len; + return get_number(len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9B: // array (eight-byte uint64_t for n follow) { - return get_cbor_array(get_number<uint64_t>()); + uint64_t len; + return get_number(len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9F: // array (indefinite length) - { - BasicJsonType result = value_t::array; - while (get() != 0xFF) - { - result.push_back(parse_cbor_internal(false)); - } - return result; - } + return get_cbor_array(std::size_t(-1)); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -4766,62 +6329,56 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - { - return get_cbor_object(current & 0x1F); - } + return get_cbor_object(static_cast<std::size_t>(current & 0x1F)); case 0xB8: // map (one-byte uint8_t for n follows) { - return get_cbor_object(get_number<uint8_t>()); + uint8_t len; + return get_number(len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { - return get_cbor_object(get_number<uint16_t>()); + uint16_t len; + return get_number(len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { - return get_cbor_object(get_number<uint32_t>()); + uint32_t len; + return get_number(len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBB: // map (eight-byte uint64_t for n follow) { - return get_cbor_object(get_number<uint64_t>()); + uint64_t len; + return get_number(len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBF: // map (indefinite length) - { - BasicJsonType result = value_t::object; - while (get() != 0xFF) - { - auto key = get_cbor_string(); - result[key] = parse_cbor_internal(); - } - return result; - } + return get_cbor_object(std::size_t(-1)); case 0xF4: // false - { - return false; - } + return sax->boolean(false); case 0xF5: // true - { - return true; - } + return sax->boolean(true); case 0xF6: // null - { - return value_t::null; - } + return sax->null(); case 0xF9: // Half-Precision Float (two-byte IEEE 754) { const int byte1 = get(); - check_eof(); + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } const int byte2 = get(); - check_eof(); + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } // code from RFC 7049, Appendix D, Figure 3: // As half-precision floating-point numbers were only added @@ -4832,51 +6389,59 @@ class binary_reader // half-precision floating-point numbers in the C language // is shown in Fig. 3. const int half = (byte1 << 8) + byte2; - const int exp = (half >> 10) & 0x1F; - const int mant = half & 0x3FF; - double val; - if (exp == 0) - { - val = std::ldexp(mant, -24); - } - else if (exp != 31) + const double val = [&half] { - val = std::ldexp(mant + 1024, exp - 25); - } - else - { - val = (mant == 0) ? std::numeric_limits<double>::infinity() - : std::numeric_limits<double>::quiet_NaN(); - } - return (half & 0x8000) != 0 ? -val : val; + const int exp = (half >> 10) & 0x1F; + const int mant = half & 0x3FF; + assert(0 <= exp and exp <= 32); + assert(0 <= mant and mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits<double>::infinity() + : std::numeric_limits<double>::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000) != 0 + ? static_cast<number_float_t>(-val) + : static_cast<number_float_t>(val), ""); } case 0xFA: // Single-Precision Float (four-byte IEEE 754) { - return get_number<float>(); + float number; + return get_number(number) and sax->number_float(static_cast<number_float_t>(number), ""); } case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { - return get_number<double>(); + double number; + return get_number(number) and sax->number_float(static_cast<number_float_t>(number), ""); } default: // anything else (0xFF is handled inside the other types) { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + last_token)); } } } - BasicJsonType parse_msgpack_internal() + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() { switch (get()) { // EOF case std::char_traits<char>::eof(): - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + return unexpect_eof(); // positive fixint case 0x00: @@ -5007,7 +6572,7 @@ class binary_reader case 0x7D: case 0x7E: case 0x7F: - return static_cast<number_unsigned_t>(current); + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); // fixmap case 0x80: @@ -5026,9 +6591,7 @@ class binary_reader case 0x8D: case 0x8E: case 0x8F: - { - return get_msgpack_object(current & 0x0F); - } + return get_msgpack_object(static_cast<std::size_t>(current & 0x0F)); // fixarray case 0x90: @@ -5047,9 +6610,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - { - return get_msgpack_array(current & 0x0F); - } + return get_msgpack_array(static_cast<std::size_t>(current & 0x0F)); // fixstr case 0xA0: @@ -5084,73 +6645,113 @@ class binary_reader case 0xBD: case 0xBE: case 0xBF: - return get_msgpack_string(); + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } case 0xC0: // nil - return value_t::null; + return sax->null(); case 0xC2: // false - return false; + return sax->boolean(false); case 0xC3: // true - return true; + return sax->boolean(true); case 0xCA: // float 32 - return get_number<float>(); + { + float number; + return get_number(number) and sax->number_float(static_cast<number_float_t>(number), ""); + } case 0xCB: // float 64 - return get_number<double>(); + { + double number; + return get_number(number) and sax->number_float(static_cast<number_float_t>(number), ""); + } case 0xCC: // uint 8 - return get_number<uint8_t>(); + { + uint8_t number; + return get_number(number) and sax->number_unsigned(number); + } case 0xCD: // uint 16 - return get_number<uint16_t>(); + { + uint16_t number; + return get_number(number) and sax->number_unsigned(number); + } case 0xCE: // uint 32 - return get_number<uint32_t>(); + { + uint32_t number; + return get_number(number) and sax->number_unsigned(number); + } case 0xCF: // uint 64 - return get_number<uint64_t>(); + { + uint64_t number; + return get_number(number) and sax->number_unsigned(number); + } case 0xD0: // int 8 - return get_number<int8_t>(); + { + int8_t number; + return get_number(number) and sax->number_integer(number); + } case 0xD1: // int 16 - return get_number<int16_t>(); + { + int16_t number; + return get_number(number) and sax->number_integer(number); + } case 0xD2: // int 32 - return get_number<int32_t>(); + { + int32_t number; + return get_number(number) and sax->number_integer(number); + } case 0xD3: // int 64 - return get_number<int64_t>(); + { + int64_t number; + return get_number(number) and sax->number_integer(number); + } case 0xD9: // str 8 case 0xDA: // str 16 case 0xDB: // str 32 - return get_msgpack_string(); + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } case 0xDC: // array 16 { - return get_msgpack_array(get_number<uint16_t>()); + uint16_t len; + return get_number(len) and get_msgpack_array(static_cast<std::size_t>(len)); } case 0xDD: // array 32 { - return get_msgpack_array(get_number<uint32_t>()); + uint32_t len; + return get_number(len) and get_msgpack_array(static_cast<std::size_t>(len)); } case 0xDE: // map 16 { - return get_msgpack_object(get_number<uint16_t>()); + uint16_t len; + return get_number(len) and get_msgpack_object(static_cast<std::size_t>(len)); } case 0xDF: // map 32 { - return get_msgpack_object(get_number<uint32_t>()); + uint32_t len; + return get_number(len) and get_msgpack_object(static_cast<std::size_t>(len)); } - // positive fixint + // negative fixint case 0xE0: case 0xE1: case 0xE2: @@ -5183,19 +6784,29 @@ class binary_reader case 0xFD: case 0xFE: case 0xFF: - return static_cast<int8_t>(current); + return sax->number_integer(static_cast<int8_t>(current)); default: // anything else { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(112, chars_read, - "error reading MessagePack; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "error reading MessagePack; last byte: 0x" + last_token)); } } } /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + + /*! @brief get next character from the input This function provides the interface to the used input adapter. It does @@ -5210,27 +6821,44 @@ class binary_reader return (current = ia->get_character()); } + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + /* @brief read a number from the input @tparam NumberType the type of the number + @param[out] result number of type @a NumberType - @return number of type @a NumberType + @return whether conversion completed @note This function needs to respect the system's endianess, because - bytes in CBOR and MessagePack are stored in network order (big - endian) and therefore need reordering on little endian systems. - - @throw parse_error.110 if input has less than `sizeof(NumberType)` bytes + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. */ - template<typename NumberType> NumberType get_number() + template<typename NumberType> + bool get_number(NumberType& result) { // step 1: read input into array with system's byte order std::array<uint8_t, sizeof(NumberType)> vec; for (std::size_t i = 0; i < sizeof(NumberType); ++i) { get(); - check_eof(); + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } // reverse byte order prior to conversion if necessary if (is_little_endian) @@ -5244,35 +6872,37 @@ class binary_reader } // step 2: convert array into number of type T and return - NumberType result; std::memcpy(&result, vec.data(), sizeof(NumberType)); - return result; + return true; } /*! @brief create a string by reading characters from the input - @param[in] len number of bytes to read + @tparam NumberType the type of the number + @param[in] len number of characters to read + @param[out] string created by reading @a len bytes + + @return whether string creation completed @note We can not reserve @a len bytes for the result, because @a len - may be too large. Usually, @ref check_eof() detects the end of + may be too large. Usually, @ref unexpect_eof() detects the end of the input before we run out of string memory. - - @return string created by reading @a len bytes - - @throw parse_error.110 if input has less than @a len bytes */ template<typename NumberType> - std::string get_string(const NumberType len) + bool get_string(const NumberType len, string_t& result) { - std::string result; - std::generate_n(std::back_inserter(result), len, [this]() + bool success = true; + std::generate_n(std::back_inserter(result), len, [this, &success]() { get(); - check_eof(); + if (JSON_UNLIKELY(not unexpect_eof())) + { + success = false; + } return static_cast<char>(current); }); - return result; + return success; } /*! @@ -5282,14 +6912,16 @@ class binary_reader string length and then copies this number of bytes into a string. Additionally, CBOR's strings with indefinite lengths are supported. - @return string + @param[out] result created string - @throw parse_error.110 if input ended - @throw parse_error.113 if an unexpected byte is read + @return whether string creation completed */ - std::string get_cbor_string() + bool get_cbor_string(string_t& result) { - check_eof(); + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } switch (current) { @@ -5319,74 +6951,137 @@ class binary_reader case 0x76: case 0x77: { - return get_string(current & 0x1F); + return get_string(current & 0x1F, result); } case 0x78: // UTF-8 string (one-byte uint8_t for n follows) { - return get_string(get_number<uint8_t>()); + uint8_t len; + return get_number(len) and get_string(len, result); } case 0x79: // UTF-8 string (two-byte uint16_t for n follow) { - return get_string(get_number<uint16_t>()); + uint16_t len; + return get_number(len) and get_string(len, result); } case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) { - return get_string(get_number<uint32_t>()); + uint32_t len; + return get_number(len) and get_string(len, result); } case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) { - return get_string(get_number<uint64_t>()); + uint64_t len; + return get_number(len) and get_string(len, result); } case 0x7F: // UTF-8 string (indefinite length) { - std::string result; while (get() != 0xFF) { - check_eof(); - result.push_back(static_cast<char>(current)); + string_t chunk; + if (not get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); } - return result; + return true; } default: { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + last_token)); } } } - template<typename NumberType> - BasicJsonType get_cbor_array(const NumberType len) + /*! + @param[in] len the length of the array or std::size_t(-1) for an + array of indefinite size + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len) { - BasicJsonType result = value_t::array; - std::generate_n(std::back_inserter(*result.m_value.array), len, [this]() + if (JSON_UNLIKELY(not sax->start_array(len))) { - return parse_cbor_internal(); - }); - return result; + return false; + } + + if (len != std::size_t(-1)) + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not parse_cbor_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); } - template<typename NumberType> - BasicJsonType get_cbor_object(const NumberType len) + /*! + @param[in] len the length of the object or std::size_t(-1) for an + object of indefinite size + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len) { - BasicJsonType result = value_t::object; - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - len, [this]() + if (not JSON_UNLIKELY(sax->start_object(len))) { - get(); - auto key = get_cbor_string(); - auto val = parse_cbor_internal(); - return std::make_pair(std::move(key), std::move(val)); - }); - return result; + return false; + } + + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + + return sax->end_object(); } /*! @@ -5395,14 +7090,16 @@ class binary_reader This function first reads starting bytes to determine the expected string length and then copies this number of bytes into a string. - @return string + @param[out] result created string - @throw parse_error.110 if input ended - @throw parse_error.113 if an unexpected byte is read + @return whether string creation completed */ - std::string get_msgpack_string() + bool get_msgpack_string(string_t& result) { - check_eof(); + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } switch (current) { @@ -5440,81 +7137,521 @@ class binary_reader case 0xBE: case 0xBF: { - return get_string(current & 0x1F); + return get_string(current & 0x1F, result); } case 0xD9: // str 8 { - return get_string(get_number<uint8_t>()); + uint8_t len; + return get_number(len) and get_string(len, result); } case 0xDA: // str 16 { - return get_string(get_number<uint16_t>()); + uint16_t len; + return get_number(len) and get_string(len, result); } case 0xDB: // str 32 { - return get_string(get_number<uint32_t>()); + uint32_t len; + return get_number(len) and get_string(len, result); } default: { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(113, chars_read, - "expected a MessagePack string; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "expected a MessagePack string; last byte: 0x" + last_token)); } } } - template<typename NumberType> - BasicJsonType get_msgpack_array(const NumberType len) + /*! + @param[in] len the length of the array + @return whether array creation completed + */ + bool get_msgpack_array(const std::size_t len) { - BasicJsonType result = value_t::array; - std::generate_n(std::back_inserter(*result.m_value.array), len, [this]() + if (JSON_UNLIKELY(not sax->start_array(len))) { - return parse_msgpack_internal(); - }); - return result; + return false; + } + + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + } + + return sax->end_array(); } - template<typename NumberType> - BasicJsonType get_msgpack_object(const NumberType len) + /*! + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) { - BasicJsonType result = value_t::object; - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - len, [this]() + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } + + string_t key; + for (std::size_t i = 0; i < len; ++i) { get(); - auto key = get_msgpack_string(); - auto val = parse_msgpack_internal(); - return std::make_pair(std::move(key), std::move(val)); - }); - return result; + if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + key.clear(); + } + + return sax->end_object(); } /*! - @brief check if input ended - @throw parse_error.110 if input ended + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether string creation completed */ - void check_eof(const bool expect_eof = false) const + bool get_ubjson_string(string_t& result, const bool get_char = true) { - if (expect_eof) + if (get_char) { - if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + get(); // TODO: may we ignore N here? + } + + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } + + switch (current) + { + case 'U': + { + uint8_t len; + return get_number(len) and get_string(len, result); + } + + case 'i': + { + int8_t len; + return get_number(len) and get_string(len, result); + } + + case 'I': + { + int16_t len; + return get_number(len) and get_string(len, result); + } + + case 'l': + { + int32_t len; + return get_number(len) and get_string(len, result); + } + + case 'L': + { + int64_t len; + return get_number(len) and get_string(len, result); + } + + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "expected a UBJSON string; last byte: 0x" + last_token)); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': + { + uint8_t number; + if (JSON_UNLIKELY(not get_number(number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'i': + { + int8_t number; + if (JSON_UNLIKELY(not get_number(number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'I': + { + int16_t number; + if (JSON_UNLIKELY(not get_number(number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'l': + { + int32_t number; + if (JSON_UNLIKELY(not get_number(number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'L': + { + int64_t number; + if (JSON_UNLIKELY(not get_number(number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "byte after '#' must denote a number type; last byte: 0x" + last_token)); + } + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair<std::size_t, int>& result) + { + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') + { + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } + + get_ignore_noop(); + if (JSON_UNLIKELY(current != '#')) + { + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "expected '#' after UBJSON type information; last byte: 0x" + last_token)); + } + + return get_ubjson_size_value(result.first); + } + else if (current == '#') + { + return get_ubjson_size_value(result.first); + } + return true; + } + + /*! + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits<char>::eof(): // EOF + return unexpect_eof(); + + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); + + case 'U': + { + uint8_t number; + return get_number(number) and sax->number_unsigned(number); + } + + case 'i': + { + int8_t number; + return get_number(number) and sax->number_integer(number); + } + + case 'I': + { + int16_t number; + return get_number(number) and sax->number_integer(number); + } + + case 'l': + { + int32_t number; + return get_number(number) and sax->number_integer(number); + } + + case 'L': { - JSON_THROW(parse_error::create(110, chars_read, "expected end of input")); + int64_t number; + return get_number(number) and sax->number_integer(number); + } + + case 'd': + { + float number; + return get_number(number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'D': + { + double number; + return get_number(number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'C': // char + { + get(); + if (JSON_UNLIKELY(not unexpect_eof())) + { + return false; + } + if (JSON_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token)); + } + string_t s(1, static_cast<char>(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) and sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, "error reading UBJSON; last byte: 0x" + last_token)); + } + } + } + + /*! + @return whether array creation completed + */ + bool get_ubjson_array() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + } } } else { - if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) { - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + return false; + } + + while (current != ']') + { + if (JSON_UNLIKELY(not parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); + } + } + + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); } } + + return sax->end_object(); + } + + /*! + @return whether the last read character is not EOF + */ + bool unexpect_eof() const + { + if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, "<end of file>", parse_error::create(110, chars_read, "unexpected end of input")); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + char cr[3]; + snprintf(cr, 3, "%.2hhX", static_cast<unsigned char>(current)); + return std::string{cr}; } private: @@ -5529,7 +7666,34 @@ class binary_reader /// whether we can assume little endianess const bool is_little_endian = little_endianess(); + + /// the SAX parser + json_sax_t* sax = nullptr; }; +} +} + +// #include <nlohmann/detail/output/binary_writer.hpp> + + +#include <algorithm> // reverse +#include <array> // array +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstring> // memcpy +#include <limits> // numeric_limits + +// #include <nlohmann/detail/input/binary_reader.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// /*! @brief serialization to CBOR and MessagePack values @@ -5663,9 +7827,9 @@ class binary_writer break; } - case value_t::number_float: // Double-Precision Float + case value_t::number_float: { - oa->write_character(static_cast<CharType>(0xFB)); + oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); write_number(j.m_value.number_float); break; } @@ -5678,23 +7842,23 @@ class binary_writer { write_number(static_cast<uint8_t>(0x60 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { oa->write_character(static_cast<CharType>(0x78)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { oa->write_character(static_cast<CharType>(0x79)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { oa->write_character(static_cast<CharType>(0x7A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { oa->write_character(static_cast<CharType>(0x7B)); write_number(static_cast<uint64_t>(N)); @@ -5716,23 +7880,23 @@ class binary_writer { write_number(static_cast<uint8_t>(0x80 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { oa->write_character(static_cast<CharType>(0x98)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { oa->write_character(static_cast<CharType>(0x99)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { oa->write_character(static_cast<CharType>(0x9A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { oa->write_character(static_cast<CharType>(0x9B)); write_number(static_cast<uint64_t>(N)); @@ -5755,23 +7919,23 @@ class binary_writer { write_number(static_cast<uint8_t>(0xA0 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { oa->write_character(static_cast<CharType>(0xB8)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { oa->write_character(static_cast<CharType>(0xB9)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { oa->write_character(static_cast<CharType>(0xBA)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { oa->write_character(static_cast<CharType>(0xBB)); write_number(static_cast<uint64_t>(N)); @@ -5923,9 +8087,9 @@ class binary_writer break; } - case value_t::number_float: // float 64 + case value_t::number_float: { - oa->write_character(static_cast<CharType>(0xCB)); + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); write_number(j.m_value.number_float); break; } @@ -5939,19 +8103,19 @@ class binary_writer // fixstr write_number(static_cast<uint8_t>(0xA0 | N)); } - else if (N <= 255) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { // str 8 oa->write_character(static_cast<CharType>(0xD9)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 65535) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // str 16 oa->write_character(static_cast<CharType>(0xDA)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // str 32 oa->write_character(static_cast<CharType>(0xDB)); @@ -5974,13 +8138,13 @@ class binary_writer // fixarray write_number(static_cast<uint8_t>(0x90 | N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // array 16 oa->write_character(static_cast<CharType>(0xDC)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // array 32 oa->write_character(static_cast<CharType>(0xDD)); @@ -6004,13 +8168,13 @@ class binary_writer // fixmap write_number(static_cast<uint8_t>(0x80 | (N & 0xF))); } - else if (N <= 65535) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // map 16 oa->write_character(static_cast<CharType>(0xDE)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // map 32 oa->write_character(static_cast<CharType>(0xDF)); @@ -6031,6 +8195,165 @@ class binary_writer } } + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + oa->write_character(j.m_value.boolean + ? static_cast<CharType>('T') + : static_cast<CharType>('F')); + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('[')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.array->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(static_cast<CharType>('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(static_cast<CharType>('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(static_cast<CharType>(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('{')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.object->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(static_cast<CharType>('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(static_cast<CharType>('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(static_cast<CharType>('}')); + } + + break; + } + + default: + break; + } + } + private: /* @brief write a number to output input @@ -6039,10 +8362,11 @@ class binary_writer @tparam NumberType the type of the number @note This function needs to respect the system's endianess, because bytes - in CBOR and MessagePack are stored in network order (big endian) and - therefore need reordering on little endian systems. + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. */ - template<typename NumberType> void write_number(NumberType n) + template<typename NumberType> + void write_number(const NumberType n) { // step 1: write number to array of length NumberType std::array<CharType, sizeof(NumberType)> vec; @@ -6058,6 +8382,240 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + // UBJSON: write number (floating point) + template<typename NumberType, typename std::enable_if< + std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template<typename NumberType, typename std::enable_if< + std::is_unsigned<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('i')); // int8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= (std::numeric_limits<uint8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)())) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)())) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)())) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + else + { + JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); + } + } + + // UBJSON: write number (signed integer) + template<typename NumberType, typename std::enable_if< + std::is_signed<NumberType>::value and + not std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('i')); // int8 + } + write_number(static_cast<int8_t>(n)); + } + else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast<CharType>('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + // LCOV_EXCL_START + else + { + JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + + @note This function does not need to be 100% accurate when it comes to + integer limits. In case a number exceeds the limits of int64_t, + this will be detected by a later call to function + write_number_with_ubjson_prefix. Therefore, we return 'L' for any + value that does not fit the previous limits. + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + else if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + else if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + else if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + else // no check and assume int64_t (see note above) + { + return 'L'; + } + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + else // no check and assume int64_t (see note above) + { + return 'L'; + } + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: + return '['; + + case value_t::object: + return '{'; + + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_cbor_float_prefix(float) + { + return static_cast<CharType>(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double) + { + return static_cast<CharType>(0xFB); // Double-Precision Float + } + + static constexpr CharType get_msgpack_float_prefix(float) + { + return static_cast<CharType>(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double) + { + return static_cast<CharType>(0xCB); // float 64 + } + + static constexpr CharType get_ubjson_float_prefix(float) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double) + { + return 'D'; // float 64 + } + private: /// whether we can assume little endianess const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess(); @@ -6065,7 +8623,1137 @@ class binary_writer /// the output output_adapter_t<CharType> oa = nullptr; }; +} +} + +// #include <nlohmann/detail/output/serializer.hpp> + + +#include <algorithm> // reverse, remove, fill, find, none_of +#include <array> // array +#include <cassert> // assert +#include <ciso646> // and, or +#include <clocale> // localeconv, lconv +#include <cmath> // labs, isfinite, isnan, signbit +#include <cstddef> // size_t, ptrdiff_t +#include <cstdint> // uint8_t +#include <cstdio> // snprintf +#include <limits> // numeric_limits +#include <string> // string +#include <type_traits> // is_same + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/conversions/to_chars.hpp> + + +#include <cassert> // assert +#include <ciso646> // or, and, not +#include <cmath> // signbit, isfinite +#include <cstdint> // intN_t, uintN_t +#include <cstring> // memcpy, memmove + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template <typename Target, typename Source> +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + uint64_t f; + int e; + + constexpr diyfp() noexcept : f(0), e(0) {} + constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + assert(x.e == y.e); + assert(x.f >= y.f); + + return diyfp(x.f - y.f, x.e); + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const uint64_t u_lo = x.f & 0xFFFFFFFF; + const uint64_t u_hi = x.f >> 32; + const uint64_t v_lo = y.f & 0xFFFFFFFF; + const uint64_t v_hi = y.f >> 32; + + const uint64_t p0 = u_lo * v_lo; + const uint64_t p1 = u_lo * v_hi; + const uint64_t p2 = u_hi * v_lo; + const uint64_t p3 = u_hi * v_hi; + + const uint64_t p0_hi = p0 >> 32; + const uint64_t p1_lo = p1 & 0xFFFFFFFF; + const uint64_t p1_hi = p1 >> 32; + const uint64_t p2_lo = p2 & 0xFFFFFFFF; + const uint64_t p2_hi = p2 >> 32; + + uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += uint64_t{1} << (64 - 32 - 1); // round, ties up + + const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32); + + return diyfp(h, x.e + y.e + 64); + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + assert(x.f != 0); + + while ((x.f >> 63) == 0) + { + x.f <<= 1; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + assert(delta >= 0); + assert(((x.f << delta) >> delta) == x.f); + + return diyfp(x.f << delta, target_exponent); + } +}; + +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template <typename FloatType> +boundaries compute_boundaries(FloatType value) +{ + assert(std::isfinite(value)); + assert(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits<FloatType>::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type; + + const uint64_t bits = reinterpret_bits<bits_type>(value); + const uint64_t E = bits >> (kPrecision - 1); + const uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = (E == 0); + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = (F == 0 and E > 1); + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersSize = 79; + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr cached_power kCachedPowers[] = + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + assert(e >= -1500); + assert(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + (f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + assert(index >= 0); + assert(index < kCachedPowersSize); + static_cast<void>(kCachedPowersSize); // Fix warning. + + const cached_power cached = kCachedPowers[index]; + assert(kAlpha <= cached.e + e + 64); + assert(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const uint32_t n, uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + else if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + else if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + else if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + else if (n >= 100000) + { + pow10 = 100000; + return 6; + } + else if (n >= 10000) + { + pow10 = 10000; + return 5; + } + else if (n >= 1000) + { + pow10 = 1000; + return 4; + } + else if (n >= 100) + { + pow10 = 100; + return 3; + } + else if (n >= 10) + { + pow10 = 10; + return 2; + } + else + { + pow10 = 1; + return 1; + } +} + +inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta, + uint64_t rest, uint64_t ten_k) +{ + assert(len >= 1); + assert(dist <= delta); + assert(rest <= delta); + assert(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + and delta - rest >= ten_k + and (rest + ten_k < dist or dist - rest > rest + ten_k - dist)) + { + assert(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + assert(M_plus.e >= kAlpha); + assert(M_plus.e <= kGamma); + + uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e); + + uint32_t p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + assert(p1 > 0); + + uint32_t pow10; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const uint64_t rest = (uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const uint64_t ten_n = uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + assert(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + assert(p2 <= UINT64_MAX / 10); + p2 *= 10; + const uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + assert(m_plus.e == m_minus.e); + assert(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template <typename FloatType> +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3, + "internal error: not enough precision"); + + assert(std::isfinite(value)); + assert(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast<double>(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +inline char* append_exponent(char* buf, int e) +{ + assert(e > -1000); + assert(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + uint32_t k = static_cast<uint32_t>(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast<char>('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + else + { + *buf++ = static_cast<char>('0' + k / 100); + k %= 100; + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. +@pre min_exp < 0 +@pre max_exp > 0 +*/ +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + assert(min_exp < 0); + assert(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n and n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast<size_t>(n - k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (n + 2); + } + + if (0 < n and n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + assert(k > n); + + std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n)); + buf[n] = '.'; + return buf + (k + 1); + } + + if (min_exp < n and n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast<size_t>(-n)); + return buf + (2 + (-n) + k); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1)); + buf[1] = '.'; + buf += 1 + k; + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template <typename FloatType> +char* to_chars(char* first, char* last, FloatType value) +{ + static_cast<void>(last); // maybe unused - fix warning + assert(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } + + assert(last - first >= std::numeric_limits<FloatType>::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + assert(len <= std::numeric_limits<FloatType>::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10; + + assert(last - first >= kMaxExp + 2); + assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10); + assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ /////////////////// // serialization // /////////////////// @@ -6077,6 +9765,9 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + static constexpr uint8_t UTF8_ACCEPT = 0; + static constexpr uint8_t UTF8_REJECT = 1; + public: /*! @param[in] s output stream to serialize to @@ -6086,7 +9777,8 @@ class serializer : o(std::move(s)), loc(std::localeconv()), thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)), decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)), - indent_char(ichar), indent_string(512, indent_char) {} + indent_char(ichar), indent_string(512, indent_char) + {} // delete because of pointer members serializer(const serializer&) = delete; @@ -6303,171 +9995,6 @@ class serializer private: /*! - @brief returns the number of expected bytes following in UTF-8 string - - @param[in] u the first byte of a UTF-8 string - @return the number of expected bytes following - */ - static constexpr std::size_t bytes_following(const uint8_t u) - { - return ((u <= 127) ? 0 - : ((192 <= u and u <= 223) ? 1 - : ((224 <= u and u <= 239) ? 2 - : ((240 <= u and u <= 247) ? 3 : std::string::npos)))); - } - - /*! - @brief calculates the extra space to escape a JSON string - - @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences - @return the number of characters required to escape string @a s - - @complexity Linear in the length of string @a s. - */ - static std::size_t extra_space(const string_t& s, - const bool ensure_ascii) noexcept - { - std::size_t res = 0; - - for (std::size_t i = 0; i < s.size(); ++i) - { - switch (s[i]) - { - // control characters that can be escaped with a backslash - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - res += 1; - break; - } - - // control characters that need \uxxxx escaping - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x0B: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - { - // from c (1 byte) to \uxxxx (6 bytes) - res += 5; - break; - } - - default: - { - if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)) - { - const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - // invalid characters will be detected by throw_if_invalid_utf8 - assert (bytes != std::string::npos); - - if (bytes == 3) - { - // codepoints that need 4 bytes (i.e., 3 additional - // bytes) in UTF-8 need a surrogate pair when \u - // escaping is used: from 4 bytes to \uxxxx\uxxxx - // (12 bytes) - res += (12 - bytes - 1); - } - else - { - // from x bytes to \uxxxx (6 bytes) - res += (6 - bytes - 1); - } - - // skip the additional bytes - i += bytes; - } - break; - } - } - } - - return res; - } - - static void escape_codepoint(int codepoint, string_t& result, std::size_t& pos) - { - // expecting a proper codepoint - assert(0x00 <= codepoint and codepoint <= 0x10FFFF); - - // the last written character was the backslash before the 'u' - assert(result[pos] == '\\'); - - // write the 'u' - result[++pos] = 'u'; - - // convert a number 0..15 to its hex representation (0..f) - static const std::array<char, 16> hexify = - { - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - } - }; - - if (codepoint < 0x10000) - { - // codepoints U+0000..U+FFFF can be represented as \uxxxx. - result[++pos] = hexify[(codepoint >> 12) & 0x0F]; - result[++pos] = hexify[(codepoint >> 8) & 0x0F]; - result[++pos] = hexify[(codepoint >> 4) & 0x0F]; - result[++pos] = hexify[codepoint & 0x0F]; - } - else - { - // codepoints U+10000..U+10FFFF need a surrogate pair to be - // represented as \uxxxx\uxxxx. - // http://www.unicode.org/faq/utf_bom.html#utf16-4 - codepoint -= 0x10000; - const int high_surrogate = 0xD800 | ((codepoint >> 10) & 0x3FF); - const int low_surrogate = 0xDC00 | (codepoint & 0x3FF); - result[++pos] = hexify[(high_surrogate >> 12) & 0x0F]; - result[++pos] = hexify[(high_surrogate >> 8) & 0x0F]; - result[++pos] = hexify[(high_surrogate >> 4) & 0x0F]; - result[++pos] = hexify[high_surrogate & 0x0F]; - ++pos; // backslash is already in output - result[++pos] = 'u'; - result[++pos] = hexify[(low_surrogate >> 12) & 0x0F]; - result[++pos] = hexify[(low_surrogate >> 8) & 0x0F]; - result[++pos] = hexify[(low_surrogate >> 4) & 0x0F]; - result[++pos] = hexify[low_surrogate & 0x0F]; - } - - ++pos; - } - - /*! @brief dump escaped string Escape a string by replacing certain special characters by a sequence of an @@ -6481,145 +10008,146 @@ class serializer @complexity Linear in the length of string @a s. */ - void dump_escaped(const string_t& s, const bool ensure_ascii) const + void dump_escaped(const string_t& s, const bool ensure_ascii) { - throw_if_invalid_utf8(s); - - const auto space = extra_space(s, ensure_ascii); - if (space == 0) - { - o->write_characters(s.c_str(), s.size()); - return; - } - - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; + uint32_t codepoint; + uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer for (std::size_t i = 0; i < s.size(); ++i) { - switch (s[i]) - { - case '"': // quotation mark (0x22) - { - result[pos + 1] = '"'; - pos += 2; - break; - } - - case '\\': // reverse solidus (0x5C) - { - // nothing to change - pos += 2; - break; - } - - case '\b': // backspace (0x08) - { - result[pos + 1] = 'b'; - pos += 2; - break; - } - - case '\f': // formfeed (0x0C) - { - result[pos + 1] = 'f'; - pos += 2; - break; - } + const auto byte = static_cast<uint8_t>(s[i]); - case '\n': // newline (0x0A) - { - result[pos + 1] = 'n'; - pos += 2; - break; - } - - case '\r': // carriage return (0x0D) + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point { - result[pos + 1] = 'r'; - pos += 2; - break; - } + switch (codepoint) + { + case 0x08: // backspace + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; + } - case '\t': // horizontal tab (0x09) - { - result[pos + 1] = 't'; - pos += 2; - break; - } + case 0x09: // horizontal tab + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; + } - default: - { - // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((0x00 <= s[i] and s[i] <= 0x1F) or - (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))) - { - const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - // invalid characters will be detected by throw_if_invalid_utf8 - assert (bytes != std::string::npos); + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } - // check that the additional bytes are present - assert(i + bytes < s.size()); + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } - // to use \uxxxx escaping, we first need to calculate - // the codepoint from the UTF-8 bytes - int codepoint = 0; + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } - // bytes is unsigned type: - assert(bytes <= 3); - switch (bytes) + case 0x22: // quotation mark { - case 0: - { - codepoint = s[i] & 0xFF; - break; - } + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } - case 1: - { - codepoint = ((s[i] & 0x3F) << 6) - + (s[i + 1] & 0x7F); - break; - } + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } - case 2: + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F))) { - codepoint = ((s[i] & 0x1F) << 12) - + ((s[i + 1] & 0x7F) << 6) - + (s[i + 2] & 0x7F); - break; + if (codepoint <= 0xFFFF) + { + std::snprintf(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast<uint16_t>(codepoint)); + bytes += 6; + } + else + { + std::snprintf(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)), + static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF))); + bytes += 12; + } } - - case 3: + else { - codepoint = ((s[i] & 0xF) << 18) - + ((s[i + 1] & 0x7F) << 12) - + ((s[i + 2] & 0x7F) << 6) - + (s[i + 3] & 0x7F); - break; + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; } - - default: - break; // LCOV_EXCL_LINE + break; } + } - escape_codepoint(codepoint, result, pos); - i += bytes; + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; } - else + break; + } + + case UTF8_REJECT: // decode found invalid UTF-8 byte + { + std::string sn(3, '\0'); + snprintf(&sn[0], sn.size(), "%.2X", byte); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); + } + + default: // decode found yet incomplete multi-byte code point + { + if (not ensure_ascii) { - // all other characters are added as-is - result[pos++] = s[i]; + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; } break; } } } - assert(pos == result.size()); - o->write_characters(result.c_str(), result.size()); + if (JSON_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + std::string sn(3, '\0'); + snprintf(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back())); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); + } } /*! @@ -6679,14 +10207,36 @@ class serializer void dump_float(number_float_t x) { // NaN / inf - if (not std::isfinite(x) or std::isnan(x)) + if (not std::isfinite(x)) { o->write_characters("null", 4); return; } - // get number of digits for a text -> float -> text round-trip - static constexpr auto d = std::numeric_limits<number_float_t>::digits10; + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if <long double> == <double>. + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or + (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024); + + dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + char* begin = number_buffer.data(); + char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast<size_t>(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10; // the actual conversion std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x); @@ -6744,15 +10294,16 @@ class serializer followed. @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) @param[in] byte next byte to decode + @return new state - @note The function has been edited: a std::array is used and the code - point is not calculated. + @note The function has been edited: a std::array is used. @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ */ - static void decode(uint8_t& state, const uint8_t byte) + static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept { static const std::array<uint8_t, 400> utf8d = { @@ -6775,42 +10326,13 @@ class serializer }; const uint8_t type = utf8d[byte]; - state = utf8d[256u + state * 16u + type]; - } - /*! - @brief throw an exception if a string is not UTF-8 encoded - - @param[in] str UTF-8 string to check - @throw type_error.316 if passed string is not UTF-8 encoded - - @since version 3.0.0 - */ - static void throw_if_invalid_utf8(const std::string& str) - { - // start with state 0 (= accept) - uint8_t state = 0; - - for (size_t i = 0; i < str.size(); ++i) - { - const auto byte = static_cast<uint8_t>(str[i]); - decode(state, byte); - if (state == 1) - { - // state 1 means reject - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(byte); - JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str())); - } - } + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6) + : static_cast<uint32_t>(0xff >> type) & (byte); - if (state != 0) - { - // we finish reading, but do not accept: string was incomplete - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(static_cast<uint8_t>(str.back())); - JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str())); - } + state = utf8d[256u + state * 16u + type]; + return state; } private: @@ -6827,13 +10349,27 @@ class serializer /// the locale's decimal point character const char decimal_point = '\0'; + /// string buffer + std::array<char, 512> string_buffer{{}}; + /// the indentation character const char indent_char; - /// the indentation string string_t indent_string; }; +} +} + +// #include <nlohmann/detail/json_ref.hpp> + +#include <initializer_list> +#include <utility> + +namespace nlohmann +{ +namespace detail +{ template<typename BasicJsonType> class json_ref { @@ -6886,74 +10422,30 @@ class json_ref value_type* value_ref = nullptr; const bool is_rvalue; }; - -} // namespace detail - -/// namespace to hold default `to_json` / `from_json` functions -namespace -{ -constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; -constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value; +} } +// #include <nlohmann/detail/json_pointer.hpp> -/*! -@brief default JSONSerializer template argument - -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template<typename, typename> -struct adl_serializer -{ - /*! - @brief convert a JSON value to any value type - - This function is usually called by the `get()` function of the - @ref basic_json class (either explicit or via conversion operators). - - @param[in] j JSON value to read from - @param[in,out] val value to write to - */ - template<typename BasicJsonType, typename ValueType> - static void from_json(BasicJsonType&& j, ValueType& val) noexcept( - noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) - { - ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); - } - /*! - @brief convert any value type to a JSON value +#include <cassert> // assert +#include <numeric> // accumulate +#include <string> // string +#include <vector> // vector - This function is usually called by the constructors of the @ref basic_json - class. +// #include <nlohmann/detail/macro_scope.hpp> - @param[in,out] j JSON value to write to - @param[in] val value to read from - */ - template<typename BasicJsonType, typename ValueType> - static void to_json(BasicJsonType& j, ValueType&& val) noexcept( - noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) - { - ::nlohmann::to_json(j, std::forward<ValueType>(val)); - } -}; - -/*! -@brief JSON Pointer +// #include <nlohmann/detail/exceptions.hpp> -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. +// #include <nlohmann/detail/value_t.hpp> -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) -@since version 2.0.0 -*/ +namespace nlohmann +{ +template<typename BasicJsonType> class json_pointer { - /// allow basic_json to access private members + // allow basic_json to access private members NLOHMANN_BASIC_JSON_TPL_DECLARATION friend class basic_json; @@ -6967,19 +10459,21 @@ class json_pointer @param[in] s string representing the JSON pointer; if omitted, the empty string is assumed which references the whole JSON value - @throw parse_error.107 if the given JSON pointer @a s is nonempty and - does not begin with a slash (`/`); see example below + @throw parse_error.107 if the given JSON pointer @a s is nonempty and does + not begin with a slash (`/`); see example below - @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s - is not followed by `0` (representing `~`) or `1` (representing `/`); - see example below + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is + not followed by `0` (representing `~`) or `1` (representing `/`); see + example below - @liveexample{The example shows the construction several valid JSON - pointers as well as the exceptional behavior.,json_pointer} + @liveexample{The example shows the construction several valid JSON pointers + as well as the exceptional behavior.,json_pointer} @since version 2.0.0 */ - explicit json_pointer(const std::string& s = "") : reference_tokens(split(s)) {} + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} /*! @brief return a string representation of the JSON pointer @@ -7021,7 +10515,7 @@ class json_pointer */ static int array_index(const std::string& s) { - size_t processed_chars = 0; + std::size_t processed_chars = 0; const int res = std::stoi(s, &processed_chars); // check if the string was completely read @@ -7076,8 +10570,66 @@ class json_pointer @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const; + BasicJsonType& get_and_create(BasicJsonType& j) const + { + using size_type = typename BasicJsonType::size_type; + auto result = &j; + + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case detail::value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case detail::value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // create an entry in the array + JSON_TRY + { + result = &result->operator[](static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); + } + } + + return *result; + } /*! @brief return a reference to the pointed to value @@ -7098,8 +10650,75 @@ class json_pointer @throw parse_error.109 if an array index was not a number @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const; + BasicJsonType& get_unchecked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->m_type == detail::value_t::null) + { + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const char x) + { + return (x >= '0' and x <= '9'); + }); + + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums or reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; + } + + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @throw parse_error.106 if an array index begins with '0' @@ -7107,8 +10726,57 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const; + BasicJsonType& get_checked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @brief return a const reference to the pointed to value @@ -7123,8 +10791,58 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - const NLOHMANN_BASIC_JSON_TPL& get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const; + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // use unchecked array access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @throw parse_error.106 if an array index begins with '0' @@ -7132,8 +10850,57 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - const NLOHMANN_BASIC_JSON_TPL& get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const; + const BasicJsonType& get_checked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @brief split the string input to reference tokens @@ -7252,10 +11019,57 @@ class json_pointer @note Empty objects or arrays are flattened to `null`. */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION static void flatten(const std::string& reference_string, - const NLOHMANN_BASIC_JSON_TPL& value, - NLOHMANN_BASIC_JSON_TPL& result); + const BasicJsonType& value, + BasicJsonType& result) + { + switch (value.m_type) + { + case detail::value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case detail::value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } /*! @param[in] value flattened JSON @@ -7267,19 +11081,108 @@ class json_pointer @throw type_error.315 if object values are not primitive @throw type_error.313 if value cannot be unflattened */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - static NLOHMANN_BASIC_JSON_TPL - unflatten(const NLOHMANN_BASIC_JSON_TPL& value); + static BasicJsonType + unflatten(const BasicJsonType& value) + { + if (JSON_UNLIKELY(not value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); + } + + BasicJsonType result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (JSON_UNLIKELY(not element.second.is_primitive())) + { + JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); + } + + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } friend bool operator==(json_pointer const& lhs, - json_pointer const& rhs) noexcept; + json_pointer const& rhs) noexcept + { + return (lhs.reference_tokens == rhs.reference_tokens); + } friend bool operator!=(json_pointer const& lhs, - json_pointer const& rhs) noexcept; + json_pointer const& rhs) noexcept + { + return not (lhs == rhs); + } /// the reference tokens std::vector<std::string> reference_tokens; }; +} + +// #include <nlohmann/adl_serializer.hpp> + + +#include <utility> + +// #include <nlohmann/detail/conversions/from_json.hpp> + +// #include <nlohmann/detail/conversions/to_json.hpp> + + +namespace nlohmann +{ +template<typename, typename> +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template<typename BasicJsonType, typename ValueType> + static void from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) + { + ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template<typename BasicJsonType, typename ValueType> + static void to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) + { + ::nlohmann::to_json(j, std::forward<ValueType>(val)); + } +}; +} + + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ /*! @brief a class to store JSON values @@ -7305,42 +11208,42 @@ and `from_json()` (@ref adl_serializer by default) @requirement The class satisfies the following concept requirements: - Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible): JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible): A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible): A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable): A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable): A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible): JSON values can be destructed. - Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType): JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout): All non-static data members are private and standard layout types, the class has no virtual functions or (virtual) base classes. - Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable): JSON values can be compared with `==`, see @ref operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable): JSON values can be compared with `<`, see @ref operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable): Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer): JSON values can be compared against `std::nullptr_t` objects which are used to model the `null` value. - Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): + - [Container](https://en.cppreference.com/w/cpp/named_req/Container): JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer); JSON values can be used like STL containers and provide reverse iterator access. @@ -7367,15 +11270,19 @@ class basic_json { private: template<detail::value_t> friend struct detail::external_constructor; - friend ::nlohmann::json_pointer; + friend ::nlohmann::json_pointer<basic_json>; friend ::nlohmann::detail::parser<basic_json>; friend ::nlohmann::detail::serializer<basic_json>; template<typename BasicJsonType> friend class ::nlohmann::detail::iter_impl; template<typename BasicJsonType, typename CharType> friend class ::nlohmann::detail::binary_writer; - template<typename BasicJsonType> + template<typename BasicJsonType, typename SAX> friend class ::nlohmann::detail::binary_reader; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_parser; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_callback_parser; /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL; @@ -7403,13 +11310,17 @@ class basic_json public: using value_t = detail::value_t; - /// @copydoc nlohmann::json_pointer - using json_pointer = ::nlohmann::json_pointer; + /// JSON Pointer, see @ref nlohmann::json_pointer + using json_pointer = ::nlohmann::json_pointer<basic_json>; template<typename T, typename SFINAE> using json_serializer = JSONSerializer<T, SFINAE>; /// helper type for initializer lists of basic_json values using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>; + using input_format_t = detail::input_format_t; + /// SAX interface type, see @ref nlohmann::json_sax + using json_sax_t = json_sax<basic_json>; + //////////////// // exceptions // //////////////// @@ -7517,10 +11428,13 @@ class basic_json result["copyright"] = "(C) 2013-2017 Niels Lohmann"; result["name"] = "JSON for Modern C++"; result["url"] = "https://github.com/nlohmann/json"; - result["version"] = - { - {"string", "3.0.1"}, {"major", 3}, {"minor", 0}, {"patch", 1} - }; + result["version"]["string"] = + std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_PATCH); + result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR; + result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR; + result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH; #ifdef _WIN32 result["platform"] = "win32"; @@ -7622,10 +11536,10 @@ class basic_json - When all names are unique, objects will be interoperable in the sense that all software implementations receiving that object will agree on the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. + - When the names within an object are not unique, it is unspecified which + one of the values for a given key will be chosen. For instance, + `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or + `{"key": 2}`. - Internally, name/value pairs are stored in lexicographical order of the names. Objects will also be serialized (see @ref dump) in this order. For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored @@ -8139,7 +12053,7 @@ class basic_json object = nullptr; // silence warning, see #821 if (JSON_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.0.1")); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.2.0")); // LCOV_EXCL_LINE } break; } @@ -8182,7 +12096,7 @@ class basic_json array = create<array_t>(std::move(value)); } - void destroy(value_t t) + void destroy(value_t t) noexcept { switch (t) { @@ -8227,7 +12141,7 @@ class basic_json value is changed, because the invariant expresses a relationship between @a m_type and @a m_value. */ - void assert_invariant() const + void assert_invariant() const noexcept { assert(m_type != value_t::object or m_value.object != nullptr); assert(m_type != value_t::array or m_value.array != nullptr); @@ -8307,7 +12221,6 @@ class basic_json */ using parser_callback_t = typename parser::parser_callback_t; - ////////////////// // constructors // ////////////////// @@ -8409,6 +12322,7 @@ class basic_json - @a CompatibleType is not derived from `std::istream`, - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move constructors), + - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments) - @a CompatibleType is not a @ref basic_json nested type (e.g., @ref json_pointer, @ref iterator, etc ...) - @ref @ref json_serializer<U> has a @@ -8432,21 +12346,91 @@ class basic_json @since version 2.1.0 */ - template<typename CompatibleType, typename U = detail::uncvref_t<CompatibleType>, - detail::enable_if_t<not std::is_base_of<std::istream, U>::value and - not std::is_same<U, basic_json_t>::value and - not detail::is_basic_json_nested_type< - basic_json_t, U>::value and - detail::has_to_json<basic_json, U>::value, - int> = 0> - basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer<U>::to_json( - std::declval<basic_json_t&>(), std::forward<CompatibleType>(val)))) + template <typename CompatibleType, + typename U = detail::uncvref_t<CompatibleType>, + detail::enable_if_t< + detail::is_compatible_type<basic_json_t, U>::value, int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept( + JSONSerializer<U>::to_json(std::declval<basic_json_t&>(), + std::forward<CompatibleType>(val)))) { JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val)); assert_invariant(); } /*! + @brief create a JSON value from an existing one + + This is a constructor for existing @ref basic_json types. + It does not hijack copy/move constructors, since the parameter has different + template arguments than the current ones. + + The constructor tries to convert the internal @ref m_value of the parameter. + + @tparam BasicJsonType a type such that: + - @a BasicJsonType is a @ref basic_json type. + - @a BasicJsonType has different template arguments than @ref basic_json_t. + + @param[in] val the @ref basic_json value to be converted. + + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. + + @exceptionsafety Depends on the called constructor. For types directly + supported by the library (i.e., all types for which no `to_json()` function + was provided), strong guarantee holds: if an exception is thrown, there are + no changes to any JSON value. + + @since version 3.2.0 + */ + template <typename BasicJsonType, + detail::enable_if_t< + detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0> + basic_json(const BasicJsonType& val) + { + using other_boolean_t = typename BasicJsonType::boolean_t; + using other_number_float_t = typename BasicJsonType::number_float_t; + using other_number_integer_t = typename BasicJsonType::number_integer_t; + using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using other_string_t = typename BasicJsonType::string_t; + using other_object_t = typename BasicJsonType::object_t; + using other_array_t = typename BasicJsonType::array_t; + + switch (val.type()) + { + case value_t::boolean: + JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>()); + break; + case value_t::number_float: + JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>()); + break; + case value_t::number_integer: + JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>()); + break; + case value_t::number_unsigned: + JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>()); + break; + case value_t::string: + JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>()); + break; + case value_t::object: + JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>()); + break; + case value_t::array: + JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>()); + break; + case value_t::null: + *this = nullptr; + break; + case value_t::discarded: + m_type = value_t::discarded; + break; + } + assert_invariant(); + } + + /*! @brief create a container (array or object) from an initializer list Creates a JSON value of type array or object from the passed initializer @@ -8718,7 +12702,7 @@ class basic_json @warning A precondition is enforced with a runtime assertion that will result in calling `std::abort` if this precondition is not met. Assertions can be disabled by defining `NDEBUG` at compile time. - See http://en.cppreference.com/w/cpp/error/assert for more + See https://en.cppreference.com/w/cpp/error/assert for more information. @throw invalid_iterator.201 if iterators @a first and @a last are not @@ -8858,7 +12842,7 @@ class basic_json changes to any JSON value. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. - As postcondition, it holds: `other == basic_json(other)`. @@ -8943,7 +12927,7 @@ class basic_json exceptions. @requirement This function helps `basic_json` satisfying the - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible) + [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible) requirements. @liveexample{The code below shows the move constructor explicitly called @@ -8977,7 +12961,7 @@ class basic_json @complexity Linear. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. @@ -9014,14 +12998,14 @@ class basic_json @complexity Linear. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. - All stored elements are destroyed and all memory is freed. @since version 1.0.0 */ - ~basic_json() + ~basic_json() noexcept { assert_invariant(); m_value.destroy(m_type); @@ -9078,7 +13062,7 @@ class basic_json const bool ensure_ascii = false) const { string_t result; - serializer s(detail::output_adapter<char>(result), indent_char); + serializer s(detail::output_adapter<char, string_t>(result), indent_char); if (indent >= 0) { @@ -9619,11 +13603,34 @@ class basic_json } /*! + @brief get special-case overload + + This overloads converts the current @ref basic_json in a different + @ref basic_json type + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this, converted into @tparam BasicJsonType + + @complexity Depending on the implementation of the called `from_json()` + method. + + @since version 3.2.0 + */ + template<typename BasicJsonType, detail::enable_if_t< + not std::is_same<BasicJsonType, basic_json>::value and + detail::is_basic_json<BasicJsonType>::value, int> = 0> + BasicJsonType get() const + { + return *this; + } + + /*! @brief get a value (explicit) Explicit type conversion between the JSON value and a compatible value - which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) - and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). The value is converted by calling the @ref json_serializer<ValueType> `from_json()` method. @@ -9659,7 +13666,7 @@ class basic_json */ template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>, detail::enable_if_t < - not std::is_same<basic_json_t, ValueType>::value and + not detail::is_basic_json<ValueType>::value and detail::has_from_json<basic_json_t, ValueType>::value and not detail::has_non_default_from_json<basic_json_t, ValueType>::value, int> = 0> @@ -9683,8 +13690,8 @@ class basic_json @brief get a value (explicit); special case Explicit type conversion between the JSON value and a compatible value - which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) - and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). The value is converted by calling the @ref json_serializer<ValueType> `from_json()` method. @@ -9925,13 +13932,14 @@ class basic_json template < typename ValueType, typename std::enable_if < not std::is_pointer<ValueType>::value and not std::is_same<ValueType, detail::json_ref<basic_json>>::value and - not std::is_same<ValueType, typename string_t::value_type>::value + not std::is_same<ValueType, typename string_t::value_type>::value and + not detail::is_basic_json<ValueType>::value #ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value -#endif -#if defined(JSON_HAS_CPP_17) +#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914 and not std::is_same<ValueType, typename std::string_view>::value #endif +#endif , int >::type = 0 > operator ValueType() const { @@ -10512,7 +14520,7 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw type_error.306 if the JSON value is not an objec; in that case, + @throw type_error.306 if the JSON value is not an object; in that case, using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -10536,7 +14544,7 @@ class basic_json { return ptr.get_checked(this); } - JSON_CATCH (out_of_range&) + JSON_INTERNAL_CATCH (out_of_range&) { return default_value; } @@ -11047,7 +15055,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. @@ -11086,7 +15094,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).begin()`. @@ -11118,7 +15126,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. @@ -11157,7 +15165,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).end()`. @@ -11187,7 +15195,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `reverse_iterator(end())`. @@ -11224,7 +15232,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `reverse_iterator(begin())`. @@ -11261,7 +15269,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`. @@ -11290,7 +15298,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).rend()`. @@ -11361,18 +15369,89 @@ class basic_json @note The name of this function is not yet final and may change in the future. + + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use @ref items() instead; + that is, replace `json::iterator_wrapper(j)` with `j.items()`. */ - static iteration_proxy<iterator> iterator_wrapper(reference ref) + JSON_DEPRECATED + static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept { - return iteration_proxy<iterator>(ref); + return ref.items(); } /*! @copydoc iterator_wrapper(reference) */ - static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) + JSON_DEPRECATED + static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept + { + return ref.items(); + } + + /*! + @brief helper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + For loop without `items()` function: + + @code{cpp} + for (auto it = j_object.begin(); it != j_object.end(); ++it) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + Range-based for loop without `items()` function: + + @code{cpp} + for (auto it : j_object) + { + // "it" is of type json::reference and has no key() member + std::cout << "value: " << it << '\n'; + } + @endcode + + Range-based for loop with `items()` function: + + @code{cpp} + for (auto it : j_object.items()) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + @note When iterating over an array, `key()` will return the index of the + element as string (see example). For primitive types (e.g., numbers), + `key()` returns an empty string. + + @return iteration proxy object wrapping @a ref with an interface to use in + range-based for loops + + @liveexample{The following code shows how the function is used.,items} + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 3.1.0. + */ + iteration_proxy<iterator> items() noexcept { - return iteration_proxy<const_iterator>(ref); + return iteration_proxy<iterator>(*this); + } + + /*! + @copydoc items() + */ + iteration_proxy<const_iterator> items() const noexcept + { + return iteration_proxy<const_iterator>(*this); } /// @} @@ -11417,7 +15496,7 @@ class basic_json false in the case of a string. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `begin() == end()`. @@ -11488,7 +15567,7 @@ class basic_json the case of a string. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `std::distance(begin(), end())`. @@ -11558,7 +15637,7 @@ class basic_json @exceptionsafety No-throw guarantee: this function never throws exceptions. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of returning `b.size()` where `b` is the largest @@ -12312,7 +16391,7 @@ class basic_json // passed iterators must belong to objects if (JSON_UNLIKELY(not first.m_object->is_object() - or not first.m_object->is_object())) + or not last.m_object->is_object())) { JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); } @@ -12943,8 +17022,8 @@ class basic_json /*! @brief serialize to stream - @deprecated This stream operator is deprecated and will be removed in a - future version of the library. Please use + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use @ref operator<<(std::ostream&, const basic_json&) instead; that is, replace calls like `j >> o;` with `o << j;`. @since version 1.0.0; deprecated since version 3.0.0 @@ -13027,7 +17106,7 @@ class basic_json @since version 2.0.3 (contiguous containers) */ - static basic_json parse(detail::input_adapter i, + static basic_json parse(detail::input_adapter&& i, const parser_callback_t cb = nullptr, const bool allow_exceptions = true) { @@ -13036,26 +17115,80 @@ class basic_json return result; } - /*! - @copydoc basic_json parse(detail::input_adapter, const parser_callback_t) - */ - static basic_json parse(detail::input_adapter& i, - const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) - { - basic_json result; - parser(i, cb, allow_exceptions).parse(true, result); - return result; - } - - static bool accept(detail::input_adapter i) + static bool accept(detail::input_adapter&& i) { return parser(i).accept(true); } - static bool accept(detail::input_adapter& i) + /*! + @brief generate SAX events + + The SAX event lister must follow the interface of @ref json_sax. + + This function reads from a compatible input. Examples are: + - an array of 1-byte values + - strings with character/literal type with size of 1 byte + - input streams + - container with contiguous storage of 1-byte values. Compatible container + types include `std::vector`, `std::string`, `std::array`, + `std::valarray`, and `std::initializer_list`. Furthermore, C-style + arrays can be used with `std::begin()`/`std::end()`. User-defined + containers can be used as long as they implement random-access iterators + and a contiguous storage. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @param[in] i input to read from + @param[in,out] sax SAX event listener + @param[in] format the format to parse (JSON, CBOR, MessagePack, or UBJSON) + @param[in] strict whether the input has to be consumed completely + + @return return value of the last processed SAX event + + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the SAX consumer @a sax has + a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `sax_parse()` function + reading from string and processing the events with a user-defined SAX + event consumer.,sax_parse} + + @since version 3.2.0 + */ + template <typename SAX> + static bool sax_parse(detail::input_adapter&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true) { - return parser(i).accept(true); + assert(sax); + switch (format) + { + case input_format_t::json: + return parser(std::move(i)).sax_parse(sax, strict); + default: + return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict); + } } /*! @@ -13127,10 +17260,19 @@ class basic_json return parser(detail::input_adapter(first, last)).accept(true); } + template<class IteratorType, class SAX, typename std::enable_if< + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0> + static bool sax_parse(IteratorType first, IteratorType last, SAX* sax) + { + return parser(detail::input_adapter(first, last)).sax_parse(sax); + } + /*! @brief deserialize from stream - @deprecated This stream operator is deprecated and will be removed in a - future version of the library. Please use + @deprecated This stream operator is deprecated and will be removed in + version 4.0.0 of the library. Please use @ref operator>>(std::istream&, basic_json&) instead; that is, replace calls like `j << i;` with `i >> j;`. @since version 1.0.0; deprecated since version 3.0.0 @@ -13331,9 +17473,11 @@ class basic_json vector in CBOR format.,to_cbor} @sa http://cbor.io - @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the + @sa @ref from_cbor(detail::input_adapter, const bool strict) for the analogous deserialization @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format @since version 2.0.9 */ @@ -13429,6 +17573,8 @@ class basic_json @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the analogous deserialization @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format @since version 2.0.9 */ @@ -13450,6 +17596,107 @@ class basic_json } /*! + @brief create a UBJSON serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the UBJSON + (Universal Binary JSON) serialization format. UBJSON aims to be more compact + than JSON itself, yet more efficient to parse. + + The library uses the following mapping from JSON values types to + UBJSON types according to the UBJSON specification: + + JSON value type | value/range | UBJSON type | marker + --------------- | --------------------------------- | ----------- | ------ + null | `null` | null | `Z` + boolean | `true` | true | `T` + boolean | `false` | false | `F` + number_integer | -9223372036854775808..-2147483649 | int64 | `L` + number_integer | -2147483648..-32769 | int32 | `l` + number_integer | -32768..-129 | int16 | `I` + number_integer | -128..127 | int8 | `i` + number_integer | 128..255 | uint8 | `U` + number_integer | 256..32767 | int16 | `I` + number_integer | 32768..2147483647 | int32 | `l` + number_integer | 2147483648..9223372036854775807 | int64 | `L` + number_unsigned | 0..127 | int8 | `i` + number_unsigned | 128..255 | uint8 | `U` + number_unsigned | 256..32767 | int16 | `I` + number_unsigned | 32768..2147483647 | int32 | `l` + number_unsigned | 2147483648..9223372036854775807 | int64 | `L` + number_float | *any value* | float64 | `D` + string | *with shortest length indicator* | string | `S` + array | *see notes on optimized format* | array | `[` + object | *see notes on optimized format* | map | `{` + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a UBJSON value. + + @note The following values can **not** be converted to a UBJSON value: + - strings with more than 9223372036854775807 bytes (theoretical) + - unsigned integer numbers above 9223372036854775807 + + @note The following markers are not used in the conversion: + - `Z`: no-op values are not created. + - `C`: single-byte strings are serialized with `S` markers. + + @note Any UBJSON output created @ref to_ubjson can be successfully parsed + by @ref from_ubjson. + + @note If NaN or Infinity are stored inside a JSON number, they are + serialized properly. This behavior differs from the @ref dump() + function which serializes NaN or Infinity to `null`. + + @note The optimized formats for containers are supported: Parameter + @a use_size adds size information to the beginning of a container and + removes the closing marker. Parameter @a use_type further checks + whether all elements of a container have the same type and adds the + type marker to the beginning of the container. The @a use_type + parameter must only be used together with @a use_size = true. Note + that @a use_size = true alone may result in larger representations - + the benefit of this parameter is that the receiving side is + immediately informed on the number of elements of the container. + + @param[in] j JSON value to serialize + @param[in] use_size whether to add size annotations to container types + @param[in] use_type whether to add type annotations to container types + (must be combined with @a use_size = true) + @return UBJSON serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in UBJSON format.,to_ubjson} + + @sa http://ubjson.org + @sa @ref from_ubjson(detail::input_adapter, const bool strict) for the + analogous deserialization + @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + + @since version 3.1.0 + */ + static std::vector<uint8_t> to_ubjson(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector<uint8_t> result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type); + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<char> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<char>(o).write_ubjson(j, use_size, use_type); + } + + /*! @brief create a JSON value from an input in CBOR format Deserializes a given input @a i to a JSON value using the CBOR (Concise @@ -13518,6 +17765,9 @@ class basic_json @param[in] i an input in CBOR format convertible to an input adapter @param[in] strict whether to expect the input to be consumed until EOF (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + @return deserialized JSON value @throw parse_error.110 if the given input ends prematurely or the end of @@ -13533,27 +17783,39 @@ class basic_json @sa http://cbor.io @sa @ref to_cbor(const basic_json&) for the analogous serialization - @sa @ref from_msgpack(detail::input_adapter, const bool) for the + @sa @ref from_msgpack(detail::input_adapter, const bool, const bool) for the related MessagePack format + @sa @ref from_ubjson(detail::input_adapter, const bool, const bool) for the + related UBJSON format @since version 2.0.9; parameter @a start_index since 2.1.1; changed to consume input adapters, removed start_index parameter, and added - @a strict parameter since 3.0.0 + @a strict parameter since 3.0.0; added @allow_exceptions parameter + since 3.2.0 */ - static basic_json from_cbor(detail::input_adapter i, - const bool strict = true) + static basic_json from_cbor(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(i).parse_cbor(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! - @copydoc from_cbor(detail::input_adapter, const bool) + @copydoc from_cbor(detail::input_adapter, const bool, const bool) */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> - static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true) + static basic_json from_cbor(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_cbor(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! @@ -13606,6 +17868,10 @@ class basic_json adapter @param[in] strict whether to expect the input to be consumed until EOF (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -13620,27 +17886,121 @@ class basic_json @sa http://msgpack.org @sa @ref to_msgpack(const basic_json&) for the analogous serialization - @sa @ref from_cbor(detail::input_adapter, const bool) for the related CBOR - format + @sa @ref from_cbor(detail::input_adapter, const bool, const bool) for the + related CBOR format + @sa @ref from_ubjson(detail::input_adapter, const bool, const bool) for + the related UBJSON format @since version 2.0.9; parameter @a start_index since 2.1.1; changed to consume input adapters, removed start_index parameter, and added - @a strict parameter since 3.0.0 + @a strict parameter since 3.0.0; added @allow_exceptions parameter + since 3.2.0 */ - static basic_json from_msgpack(detail::input_adapter i, - const bool strict = true) + static basic_json from_msgpack(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(i).parse_msgpack(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! - @copydoc from_msgpack(detail::input_adapter, const bool) + @copydoc from_msgpack(detail::input_adapter, const bool, const bool) */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> - static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true) + static basic_json from_msgpack(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_msgpack(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @brief create a JSON value from an input in UBJSON format + + Deserializes a given input @a i to a JSON value using the UBJSON (Universal + Binary JSON) serialization format. + + The library maps UBJSON types to JSON value types as follows: + + UBJSON type | JSON value type | marker + ----------- | --------------------------------------- | ------ + no-op | *no value, next value is read* | `N` + null | `null` | `Z` + false | `false` | `F` + true | `true` | `T` + float32 | number_float | `d` + float64 | number_float | `D` + uint8 | number_unsigned | `U` + int8 | number_integer | `i` + int16 | number_integer | `I` + int32 | number_integer | `l` + int64 | number_integer | `L` + string | string | `S` + char | string | `C` + array | array (optimized values are supported) | `[` + object | object (optimized values are supported) | `{` + + @note The mapping is **complete** in the sense that any UBJSON value can + be converted to a JSON value. + + @param[in] i an input in UBJSON format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.110 if the given input ends prematurely or the end of + file was not reached when @a strict was set to true + @throw parse_error.112 if a parse error occurs + @throw parse_error.113 if a string could not be parsed successfully + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + UBJSON format to a JSON value.,from_ubjson} + + @sa http://ubjson.org + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + analogous serialization + @sa @ref from_cbor(detail::input_adapter, const bool, const bool) for the + related CBOR format + @sa @ref from_msgpack(detail::input_adapter, const bool, const bool) for + the related MessagePack format + + @since version 3.1.0; added @allow_exceptions parameter since 3.2.0 + */ + static basic_json from_ubjson(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_ubjson(detail::input_adapter, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_ubjson(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /// @} @@ -14017,11 +18377,13 @@ class basic_json break; } + // LCOV_EXCL_START default: { // if there exists a parent it cannot be primitive - assert(false); // LCOV_EXCL_LINE + assert(false); } + // LCOV_EXCL_STOP } } }; @@ -14066,7 +18428,7 @@ class basic_json // wrapper to get a value for an operation const auto get_value = [&val](const std::string & op, const std::string & member, - bool string_type) -> basic_json& + bool string_type) -> basic_json & { // find value auto it = val.m_value.object->find(member); @@ -14163,7 +18525,7 @@ class basic_json // the "path" location must exist - use at() success = (result.at(ptr) == get_value("test", "value", false)); } - JSON_CATCH (out_of_range&) + JSON_INTERNAL_CATCH (out_of_range&) { // ignore out of range errors: success remains false } @@ -14216,6 +18578,7 @@ class basic_json diff for two JSON values.,diff} @sa @ref patch -- apply a JSON patch + @sa @ref merge_patch -- apply a JSON Merge Patch @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) @@ -14347,418 +18710,86 @@ class basic_json } /// @} -}; - -///////////// -// presets // -///////////// - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. -@since version 1.0.0 -*/ -using json = basic_json<>; + //////////////////////////////// + // JSON Merge Patch functions // + //////////////////////////////// -////////////////// -// json_pointer // -////////////////// - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - auto result = &j; - - // in case no reference tokens exist, return a reference to the JSON value - // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) - { - switch (result->m_type) - { - case detail::value_t::null: - { - if (reference_token == "0") - { - // start a new array if reference token is 0 - result = &result->operator[](0); - } - else - { - // start a new object otherwise - result = &result->operator[](reference_token); - } - break; - } - - case detail::value_t::object: - { - // create an entry in the object - result = &result->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - // create an entry in the array - JSON_TRY - { - result = &result->operator[](static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } + /// @name JSON Merge Patch functions + /// @{ - /* - The following code is only reached if there exists a reference - token _and_ the current value is primitive. In this case, we have - an error situation, because primitive values may only occur as - single value; that is, with an empty list of reference tokens. - */ - default: - JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); - } - } + /*! + @brief applies a JSON Merge Patch + + The merge patch format is primarily intended for use with the HTTP PATCH + method as a means of describing a set of modifications to a target + resource's content. This function applies a merge patch to the current + JSON value. + + The function implements the following algorithm from Section 2 of + [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396): + + ``` + define MergePatch(Target, Patch): + if Patch is an Object: + if Target is not an Object: + Target = {} // Ignore the contents and set it to an empty Object + for each Name/Value pair in Patch: + if Value is null: + if Name exists in Target: + remove the Name/Value pair from Target + else: + Target[Name] = MergePatch(Target[Name], Value) + return Target + else: + return Patch + ``` + + Thereby, `Target` is the current object; that is, the patch is applied to + the current value. + + @param[in] patch the patch to apply + + @complexity Linear in the lengths of @a patch. + + @liveexample{The following code shows how a JSON Merge Patch is applied to + a JSON document.,merge_patch} - return *result; -} + @sa @ref patch -- apply a JSON patch + @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) + @since version 3.0.0 + */ + void merge_patch(const basic_json& patch) { - // convert null values to arrays or objects before continuing - if (ptr->m_type == detail::value_t::null) - { - // check if reference token is a number - const bool nums = - std::all_of(reference_token.begin(), reference_token.end(), - [](const char x) - { - return (x >= '0' and x <= '9'); - }); - - // change value to array for numbers or "-" or to object otherwise - *ptr = (nums or reference_token == "-") - ? detail::value_t::array - : detail::value_t::object; - } - - switch (ptr->m_type) + if (patch.is_object()) { - case detail::value_t::object: + if (not is_object()) { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; + *this = object(); } - - case detail::value_t::array: + for (auto it = patch.begin(); it != patch.end(); ++it) { - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - if (reference_token == "-") + if (it.value().is_null()) { - // explicitly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); + erase(it.key()); } else { - // convert array index to number; unchecked access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } + operator[](it.key()).merge_patch(it.value()); } - break; } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -const NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" cannot be used for const access - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // use unchecked array access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -const NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -void json_pointer::flatten(const std::string& reference_string, - const NLOHMANN_BASIC_JSON_TPL& value, - NLOHMANN_BASIC_JSON_TPL& result) -{ - switch (value.m_type) - { - case detail::value_t::array: - { - if (value.m_value.array->empty()) - { - // flatten empty array as null - result[reference_string] = nullptr; - } - else - { - // iterate array and use index as reference string - for (std::size_t i = 0; i < value.m_value.array->size(); ++i) - { - flatten(reference_string + "/" + std::to_string(i), - value.m_value.array->operator[](i), result); - } - } - break; - } - - case detail::value_t::object: - { - if (value.m_value.object->empty()) - { - // flatten empty object as null - result[reference_string] = nullptr; - } - else - { - // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) - { - flatten(reference_string + "/" + escape(element.first), element.second, result); - } - } - break; - } - - default: - { - // add primitive value with its reference string - result[reference_string] = value; - break; - } - } -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL -json_pointer::unflatten(const NLOHMANN_BASIC_JSON_TPL& value) -{ - if (JSON_UNLIKELY(not value.is_object())) - { - JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); - } - - NLOHMANN_BASIC_JSON_TPL result; - - // iterate the JSON object values - for (const auto& element : *value.m_value.object) - { - if (JSON_UNLIKELY(not element.second.is_primitive())) + else { - JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); + *this = patch; } - - // assign value to reference pointed to by JSON pointer; Note that if - // the JSON pointer is "" (i.e., points to the whole value), function - // get_and_create returns a reference to result itself. An assignment - // will then create a primitive value. - json_pointer(element.first).get_and_create(result) = element.second; } - return result; -} - -inline bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept -{ - return (lhs.reference_tokens == rhs.reference_tokens); -} - -inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept -{ - return not (lhs == rhs); -} + /// @} +}; } // namespace nlohmann - /////////////////////// // nonmember support // /////////////////////// @@ -14772,11 +18803,10 @@ namespace std @since version 1.0.0 */ template<> -inline void swap(nlohmann::json& j1, - nlohmann::json& j2) noexcept( - is_nothrow_move_constructible<nlohmann::json>::value and - is_nothrow_move_assignable<nlohmann::json>::value - ) +inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept( + is_nothrow_move_constructible<nlohmann::json>::value and + is_nothrow_move_assignable<nlohmann::json>::value +) { j1.swap(j2); } @@ -14853,6 +18883,9 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std return nlohmann::json::json_pointer(std::string(s, n)); } +// #include <nlohmann/detail/macro_unscope.hpp> + + // restore GCC/clang diagnostic settings #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop @@ -14862,13 +18895,18 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #endif // clean up +#undef JSON_INTERNAL_CATCH #undef JSON_CATCH #undef JSON_THROW #undef JSON_TRY #undef JSON_LIKELY #undef JSON_UNLIKELY #undef JSON_DEPRECATED +#undef JSON_HAS_CPP_14 +#undef JSON_HAS_CPP_17 #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION #undef NLOHMANN_BASIC_JSON_TPL +#undef NLOHMANN_JSON_HAS_HELPER + #endif diff --git a/ext/librabbitmq/centos_x64/include/amqp.h b/ext/librabbitmq/centos_x64/include/amqp.h new file mode 100644 index 00000000..2983b166 --- /dev/null +++ b/ext/librabbitmq/centos_x64/include/amqp.h @@ -0,0 +1,2538 @@ +/** \file */ +/* + * ***** BEGIN LICENSE BLOCK ***** + * Version: MIT + * + * Portions created by Alan Antonuk are Copyright (c) 2012-2014 + * Alan Antonuk. All Rights Reserved. + * + * Portions created by VMware are Copyright (c) 2007-2012 VMware, Inc. + * All Rights Reserved. + * + * Portions created by Tony Garnock-Jones are Copyright (c) 2009-2010 + * VMware, Inc. and Tony Garnock-Jones. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * ***** END LICENSE BLOCK ***** + */ + +#ifndef AMQP_H +#define AMQP_H + +/** \cond HIDE_FROM_DOXYGEN */ + +#ifdef __cplusplus +#define AMQP_BEGIN_DECLS extern "C" { +#define AMQP_END_DECLS } +#else +#define AMQP_BEGIN_DECLS +#define AMQP_END_DECLS +#endif + +/* + * \internal + * Important API decorators: + * AMQP_PUBLIC_FUNCTION - a public API function + * AMQP_PUBLIC_VARIABLE - a public API external variable + * AMQP_CALL - calling convension (used on Win32) + */ + +#if defined(_WIN32) && defined(_MSC_VER) +#if defined(AMQP_BUILD) && !defined(AMQP_STATIC) +#define AMQP_PUBLIC_FUNCTION __declspec(dllexport) +#define AMQP_PUBLIC_VARIABLE __declspec(dllexport) extern +#else +#define AMQP_PUBLIC_FUNCTION +#if !defined(AMQP_STATIC) +#define AMQP_PUBLIC_VARIABLE __declspec(dllimport) extern +#else +#define AMQP_PUBLIC_VARIABLE extern +#endif +#endif +#define AMQP_CALL __cdecl + +#elif defined(_WIN32) && defined(__BORLANDC__) +#if defined(AMQP_BUILD) && !defined(AMQP_STATIC) +#define AMQP_PUBLIC_FUNCTION __declspec(dllexport) +#define AMQP_PUBLIC_VARIABLE __declspec(dllexport) extern +#else +#define AMQP_PUBLIC_FUNCTION +#if !defined(AMQP_STATIC) +#define AMQP_PUBLIC_VARIABLE __declspec(dllimport) extern +#else +#define AMQP_PUBLIC_VARIABLE extern +#endif +#endif +#define AMQP_CALL __cdecl + +#elif defined(_WIN32) && defined(__MINGW32__) +#if defined(AMQP_BUILD) && !defined(AMQP_STATIC) +#define AMQP_PUBLIC_FUNCTION __declspec(dllexport) +#define AMQP_PUBLIC_VARIABLE __declspec(dllexport) extern +#else +#define AMQP_PUBLIC_FUNCTION +#if !defined(AMQP_STATIC) +#define AMQP_PUBLIC_VARIABLE __declspec(dllimport) extern +#else +#define AMQP_PUBLIC_VARIABLE extern +#endif +#endif +#define AMQP_CALL __cdecl + +#elif defined(_WIN32) && defined(__CYGWIN__) +#if defined(AMQP_BUILD) && !defined(AMQP_STATIC) +#define AMQP_PUBLIC_FUNCTION __declspec(dllexport) +#define AMQP_PUBLIC_VARIABLE __declspec(dllexport) +#else +#define AMQP_PUBLIC_FUNCTION +#if !defined(AMQP_STATIC) +#define AMQP_PUBLIC_VARIABLE __declspec(dllimport) extern +#else +#define AMQP_PUBLIC_VARIABLE extern +#endif +#endif +#define AMQP_CALL __cdecl + +#elif defined(__GNUC__) && __GNUC__ >= 4 +#define AMQP_PUBLIC_FUNCTION __attribute__((visibility("default"))) +#define AMQP_PUBLIC_VARIABLE __attribute__((visibility("default"))) extern +#define AMQP_CALL +#else +#define AMQP_PUBLIC_FUNCTION +#define AMQP_PUBLIC_VARIABLE extern +#define AMQP_CALL +#endif + +#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1) +#define AMQP_DEPRECATED(function) function __attribute__((__deprecated__)) +#elif defined(_MSC_VER) +#define AMQP_DEPRECATED(function) __declspec(deprecated) function +#else +#define AMQP_DEPRECATED(function) +#endif + +/* Define ssize_t on Win32/64 platforms + See: http://lists.cs.uiuc.edu/pipermail/llvmdev/2010-April/030649.html for + details + */ +#if !defined(_W64) +#if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +#define _W64 __w64 +#else +#define _W64 +#endif +#endif + +#ifdef _MSC_VER +#ifdef _WIN64 +typedef __int64 ssize_t; +#else +typedef _W64 int ssize_t; +#endif +#endif + +#if defined(_WIN32) && defined(__MINGW32__) +#include <sys/types.h> +#endif + +/** \endcond */ + +#include <stddef.h> +#include <stdint.h> + +struct timeval; + +AMQP_BEGIN_DECLS + +/** + * \def AMQP_VERSION_MAJOR + * + * Major library version number compile-time constant + * + * The major version is incremented when backwards incompatible API changes + * are made. + * + * \sa AMQP_VERSION, AMQP_VERSION_STRING + * + * \since v0.4.0 + */ + +/** + * \def AMQP_VERSION_MINOR + * + * Minor library version number compile-time constant + * + * The minor version is incremented when new APIs are added. Existing APIs + * are left alone. + * + * \sa AMQP_VERSION, AMQP_VERSION_STRING + * + * \since v0.4.0 + */ + +/** + * \def AMQP_VERSION_PATCH + * + * Patch library version number compile-time constant + * + * The patch version is incremented when library code changes, but the API + * is not changed. + * + * \sa AMQP_VERSION, AMQP_VERSION_STRING + * + * \since v0.4.0 + */ + +/** + * \def AMQP_VERSION_IS_RELEASE + * + * Version constant set to 1 for tagged release, 0 otherwise + * + * NOTE: versions that are not tagged releases are not guaranteed to be API/ABI + * compatible with older releases, and may change commit-to-commit. + * + * \sa AMQP_VERSION, AMQP_VERSION_STRING + * + * \since v0.4.0 + */ +/* + * Developer note: when changing these, be sure to update SOVERSION constants + * in CMakeLists.txt and configure.ac + */ + +#define AMQP_VERSION_MAJOR 0 +#define AMQP_VERSION_MINOR 10 +#define AMQP_VERSION_PATCH 0 +#define AMQP_VERSION_IS_RELEASE 0 + +/** + * \def AMQP_VERSION_CODE + * + * Helper macro to geneate a packed version code suitable for + * comparison with AMQP_VERSION. + * + * \sa amqp_version_number() AMQP_VERSION_MAJOR, AMQP_VERSION_MINOR, + * AMQP_VERSION_PATCH, AMQP_VERSION_IS_RELEASE, AMQP_VERSION + * + * \since v0.6.1 + */ +#define AMQP_VERSION_CODE(major, minor, patch, release) \ + ((major << 24) | (minor << 16) | (patch << 8) | (release)) + +/** + * \def AMQP_VERSION + * + * Packed version number + * + * AMQP_VERSION is a 4-byte unsigned integer with the most significant byte + * set to AMQP_VERSION_MAJOR, the second most significant byte set to + * AMQP_VERSION_MINOR, third most significant byte set to AMQP_VERSION_PATCH, + * and the lowest byte set to AMQP_VERSION_IS_RELEASE. + * + * For example version 2.3.4 which is released version would be encoded as + * 0x02030401 + * + * \sa amqp_version_number() AMQP_VERSION_MAJOR, AMQP_VERSION_MINOR, + * AMQP_VERSION_PATCH, AMQP_VERSION_IS_RELEASE, AMQP_VERSION_CODE + * + * \since v0.4.0 + */ +#define AMQP_VERSION \ + AMQP_VERSION_CODE(AMQP_VERSION_MAJOR, AMQP_VERSION_MINOR, \ + AMQP_VERSION_PATCH, AMQP_VERSION_IS_RELEASE) + +/** \cond HIDE_FROM_DOXYGEN */ +#define AMQ_STRINGIFY(s) AMQ_STRINGIFY_HELPER(s) +#define AMQ_STRINGIFY_HELPER(s) #s + +#define AMQ_VERSION_STRING \ + AMQ_STRINGIFY(AMQP_VERSION_MAJOR) \ + "." AMQ_STRINGIFY(AMQP_VERSION_MINOR) "." AMQ_STRINGIFY(AMQP_VERSION_PATCH) +/** \endcond */ + +/** + * \def AMQP_VERSION_STRING + * + * Version string compile-time constant + * + * Non-released versions of the library will have "-pre" appended to the + * version string + * + * \sa amqp_version() + * + * \since v0.4.0 + */ +#if AMQP_VERSION_IS_RELEASE +#define AMQP_VERSION_STRING AMQ_VERSION_STRING +#else +#define AMQP_VERSION_STRING AMQ_VERSION_STRING "-pre" +#endif + +/** + * Returns the rabbitmq-c version as a packed integer. + * + * See \ref AMQP_VERSION + * + * \return packed 32-bit integer representing version of library at runtime + * + * \sa AMQP_VERSION, amqp_version() + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +uint32_t AMQP_CALL amqp_version_number(void); + +/** + * Returns the rabbitmq-c version as a string. + * + * See \ref AMQP_VERSION_STRING + * + * \return a statically allocated string describing the version of rabbitmq-c. + * + * \sa amqp_version_number(), AMQP_VERSION_STRING, AMQP_VERSION + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +char const *AMQP_CALL amqp_version(void); + +/** + * \def AMQP_DEFAULT_FRAME_SIZE + * + * Default frame size (128Kb) + * + * \sa amqp_login(), amqp_login_with_properties() + * + * \since v0.4.0 + */ +#define AMQP_DEFAULT_FRAME_SIZE 131072 + +/** + * \def AMQP_DEFAULT_MAX_CHANNELS + * + * Default maximum number of channels (2047, RabbitMQ default limit of 2048, + * minus 1 for channel 0). RabbitMQ set a default limit of 2048 channels per + * connection in v3.7.5 to prevent broken clients from leaking too many + * channels. + * + * \sa amqp_login(), amqp_login_with_properties() + * + * \since v0.4.0 + */ +#define AMQP_DEFAULT_MAX_CHANNELS 2047 + +/** + * \def AMQP_DEFAULT_HEARTBEAT + * + * Default heartbeat interval (0, heartbeat disabled) + * + * \sa amqp_login(), amqp_login_with_properties() + * + * \since v0.4.0 + */ +#define AMQP_DEFAULT_HEARTBEAT 0 + +/** + * \def AMQP_DEFAULT_VHOST + * + * Default RabbitMQ vhost: "/" + * + * \sa amqp_login(), amqp_login_with_properties() + * + * \since v0.9.0 + */ +#define AMQP_DEFAULT_VHOST "/" + +/** + * boolean type 0 = false, true otherwise + * + * \since v0.1 + */ +typedef int amqp_boolean_t; + +/** + * Method number + * + * \since v0.1 + */ +typedef uint32_t amqp_method_number_t; + +/** + * Bitmask for flags + * + * \since v0.1 + */ +typedef uint32_t amqp_flags_t; + +/** + * Channel type + * + * \since v0.1 + */ +typedef uint16_t amqp_channel_t; + +/** + * Buffer descriptor + * + * \since v0.1 + */ +typedef struct amqp_bytes_t_ { + size_t len; /**< length of the buffer in bytes */ + void *bytes; /**< pointer to the beginning of the buffer */ +} amqp_bytes_t; + +/** + * Decimal data type + * + * \since v0.1 + */ +typedef struct amqp_decimal_t_ { + uint8_t decimals; /**< the location of the decimal point */ + uint32_t value; /**< the value before the decimal point is applied */ +} amqp_decimal_t; + +/** + * AMQP field table + * + * An AMQP field table is a set of key-value pairs. + * A key is a UTF-8 encoded string up to 128 bytes long, and are not null + * terminated. + * A value can be one of several different datatypes. \sa + * amqp_field_value_kind_t + * + * \sa amqp_table_entry_t + * + * \since v0.1 + */ +typedef struct amqp_table_t_ { + int num_entries; /**< length of entries array */ + struct amqp_table_entry_t_ *entries; /**< an array of table entries */ +} amqp_table_t; + +/** + * An AMQP Field Array + * + * A repeated set of field values, all must be of the same type + * + * \since v0.1 + */ +typedef struct amqp_array_t_ { + int num_entries; /**< Number of entries in the table */ + struct amqp_field_value_t_ *entries; /**< linked list of field values */ +} amqp_array_t; + +/* + 0-9 0-9-1 Qpid/Rabbit Type Remarks +--------------------------------------------------------------------------- + t t Boolean + b b Signed 8-bit + B Unsigned 8-bit + U s Signed 16-bit (A1) + u Unsigned 16-bit + I I I Signed 32-bit + i Unsigned 32-bit + L l Signed 64-bit (B) + l Unsigned 64-bit + f f 32-bit float + d d 64-bit float + D D D Decimal + s Short string (A2) + S S S Long string + A Nested Array + T T T Timestamp (u64) + F F F Nested Table + V V V Void + x Byte array + +Remarks: + + A1, A2: Notice how the types **CONFLICT** here. In Qpid and Rabbit, + 's' means a signed 16-bit integer; in 0-9-1, it means a + short string. + + B: Notice how the signednesses **CONFLICT** here. In Qpid and Rabbit, + 'l' means a signed 64-bit integer; in 0-9-1, it means an unsigned + 64-bit integer. + +I'm going with the Qpid/Rabbit types, where there's a conflict, and +the 0-9-1 types otherwise. 0-8 is a subset of 0-9, which is a subset +of the other two, so this will work for both 0-8 and 0-9-1 branches of +the code. +*/ + +/** + * A field table value + * + * \since v0.1 + */ +typedef struct amqp_field_value_t_ { + uint8_t kind; /**< the type of the entry /sa amqp_field_value_kind_t */ + union { + amqp_boolean_t boolean; /**< boolean type AMQP_FIELD_KIND_BOOLEAN */ + int8_t i8; /**< int8_t type AMQP_FIELD_KIND_I8 */ + uint8_t u8; /**< uint8_t type AMQP_FIELD_KIND_U8 */ + int16_t i16; /**< int16_t type AMQP_FIELD_KIND_I16 */ + uint16_t u16; /**< uint16_t type AMQP_FIELD_KIND_U16 */ + int32_t i32; /**< int32_t type AMQP_FIELD_KIND_I32 */ + uint32_t u32; /**< uint32_t type AMQP_FIELD_KIND_U32 */ + int64_t i64; /**< int64_t type AMQP_FIELD_KIND_I64 */ + uint64_t u64; /**< uint64_t type AMQP_FIELD_KIND_U64, + AMQP_FIELD_KIND_TIMESTAMP */ + float f32; /**< float type AMQP_FIELD_KIND_F32 */ + double f64; /**< double type AMQP_FIELD_KIND_F64 */ + amqp_decimal_t decimal; /**< amqp_decimal_t AMQP_FIELD_KIND_DECIMAL */ + amqp_bytes_t bytes; /**< amqp_bytes_t type AMQP_FIELD_KIND_UTF8, + AMQP_FIELD_KIND_BYTES */ + amqp_table_t table; /**< amqp_table_t type AMQP_FIELD_KIND_TABLE */ + amqp_array_t array; /**< amqp_array_t type AMQP_FIELD_KIND_ARRAY */ + } value; /**< a union of the value */ +} amqp_field_value_t; + +/** + * An entry in a field-table + * + * \sa amqp_table_encode(), amqp_table_decode(), amqp_table_clone() + * + * \since v0.1 + */ +typedef struct amqp_table_entry_t_ { + amqp_bytes_t key; /**< the table entry key. Its a null-terminated UTF-8 + * string, with a maximum size of 128 bytes */ + amqp_field_value_t value; /**< the table entry values */ +} amqp_table_entry_t; + +/** + * Field value types + * + * \since v0.1 + */ +typedef enum { + AMQP_FIELD_KIND_BOOLEAN = + 't', /**< boolean type. 0 = false, 1 = true @see amqp_boolean_t */ + AMQP_FIELD_KIND_I8 = 'b', /**< 8-bit signed integer, datatype: int8_t */ + AMQP_FIELD_KIND_U8 = 'B', /**< 8-bit unsigned integer, datatype: uint8_t */ + AMQP_FIELD_KIND_I16 = 's', /**< 16-bit signed integer, datatype: int16_t */ + AMQP_FIELD_KIND_U16 = 'u', /**< 16-bit unsigned integer, datatype: uint16_t */ + AMQP_FIELD_KIND_I32 = 'I', /**< 32-bit signed integer, datatype: int32_t */ + AMQP_FIELD_KIND_U32 = 'i', /**< 32-bit unsigned integer, datatype: uint32_t */ + AMQP_FIELD_KIND_I64 = 'l', /**< 64-bit signed integer, datatype: int64_t */ + AMQP_FIELD_KIND_U64 = 'L', /**< 64-bit unsigned integer, datatype: uint64_t */ + AMQP_FIELD_KIND_F32 = + 'f', /**< single-precision floating point value, datatype: float */ + AMQP_FIELD_KIND_F64 = + 'd', /**< double-precision floating point value, datatype: double */ + AMQP_FIELD_KIND_DECIMAL = + 'D', /**< amqp-decimal value, datatype: amqp_decimal_t */ + AMQP_FIELD_KIND_UTF8 = 'S', /**< UTF-8 null-terminated character string, + datatype: amqp_bytes_t */ + AMQP_FIELD_KIND_ARRAY = 'A', /**< field array (repeated values of another + datatype. datatype: amqp_array_t */ + AMQP_FIELD_KIND_TIMESTAMP = 'T', /**< 64-bit timestamp. datatype uint64_t */ + AMQP_FIELD_KIND_TABLE = 'F', /**< field table. encapsulates a table inside a + table entry. datatype: amqp_table_t */ + AMQP_FIELD_KIND_VOID = 'V', /**< empty entry */ + AMQP_FIELD_KIND_BYTES = + 'x' /**< unformatted byte string, datatype: amqp_bytes_t */ +} amqp_field_value_kind_t; + +/** + * A list of allocation blocks + * + * \since v0.1 + */ +typedef struct amqp_pool_blocklist_t_ { + int num_blocks; /**< Number of blocks in the block list */ + void **blocklist; /**< Array of memory blocks */ +} amqp_pool_blocklist_t; + +/** + * A memory pool + * + * \since v0.1 + */ +typedef struct amqp_pool_t_ { + size_t pagesize; /**< the size of the page in bytes. Allocations less than or + * equal to this size are allocated in the pages block list. + * Allocations greater than this are allocated in their own + * own block in the large_blocks block list */ + + amqp_pool_blocklist_t pages; /**< blocks that are the size of pagesize */ + amqp_pool_blocklist_t + large_blocks; /**< allocations larger than the pagesize */ + + int next_page; /**< an index to the next unused page block */ + char *alloc_block; /**< pointer to the current allocation block */ + size_t alloc_used; /**< number of bytes in the current allocation block that + has been used */ +} amqp_pool_t; + +/** + * An amqp method + * + * \since v0.1 + */ +typedef struct amqp_method_t_ { + amqp_method_number_t id; /**< the method id number */ + void *decoded; /**< pointer to the decoded method, + * cast to the appropriate type to use */ +} amqp_method_t; + +/** + * An AMQP frame + * + * \since v0.1 + */ +typedef struct amqp_frame_t_ { + uint8_t frame_type; /**< frame type. The types: + * - AMQP_FRAME_METHOD - use the method union member + * - AMQP_FRAME_HEADER - use the properties union member + * - AMQP_FRAME_BODY - use the body_fragment union member + */ + amqp_channel_t channel; /**< the channel the frame was received on */ + union { + amqp_method_t + method; /**< a method, use if frame_type == AMQP_FRAME_METHOD */ + struct { + uint16_t class_id; /**< the class for the properties */ + uint64_t body_size; /**< size of the body in bytes */ + void *decoded; /**< the decoded properties */ + amqp_bytes_t raw; /**< amqp-encoded properties structure */ + } properties; /**< message header, a.k.a., properties, + use if frame_type == AMQP_FRAME_HEADER */ + amqp_bytes_t body_fragment; /**< a body fragment, use if frame_type == + AMQP_FRAME_BODY */ + struct { + uint8_t transport_high; /**< @internal first byte of handshake */ + uint8_t transport_low; /**< @internal second byte of handshake */ + uint8_t protocol_version_major; /**< @internal third byte of handshake */ + uint8_t protocol_version_minor; /**< @internal fourth byte of handshake */ + } protocol_header; /**< Used only when doing the initial handshake with the + broker, don't use otherwise */ + } payload; /**< the payload of the frame */ +} amqp_frame_t; + +/** + * Response type + * + * \since v0.1 + */ +typedef enum amqp_response_type_enum_ { + AMQP_RESPONSE_NONE = 0, /**< the library got an EOF from the socket */ + AMQP_RESPONSE_NORMAL, /**< response normal, the RPC completed successfully */ + AMQP_RESPONSE_LIBRARY_EXCEPTION, /**< library error, an error occurred in the + library, examine the library_error */ + AMQP_RESPONSE_SERVER_EXCEPTION /**< server exception, the broker returned an + error, check replay */ +} amqp_response_type_enum; + +/** + * Reply from a RPC method on the broker + * + * \since v0.1 + */ +typedef struct amqp_rpc_reply_t_ { + amqp_response_type_enum reply_type; /**< the reply type: + * - AMQP_RESPONSE_NORMAL - the RPC + * completed successfully + * - AMQP_RESPONSE_SERVER_EXCEPTION - the + * broker returned + * an exception, check the reply field + * - AMQP_RESPONSE_LIBRARY_EXCEPTION - the + * library + * encountered an error, check the + * library_error field + */ + amqp_method_t reply; /**< in case of AMQP_RESPONSE_SERVER_EXCEPTION this + * field will be set to the method returned from the + * broker */ + int library_error; /**< in case of AMQP_RESPONSE_LIBRARY_EXCEPTION this + * field will be set to an error code. An error + * string can be retrieved using amqp_error_string */ +} amqp_rpc_reply_t; + +/** + * SASL method type + * + * \since v0.1 + */ +typedef enum amqp_sasl_method_enum_ { + AMQP_SASL_METHOD_UNDEFINED = -1, /**< Invalid SASL method */ + AMQP_SASL_METHOD_PLAIN = + 0, /**< the PLAIN SASL method for authentication to the broker */ + AMQP_SASL_METHOD_EXTERNAL = + 1 /**< the EXTERNAL SASL method for authentication to the broker */ +} amqp_sasl_method_enum; + +/** + * connection state object + * + * \since v0.1 + */ +typedef struct amqp_connection_state_t_ *amqp_connection_state_t; + +/** + * Socket object + * + * \since v0.4.0 + */ +typedef struct amqp_socket_t_ amqp_socket_t; + +/** + * Status codes + * + * \since v0.4.0 + */ +/* NOTE: When updating this enum, update the strings in librabbitmq/amqp_api.c + */ +typedef enum amqp_status_enum_ { + AMQP_STATUS_OK = 0x0, /**< Operation successful */ + AMQP_STATUS_NO_MEMORY = -0x0001, /**< Memory allocation + failed */ + AMQP_STATUS_BAD_AMQP_DATA = -0x0002, /**< Incorrect or corrupt + data was received from + the broker. This is a + protocol error. */ + AMQP_STATUS_UNKNOWN_CLASS = -0x0003, /**< An unknown AMQP class + was received. This is + a protocol error. */ + AMQP_STATUS_UNKNOWN_METHOD = -0x0004, /**< An unknown AMQP method + was received. This is + a protocol error. */ + AMQP_STATUS_HOSTNAME_RESOLUTION_FAILED = -0x0005, /**< Unable to resolve the + * hostname */ + AMQP_STATUS_INCOMPATIBLE_AMQP_VERSION = -0x0006, /**< The broker advertised + an incompaible AMQP + version */ + AMQP_STATUS_CONNECTION_CLOSED = -0x0007, /**< The connection to the + broker has been closed + */ + AMQP_STATUS_BAD_URL = -0x0008, /**< malformed AMQP URL */ + AMQP_STATUS_SOCKET_ERROR = -0x0009, /**< A socket error + occurred */ + AMQP_STATUS_INVALID_PARAMETER = -0x000A, /**< An invalid parameter + was passed into the + function */ + AMQP_STATUS_TABLE_TOO_BIG = -0x000B, /**< The amqp_table_t object + cannot be serialized + because the output + buffer is too small */ + AMQP_STATUS_WRONG_METHOD = -0x000C, /**< The wrong method was + received */ + AMQP_STATUS_TIMEOUT = -0x000D, /**< Operation timed out */ + AMQP_STATUS_TIMER_FAILURE = -0x000E, /**< The underlying system + timer facility failed */ + AMQP_STATUS_HEARTBEAT_TIMEOUT = -0x000F, /**< Timed out waiting for + heartbeat */ + AMQP_STATUS_UNEXPECTED_STATE = -0x0010, /**< Unexpected protocol + state */ + AMQP_STATUS_SOCKET_CLOSED = -0x0011, /**< Underlying socket is + closed */ + AMQP_STATUS_SOCKET_INUSE = -0x0012, /**< Underlying socket is + already open */ + AMQP_STATUS_BROKER_UNSUPPORTED_SASL_METHOD = -0x0013, /**< Broker does not + support the requested + SASL mechanism */ + AMQP_STATUS_UNSUPPORTED = -0x0014, /**< Parameter is unsupported + in this version */ + _AMQP_STATUS_NEXT_VALUE = -0x0015, /**< Internal value */ + + AMQP_STATUS_TCP_ERROR = -0x0100, /**< A generic TCP error + occurred */ + AMQP_STATUS_TCP_SOCKETLIB_INIT_ERROR = -0x0101, /**< An error occurred trying + to initialize the + socket library*/ + _AMQP_STATUS_TCP_NEXT_VALUE = -0x0102, /**< Internal value */ + + AMQP_STATUS_SSL_ERROR = -0x0200, /**< A generic SSL error + occurred. */ + AMQP_STATUS_SSL_HOSTNAME_VERIFY_FAILED = -0x0201, /**< SSL validation of + hostname against + peer certificate + failed */ + AMQP_STATUS_SSL_PEER_VERIFY_FAILED = -0x0202, /**< SSL validation of peer + certificate failed. */ + AMQP_STATUS_SSL_CONNECTION_FAILED = -0x0203, /**< SSL handshake failed. */ + _AMQP_STATUS_SSL_NEXT_VALUE = -0x0204 /**< Internal value */ +} amqp_status_enum; + +/** + * AMQP delivery modes. + * Use these values for the #amqp_basic_properties_t::delivery_mode field. + * + * \since v0.5 + */ +typedef enum { + AMQP_DELIVERY_NONPERSISTENT = 1, /**< Non-persistent message */ + AMQP_DELIVERY_PERSISTENT = 2 /**< Persistent message */ +} amqp_delivery_mode_enum; + +AMQP_END_DECLS + +#include <amqp_framing.h> + +AMQP_BEGIN_DECLS + +/** + * Empty bytes structure + * + * \since v0.2 + */ +AMQP_PUBLIC_VARIABLE const amqp_bytes_t amqp_empty_bytes; + +/** + * Empty table structure + * + * \since v0.2 + */ +AMQP_PUBLIC_VARIABLE const amqp_table_t amqp_empty_table; + +/** + * Empty table array structure + * + * \since v0.2 + */ +AMQP_PUBLIC_VARIABLE const amqp_array_t amqp_empty_array; + +/* Compatibility macros for the above, to avoid the need to update + code written against earlier versions of librabbitmq. */ + +/** + * \def AMQP_EMPTY_BYTES + * + * Deprecated, use \ref amqp_empty_bytes instead + * + * \deprecated use \ref amqp_empty_bytes instead + * + * \since v0.1 + */ +#define AMQP_EMPTY_BYTES amqp_empty_bytes + +/** + * \def AMQP_EMPTY_TABLE + * + * Deprecated, use \ref amqp_empty_table instead + * + * \deprecated use \ref amqp_empty_table instead + * + * \since v0.1 + */ +#define AMQP_EMPTY_TABLE amqp_empty_table + +/** + * \def AMQP_EMPTY_ARRAY + * + * Deprecated, use \ref amqp_empty_array instead + * + * \deprecated use \ref amqp_empty_array instead + * + * \since v0.1 + */ +#define AMQP_EMPTY_ARRAY amqp_empty_array + +/** + * Initializes an amqp_pool_t memory allocation pool for use + * + * Readies an allocation pool for use. An amqp_pool_t + * must be initialized before use + * + * \param [in] pool the amqp_pool_t structure to initialize. + * Calling this function on a pool a pool that has + * already been initialized will result in undefined + * behavior + * \param [in] pagesize the unit size that the pool will allocate + * memory chunks in. Anything allocated against the pool + * with a requested size will be carved out of a block + * this size. Allocations larger than this will be + * allocated individually + * + * \sa recycle_amqp_pool(), empty_amqp_pool(), amqp_pool_alloc(), + * amqp_pool_alloc_bytes(), amqp_pool_t + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL init_amqp_pool(amqp_pool_t *pool, size_t pagesize); + +/** + * Recycles an amqp_pool_t memory allocation pool + * + * Recycles the space allocate by the pool + * + * This invalidates all allocations made against the pool before this call is + * made, any use of any allocations made before recycle_amqp_pool() is called + * will result in undefined behavior. + * + * Note: this may or may not release memory, to force memory to be released + * call empty_amqp_pool(). + * + * \param [in] pool the amqp_pool_t to recycle + * + * \sa recycle_amqp_pool(), empty_amqp_pool(), amqp_pool_alloc(), + * amqp_pool_alloc_bytes() + * + * \since v0.1 + * + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL recycle_amqp_pool(amqp_pool_t *pool); + +/** + * Empties an amqp memory pool + * + * Releases all memory associated with an allocation pool + * + * \param [in] pool the amqp_pool_t to empty + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL empty_amqp_pool(amqp_pool_t *pool); + +/** + * Allocates a block of memory from an amqp_pool_t memory pool + * + * Memory will be aligned on a 8-byte boundary. If a 0-length allocation is + * requested, a NULL pointer will be returned. + * + * \param [in] pool the allocation pool to allocate the memory from + * \param [in] amount the size of the allocation in bytes. + * \return a pointer to the memory block, or NULL if the allocation cannot + * be satisfied. + * + * \sa init_amqp_pool(), recycle_amqp_pool(), empty_amqp_pool(), + * amqp_pool_alloc_bytes() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void *AMQP_CALL amqp_pool_alloc(amqp_pool_t *pool, size_t amount); + +/** + * Allocates a block of memory from an amqp_pool_t to an amqp_bytes_t + * + * Memory will be aligned on a 8-byte boundary. If a 0-length allocation is + * requested, output.bytes = NULL. + * + * \param [in] pool the allocation pool to allocate the memory from + * \param [in] amount the size of the allocation in bytes + * \param [in] output the location to store the pointer. On success + * output.bytes will be set to the beginning of the buffer + * output.len will be set to amount + * On error output.bytes will be set to NULL and output.len + * set to 0 + * + * \sa init_amqp_pool(), recycle_amqp_pool(), empty_amqp_pool(), + * amqp_pool_alloc() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_pool_alloc_bytes(amqp_pool_t *pool, size_t amount, + amqp_bytes_t *output); + +/** + * Wraps a c string in an amqp_bytes_t + * + * Takes a string, calculates its length and creates an + * amqp_bytes_t that points to it. The string is not duplicated. + * + * For a given input cstr, The amqp_bytes_t output.bytes is the + * same as cstr, output.len is the length of the string not including + * the \0 terminator + * + * This function uses strlen() internally so cstr must be properly + * terminated + * + * \param [in] cstr the c string to wrap + * \return an amqp_bytes_t that describes the string + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_bytes_t AMQP_CALL amqp_cstring_bytes(char const *cstr); + +/** + * Duplicates an amqp_bytes_t buffer. + * + * The buffer is cloned and the contents copied. + * + * The memory associated with the output is allocated + * with amqp_bytes_malloc() and should be freed with + * amqp_bytes_free() + * + * \param [in] src + * \return a clone of the src + * + * \sa amqp_bytes_free(), amqp_bytes_malloc() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_bytes_t AMQP_CALL amqp_bytes_malloc_dup(amqp_bytes_t src); + +/** + * Allocates a amqp_bytes_t buffer + * + * Creates an amqp_bytes_t buffer of the specified amount, the buffer should be + * freed using amqp_bytes_free() + * + * \param [in] amount the size of the buffer in bytes + * \returns an amqp_bytes_t with amount bytes allocated. + * output.bytes will be set to NULL on error + * + * \sa amqp_bytes_free(), amqp_bytes_malloc_dup() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_bytes_t AMQP_CALL amqp_bytes_malloc(size_t amount); + +/** + * Frees an amqp_bytes_t buffer + * + * Frees a buffer allocated with amqp_bytes_malloc() or amqp_bytes_malloc_dup() + * + * Calling amqp_bytes_free on buffers not allocated with one + * of those two functions will result in undefined behavior + * + * \param [in] bytes the buffer to free + * + * \sa amqp_bytes_malloc(), amqp_bytes_malloc_dup() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_bytes_free(amqp_bytes_t bytes); + +/** + * Allocate and initialize a new amqp_connection_state_t object + * + * amqp_connection_state_t objects created with this function + * should be freed with amqp_destroy_connection() + * + * \returns an opaque pointer on success, NULL or 0 on failure. + * + * \sa amqp_destroy_connection() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_connection_state_t AMQP_CALL amqp_new_connection(void); + +/** + * Get the underlying socket descriptor for the connection + * + * \warning Use the socket returned from this function carefully, incorrect use + * of the socket outside of the library will lead to undefined behavior. + * Additionally rabbitmq-c may use the socket differently version-to-version, + * what may work in one version, may break in the next version. Be sure to + * throughly test any applications that use the socket returned by this + * function especially when using a newer version of rabbitmq-c + * + * \param [in] state the connection object + * \returns the socket descriptor if one has been set, -1 otherwise + * + * \sa amqp_tcp_socket_new(), amqp_ssl_socket_new(), amqp_socket_open() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_get_sockfd(amqp_connection_state_t state); + +/** + * Deprecated, use amqp_tcp_socket_new() or amqp_ssl_socket_new() + * + * \deprecated Use amqp_tcp_socket_new() or amqp_ssl_socket_new() + * + * Sets the socket descriptor associated with the connection. The socket + * should be connected to a broker, and should not be read to or written from + * before calling this function. A socket descriptor can be created and opened + * using amqp_open_socket() + * + * \param [in] state the connection object + * \param [in] sockfd the socket + * + * \sa amqp_open_socket(), amqp_tcp_socket_new(), amqp_ssl_socket_new() + * + * \since v0.1 + */ +AMQP_DEPRECATED(AMQP_PUBLIC_FUNCTION void AMQP_CALL + amqp_set_sockfd(amqp_connection_state_t state, int sockfd)); + +/** + * Tune client side parameters + * + * \warning This function may call abort() if the connection is in a certain + * state. As such it should probably not be called code outside the library. + * connection parameters should be specified when calling amqp_login() or + * amqp_login_with_properties() + * + * This function changes channel_max, frame_max, and heartbeat parameters, on + * the client side only. It does not try to renegotiate these parameters with + * the broker. Using this function will lead to unexpected results. + * + * \param [in] state the connection object + * \param [in] channel_max the maximum number of channels. + * The largest this can be is 65535 + * \param [in] frame_max the maximum size of an frame. + * The smallest this can be is 4096 + * The largest this can be is 2147483647 + * Unless you know what you're doing the recommended + * size is 131072 or 128KB + * \param [in] heartbeat the number of seconds between heartbeats + * + * \return AMQP_STATUS_OK on success, an amqp_status_enum value otherwise. + * Possible error codes include: + * - AMQP_STATUS_NO_MEMORY memory allocation failed. + * - AMQP_STATUS_TIMER_FAILURE the underlying system timer indicated it + * failed. + * + * \sa amqp_login(), amqp_login_with_properties() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_tune_connection(amqp_connection_state_t state, + int channel_max, int frame_max, + int heartbeat); + +/** + * Get the maximum number of channels the connection can handle + * + * The maximum number of channels is set when connection negotiation takes + * place in amqp_login() or amqp_login_with_properties(). + * + * \param [in] state the connection object + * \return the maximum number of channels. 0 if there is no limit + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_get_channel_max(amqp_connection_state_t state); + +/** + * Get the maximum size of an frame the connection can handle + * + * The maximum size of an frame is set when connection negotiation takes + * place in amqp_login() or amqp_login_with_properties(). + * + * \param [in] state the connection object + * \return the maximum size of an frame. + * + * \since v0.6 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_get_frame_max(amqp_connection_state_t state); + +/** + * Get the number of seconds between heartbeats of the connection + * + * The number of seconds between heartbeats is set when connection + * negotiation takes place in amqp_login() or amqp_login_with_properties(). + * + * \param [in] state the connection object + * \return the number of seconds between heartbeats. + * + * \since v0.6 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_get_heartbeat(amqp_connection_state_t state); + +/** + * Destroys an amqp_connection_state_t object + * + * Destroys a amqp_connection_state_t object that was created with + * amqp_new_connection(). If the connection with the broker is open, it will be + * implicitly closed with a reply code of 200 (success). Any memory that + * would be freed with amqp_maybe_release_buffers() or + * amqp_maybe_release_buffers_on_channel() will be freed, and use of that + * memory will caused undefined behavior. + * + * \param [in] state the connection object + * \return AMQP_STATUS_OK on success. amqp_status_enum value failure + * + * \sa amqp_new_connection() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_destroy_connection(amqp_connection_state_t state); + +/** + * Process incoming data + * + * \warning This is a low-level function intended for those who want to + * have greater control over input and output over the socket from the + * broker. Correctly using this function requires in-depth knowledge of AMQP + * and rabbitmq-c. + * + * For a given buffer of data received from the broker, decode the first + * frame in the buffer. If more than one frame is contained in the input buffer + * the return value will be less than the received_data size, the caller should + * adjust received_data buffer descriptor to point to the beginning of the + * buffer + the return value. + * + * \param [in] state the connection object + * \param [in] received_data a buffer of data received from the broker. The + * function will return the number of bytes of the buffer it used. The + * function copies these bytes to an internal buffer: this part of the buffer + * may be reused after this function successfully completes. + * \param [in,out] decoded_frame caller should pass in a pointer to an + * amqp_frame_t struct. If there is enough data in received_data for a + * complete frame, decoded_frame->frame_type will be set to something OTHER + * than 0. decoded_frame may contain members pointing to memory owned by + * the state object. This memory can be recycled with + * amqp_maybe_release_buffers() or amqp_maybe_release_buffers_on_channel(). + * \return number of bytes consumed from received_data or 0 if a 0-length + * buffer was passed. A negative return value indicates failure. Possible + * errors: + * - AMQP_STATUS_NO_MEMORY failure in allocating memory. The library is likely + * in an indeterminate state making recovery unlikely. Client should note the + * error and terminate the application + * - AMQP_STATUS_BAD_AMQP_DATA bad AMQP data was received. The connection + * should be shutdown immediately + * - AMQP_STATUS_UNKNOWN_METHOD: an unknown method was received from the + * broker. This is likely a protocol error and the connection should be + * shutdown immediately + * - AMQP_STATUS_UNKNOWN_CLASS: a properties frame with an unknown class + * was received from the broker. This is likely a protocol error and the + * connection should be shutdown immediately + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_handle_input(amqp_connection_state_t state, + amqp_bytes_t received_data, + amqp_frame_t *decoded_frame); + +/** + * Check to see if connection memory can be released + * + * \deprecated This function is deprecated in favor of + * amqp_maybe_release_buffers() or amqp_maybe_release_buffers_on_channel() + * + * Checks the state of an amqp_connection_state_t object to see if + * amqp_release_buffers() can be called successfully. + * + * \param [in] state the connection object + * \returns TRUE if the buffers can be released FALSE otherwise + * + * \sa amqp_release_buffers() amqp_maybe_release_buffers() + * amqp_maybe_release_buffers_on_channel() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_boolean_t AMQP_CALL amqp_release_buffers_ok(amqp_connection_state_t state); + +/** + * Release amqp_connection_state_t owned memory + * + * \deprecated This function is deprecated in favor of + * amqp_maybe_release_buffers() or amqp_maybe_release_buffers_on_channel() + * + * \warning caller should ensure amqp_release_buffers_ok() returns true before + * calling this function. Failure to do so may result in abort() being called. + * + * Release memory owned by the amqp_connection_state_t for reuse by the + * library. Use of any memory returned by the library before this function is + * called will result in undefined behavior. + * + * \note internally rabbitmq-c tries to reuse memory when possible. As a result + * its possible calling this function may not have a noticeable effect on + * memory usage. + * + * \param [in] state the connection object + * + * \sa amqp_release_buffers_ok() amqp_maybe_release_buffers() + * amqp_maybe_release_buffers_on_channel() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_release_buffers(amqp_connection_state_t state); + +/** + * Release amqp_connection_state_t owned memory + * + * Release memory owned by the amqp_connection_state_t object related to any + * channel, allowing reuse by the library. Use of any memory returned by the + * library before this function is called with result in undefined behavior. + * + * \note internally rabbitmq-c tries to reuse memory when possible. As a result + * its possible calling this function may not have a noticeable effect on + * memory usage. + * + * \param [in] state the connection object + * + * \sa amqp_maybe_release_buffers_on_channel() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_maybe_release_buffers(amqp_connection_state_t state); + +/** + * Release amqp_connection_state_t owned memory related to a channel + * + * Release memory owned by the amqp_connection_state_t object related to the + * specified channel, allowing reuse by the library. Use of any memory returned + * the library for a specific channel will result in undefined behavior. + * + * \note internally rabbitmq-c tries to reuse memory when possible. As a result + * its possible calling this function may not have a noticeable effect on + * memory usage. + * + * \param [in] state the connection object + * \param [in] channel the channel specifier for which memory should be + * released. Note that the library does not care about the state of the + * channel when calling this function + * + * \sa amqp_maybe_release_buffers() + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_maybe_release_buffers_on_channel( + amqp_connection_state_t state, amqp_channel_t channel); + +/** + * Send a frame to the broker + * + * \param [in] state the connection object + * \param [in] frame the frame to send to the broker + * \return AMQP_STATUS_OK on success, an amqp_status_enum value on error. + * Possible error codes: + * - AMQP_STATUS_BAD_AMQP_DATA the serialized form of the method or + * properties was too large to fit in a single AMQP frame, or the + * method contains an invalid value. The frame was not sent. + * - AMQP_STATUS_TABLE_TOO_BIG the serialized form of an amqp_table_t is + * too large to fit in a single AMQP frame. Frame was not sent. + * - AMQP_STATUS_UNKNOWN_METHOD an invalid method type was passed in + * - AMQP_STATUS_UNKNOWN_CLASS an invalid properties type was passed in + * - AMQP_STATUS_TIMER_FAILURE system timer indicated failure. The frame + * was sent + * - AMQP_STATUS_SOCKET_ERROR + * - AMQP_STATUS_SSL_ERROR + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_send_frame(amqp_connection_state_t state, + amqp_frame_t const *frame); + +/** + * Compare two table entries + * + * Works just like strcmp(), comparing two the table keys, datatype, then values + * + * \param [in] entry1 the entry on the left + * \param [in] entry2 the entry on the right + * \return 0 if entries are equal, 0 < if left is greater, 0 > if right is + * greater + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_table_entry_cmp(void const *entry1, void const *entry2); + +/** + * Open a socket to a remote host + * + * \deprecated This function is deprecated in favor of amqp_socket_open() + * + * Looks up the hostname, then attempts to open a socket to the host using + * the specified portnumber. It also sets various options on the socket to + * improve performance and correctness. + * + * \param [in] hostname this can be a hostname or IP address. + * Both IPv4 and IPv6 are acceptable + * \param [in] portnumber the port to connect on. RabbitMQ brokers + * listen on port 5672, and 5671 for SSL + * \return a positive value indicates success and is the sockfd. A negative + * value (see amqp_status_enum)is returned on failure. Possible error codes: + * - AMQP_STATUS_TCP_SOCKETLIB_INIT_ERROR Initialization of underlying socket + * library failed. + * - AMQP_STATUS_HOSTNAME_RESOLUTION_FAILED hostname lookup failed. + * - AMQP_STATUS_SOCKET_ERROR a socket error occurred. errno or + * WSAGetLastError() may return more useful information. + * + * \note IPv6 support was added in v0.3 + * + * \sa amqp_socket_open() amqp_set_sockfd() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_open_socket(char const *hostname, int portnumber); + +/** + * Send initial AMQP header to the broker + * + * \warning this is a low level function intended for those who want to + * interact with the broker at a very low level. Use of this function without + * understanding what it does will result in AMQP protocol errors. + * + * This function sends the AMQP protocol header to the broker. + * + * \param [in] state the connection object + * \return AMQP_STATUS_OK on success, a negative value on failure. Possible + * error codes: + * - AMQP_STATUS_CONNECTION_CLOSED the connection to the broker was closed. + * - AMQP_STATUS_SOCKET_ERROR a socket error occurred. It is likely the + * underlying socket has been closed. errno or WSAGetLastError() may provide + * further information. + * - AMQP_STATUS_SSL_ERROR a SSL error occurred. The connection to the broker + * was closed. + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_send_header(amqp_connection_state_t state); + +/** + * Checks to see if there are any incoming frames ready to be read + * + * Checks to see if there are any amqp_frame_t objects buffered by the + * amqp_connection_state_t object. Having one or more frames buffered means + * that amqp_simple_wait_frame() or amqp_simple_wait_frame_noblock() will + * return a frame without potentially blocking on a read() call. + * + * \param [in] state the connection object + * \return TRUE if there are frames enqueued, FALSE otherwise + * + * \sa amqp_simple_wait_frame() amqp_simple_wait_frame_noblock() + * amqp_data_in_buffer() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_boolean_t AMQP_CALL amqp_frames_enqueued(amqp_connection_state_t state); + +/** + * Read a single amqp_frame_t + * + * Waits for the next amqp_frame_t frame to be read from the broker. + * This function has the potential to block for a long time in the case of + * waiting for a basic.deliver method frame from the broker. + * + * The library may buffer frames. When an amqp_connection_state_t object + * has frames buffered calling amqp_simple_wait_frame() will return an + * amqp_frame_t without entering a blocking read(). You can test to see if + * an amqp_connection_state_t object has frames buffered by calling the + * amqp_frames_enqueued() function. + * + * The library has a socket read buffer. When there is data in an + * amqp_connection_state_t read buffer, amqp_simple_wait_frame() may return an + * amqp_frame_t without entering a blocking read(). You can test to see if an + * amqp_connection_state_t object has data in its read buffer by calling the + * amqp_data_in_buffer() function. + * + * \param [in] state the connection object + * \param [out] decoded_frame the frame + * \return AMQP_STATUS_OK on success, an amqp_status_enum value + * is returned otherwise. Possible errors include: + * - AMQP_STATUS_NO_MEMORY failure in allocating memory. The library is likely + * in an indeterminate state making recovery unlikely. Client should note the + * error and terminate the application + * - AMQP_STATUS_BAD_AMQP_DATA bad AMQP data was received. The connection + * should be shutdown immediately + * - AMQP_STATUS_UNKNOWN_METHOD: an unknown method was received from the + * broker. This is likely a protocol error and the connection should be + * shutdown immediately + * - AMQP_STATUS_UNKNOWN_CLASS: a properties frame with an unknown class + * was received from the broker. This is likely a protocol error and the + * connection should be shutdown immediately + * - AMQP_STATUS_HEARTBEAT_TIMEOUT timed out while waiting for heartbeat + * from the broker. The connection has been closed. + * - AMQP_STATUS_TIMER_FAILURE system timer indicated failure. + * - AMQP_STATUS_SOCKET_ERROR a socket error occurred. The connection has + * been closed + * - AMQP_STATUS_SSL_ERROR a SSL socket error occurred. The connection has + * been closed. + * + * \sa amqp_simple_wait_frame_noblock() amqp_frames_enqueued() + * amqp_data_in_buffer() + * + * \note as of v0.4.0 this function will no longer return heartbeat frames + * when enabled by specifying a non-zero heartbeat value in amqp_login(). + * Heartbeating is handled internally by the library. + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_simple_wait_frame(amqp_connection_state_t state, + amqp_frame_t *decoded_frame); + +/** + * Read a single amqp_frame_t with a timeout. + * + * Waits for the next amqp_frame_t frame to be read from the broker, up to + * a timespan specified by tv. The function will return AMQP_STATUS_TIMEOUT + * if the timeout is reached. The tv value is not modified by the function. + * + * If a 0 timeval is specified, the function behaves as if its non-blocking: it + * will test to see if a frame can be read from the broker, and return + * immediately. + * + * If NULL is passed in for tv, the function will behave like + * amqp_simple_wait_frame() and block until a frame is received from the broker + * + * The library may buffer frames. When an amqp_connection_state_t object + * has frames buffered calling amqp_simple_wait_frame_noblock() will return an + * amqp_frame_t without entering a blocking read(). You can test to see if an + * amqp_connection_state_t object has frames buffered by calling the + * amqp_frames_enqueued() function. + * + * The library has a socket read buffer. When there is data in an + * amqp_connection_state_t read buffer, amqp_simple_wait_frame_noblock() may + * return + * an amqp_frame_t without entering a blocking read(). You can test to see if an + * amqp_connection_state_t object has data in its read buffer by calling the + * amqp_data_in_buffer() function. + * + * \note This function does not return heartbeat frames. When enabled, + * heartbeating is handed internally internally by the library. + * + * \param [in,out] state the connection object + * \param [out] decoded_frame the frame + * \param [in] tv the maximum time to wait for a frame to be read. Setting + * tv->tv_sec = 0 and tv->tv_usec = 0 will do a non-blocking read. Specifying + * NULL for tv will make the function block until a frame is read. + * \return AMQP_STATUS_OK on success. An amqp_status_enum value is returned + * otherwise. Possible errors include: + * - AMQP_STATUS_TIMEOUT the timeout was reached while waiting for a frame + * from the broker. + * - AMQP_STATUS_INVALID_PARAMETER the tv parameter contains an invalid value. + * - AMQP_STATUS_NO_MEMORY failure in allocating memory. The library is likely + * in an indeterminate state making recovery unlikely. Client should note the + * error and terminate the application + * - AMQP_STATUS_BAD_AMQP_DATA bad AMQP data was received. The connection + * should be shutdown immediately + * - AMQP_STATUS_UNKNOWN_METHOD: an unknown method was received from the + * broker. This is likely a protocol error and the connection should be + * shutdown immediately + * - AMQP_STATUS_UNKNOWN_CLASS: a properties frame with an unknown class + * was received from the broker. This is likely a protocol error and the + * connection should be shutdown immediately + * - AMQP_STATUS_HEARTBEAT_TIMEOUT timed out while waiting for heartbeat + * from the broker. The connection has been closed. + * - AMQP_STATUS_TIMER_FAILURE system timer indicated failure. + * - AMQP_STATUS_SOCKET_ERROR a socket error occurred. The connection has + * been closed + * - AMQP_STATUS_SSL_ERROR a SSL socket error occurred. The connection has + * been closed. + * + * \sa amqp_simple_wait_frame() amqp_frames_enqueued() amqp_data_in_buffer() + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_simple_wait_frame_noblock(amqp_connection_state_t state, + amqp_frame_t *decoded_frame, + struct timeval *tv); + +/** + * Waits for a specific method from the broker + * + * \warning You probably don't want to use this function. If this function + * doesn't receive exactly the frame requested it closes the whole connection. + * + * Waits for a single method on a channel from the broker. + * If a frame is received that does not match expected_channel + * or expected_method the program will abort + * + * \param [in] state the connection object + * \param [in] expected_channel the channel that the method should be delivered + * on + * \param [in] expected_method the method to wait for + * \param [out] output the method + * \returns AMQP_STATUS_OK on success. An amqp_status_enum value is returned + * otherwise. Possible errors include: + * - AMQP_STATUS_WRONG_METHOD a frame containing the wrong method, wrong frame + * type or wrong channel was received. The connection is closed. + * - AMQP_STATUS_NO_MEMORY failure in allocating memory. The library is likely + * in an indeterminate state making recovery unlikely. Client should note the + * error and terminate the application + * - AMQP_STATUS_BAD_AMQP_DATA bad AMQP data was received. The connection + * should be shutdown immediately + * - AMQP_STATUS_UNKNOWN_METHOD: an unknown method was received from the + * broker. This is likely a protocol error and the connection should be + * shutdown immediately + * - AMQP_STATUS_UNKNOWN_CLASS: a properties frame with an unknown class + * was received from the broker. This is likely a protocol error and the + * connection should be shutdown immediately + * - AMQP_STATUS_HEARTBEAT_TIMEOUT timed out while waiting for heartbeat + * from the broker. The connection has been closed. + * - AMQP_STATUS_TIMER_FAILURE system timer indicated failure. + * - AMQP_STATUS_SOCKET_ERROR a socket error occurred. The connection has + * been closed + * - AMQP_STATUS_SSL_ERROR a SSL socket error occurred. The connection has + * been closed. + * + * \since v0.1 + */ + +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_simple_wait_method(amqp_connection_state_t state, + amqp_channel_t expected_channel, + amqp_method_number_t expected_method, + amqp_method_t *output); + +/** + * Sends a method to the broker + * + * This is a thin wrapper around amqp_send_frame(), providing a way to send + * a method to the broker on a specified channel. + * + * \param [in] state the connection object + * \param [in] channel the channel object + * \param [in] id the method number + * \param [in] decoded the method object + * \returns AMQP_STATUS_OK on success, an amqp_status_enum value otherwise. + * Possible errors include: + * - AMQP_STATUS_BAD_AMQP_DATA the serialized form of the method or + * properties was too large to fit in a single AMQP frame, or the + * method contains an invalid value. The frame was not sent. + * - AMQP_STATUS_TABLE_TOO_BIG the serialized form of an amqp_table_t is + * too large to fit in a single AMQP frame. Frame was not sent. + * - AMQP_STATUS_UNKNOWN_METHOD an invalid method type was passed in + * - AMQP_STATUS_UNKNOWN_CLASS an invalid properties type was passed in + * - AMQP_STATUS_TIMER_FAILURE system timer indicated failure. The frame + * was sent + * - AMQP_STATUS_SOCKET_ERROR + * - AMQP_STATUS_SSL_ERROR + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_send_method(amqp_connection_state_t state, + amqp_channel_t channel, amqp_method_number_t id, + void *decoded); + +/** + * Sends a method to the broker and waits for a method response + * + * \param [in] state the connection object + * \param [in] channel the channel object + * \param [in] request_id the method number of the request + * \param [in] expected_reply_ids a 0 terminated array of expected response + * method numbers + * \param [in] decoded_request_method the method to be sent to the broker + * \return a amqp_rpc_reply_t: + * - r.reply_type == AMQP_RESPONSE_NORMAL. RPC completed successfully + * - r.reply_type == AMQP_RESPONSE_SERVER_EXCEPTION. The broker returned an + * exception: + * - If r.reply.id == AMQP_CHANNEL_CLOSE_METHOD a channel exception + * occurred, cast r.reply.decoded to amqp_channel_close_t* to see details + * of the exception. The client should amqp_send_method() a + * amqp_channel_close_ok_t. The channel must be re-opened before it + * can be used again. Any resources associated with the channel + * (auto-delete exchanges, auto-delete queues, consumers) are invalid + * and must be recreated before attempting to use them again. + * - If r.reply.id == AMQP_CONNECTION_CLOSE_METHOD a connection exception + * occurred, cast r.reply.decoded to amqp_connection_close_t* to see + * details of the exception. The client amqp_send_method() a + * amqp_connection_close_ok_t and disconnect from the broker. + * - r.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION. An exception occurred + * within the library. Examine r.library_error and compare it against + * amqp_status_enum values to determine the error. + * + * \sa amqp_simple_rpc_decoded() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_simple_rpc( + amqp_connection_state_t state, amqp_channel_t channel, + amqp_method_number_t request_id, amqp_method_number_t *expected_reply_ids, + void *decoded_request_method); + +/** + * Sends a method to the broker and waits for a method response + * + * \param [in] state the connection object + * \param [in] channel the channel object + * \param [in] request_id the method number of the request + * \param [in] reply_id the method number expected in response + * \param [in] decoded_request_method the request method + * \return a pointer to the method returned from the broker, or NULL on error. + * On error amqp_get_rpc_reply() will return an amqp_rpc_reply_t with + * details on the error that occurred. + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +void *AMQP_CALL amqp_simple_rpc_decoded(amqp_connection_state_t state, + amqp_channel_t channel, + amqp_method_number_t request_id, + amqp_method_number_t reply_id, + void *decoded_request_method); + +/** + * Get the last global amqp_rpc_reply + * + * The API methods corresponding to most synchronous AMQP methods + * return a pointer to the decoded method result. Upon error, they + * return NULL, and we need some way of discovering what, if anything, + * went wrong. amqp_get_rpc_reply() returns the most recent + * amqp_rpc_reply_t instance corresponding to such an API operation + * for the given connection. + * + * Only use it for operations that do not themselves return + * amqp_rpc_reply_t; operations that do return amqp_rpc_reply_t + * generally do NOT update this per-connection-global amqp_rpc_reply_t + * instance. + * + * \param [in] state the connection object + * \return the most recent amqp_rpc_reply_t: + * - r.reply_type == AMQP_RESPONSE_NORMAL. RPC completed successfully + * - r.reply_type == AMQP_RESPONSE_SERVER_EXCEPTION. The broker returned an + * exception: + * - If r.reply.id == AMQP_CHANNEL_CLOSE_METHOD a channel exception + * occurred, cast r.reply.decoded to amqp_channel_close_t* to see details + * of the exception. The client should amqp_send_method() a + * amqp_channel_close_ok_t. The channel must be re-opened before it + * can be used again. Any resources associated with the channel + * (auto-delete exchanges, auto-delete queues, consumers) are invalid + * and must be recreated before attempting to use them again. + * - If r.reply.id == AMQP_CONNECTION_CLOSE_METHOD a connection exception + * occurred, cast r.reply.decoded to amqp_connection_close_t* to see + * details of the exception. The client amqp_send_method() a + * amqp_connection_close_ok_t and disconnect from the broker. + * - r.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION. An exception occurred + * within the library. Examine r.library_error and compare it against + * amqp_status_enum values to determine the error. + * + * \sa amqp_simple_rpc_decoded() + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_get_rpc_reply(amqp_connection_state_t state); + +/** + * Login to the broker + * + * After using amqp_open_socket and amqp_set_sockfd, call + * amqp_login to complete connecting to the broker + * + * \param [in] state the connection object + * \param [in] vhost the virtual host to connect to on the broker. The default + * on most brokers is "/" + * \param [in] channel_max the limit for number of channels for the connection. + * 0 means no limit, and is a good default + * (AMQP_DEFAULT_MAX_CHANNELS) + * Note that the maximum number of channels the protocol supports + * is 65535 (2^16, with the 0-channel reserved). The server can + * set a lower channel_max and then the client will use the lowest + * of the two + * \param [in] frame_max the maximum size of an AMQP frame on the wire to + * request of the broker for this connection. 4096 is the minimum + * size, 2^31-1 is the maximum, a good default is 131072 (128KB), + * or AMQP_DEFAULT_FRAME_SIZE + * \param [in] heartbeat the number of seconds between heartbeat frames to + * request of the broker. A value of 0 disables heartbeats. + * Note rabbitmq-c only has partial support for heartbeats, as of + * v0.4.0 they are only serviced during amqp_basic_publish() and + * amqp_simple_wait_frame()/amqp_simple_wait_frame_noblock() + * \param [in] sasl_method the SASL method to authenticate with the broker. + * followed by the authentication information. The following SASL + * methods are implemented: + * - AMQP_SASL_METHOD_PLAIN, the AMQP_SASL_METHOD_PLAIN argument + * should be followed by two arguments in this order: + * const char* username, and const char* password. + * - AMQP_SASL_METHOD_EXTERNAL, the AMQP_SASL_METHOD_EXTERNAL + * argument should be followed one argument: + * const char* identity. + * \return amqp_rpc_reply_t indicating success or failure. + * - r.reply_type == AMQP_RESPONSE_NORMAL. Login completed successfully + * - r.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION. In most cases errors + * from the broker when logging in will be represented by the broker closing + * the socket. In this case r.library_error will be set to + * AMQP_STATUS_CONNECTION_CLOSED. This error can represent a number of + * error conditions including: invalid vhost, authentication failure. + * - r.reply_type == AMQP_RESPONSE_SERVER_EXCEPTION. The broker returned an + * exception: + * - If r.reply.id == AMQP_CHANNEL_CLOSE_METHOD a channel exception + * occurred, cast r.reply.decoded to amqp_channel_close_t* to see details + * of the exception. The client should amqp_send_method() a + * amqp_channel_close_ok_t. The channel must be re-opened before it + * can be used again. Any resources associated with the channel + * (auto-delete exchanges, auto-delete queues, consumers) are invalid + * and must be recreated before attempting to use them again. + * - If r.reply.id == AMQP_CONNECTION_CLOSE_METHOD a connection exception + * occurred, cast r.reply.decoded to amqp_connection_close_t* to see + * details of the exception. The client amqp_send_method() a + * amqp_connection_close_ok_t and disconnect from the broker. + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_login(amqp_connection_state_t state, + char const *vhost, int channel_max, + int frame_max, int heartbeat, + amqp_sasl_method_enum sasl_method, ...); + +/** + * Login to the broker passing a properties table + * + * This function is similar to amqp_login() and differs in that it provides a + * way to pass client properties to the broker. This is commonly used to + * negotiate newer protocol features as they are supported by the broker. + * + * \param [in] state the connection object + * \param [in] vhost the virtual host to connect to on the broker. The default + * on most brokers is "/" + * \param [in] channel_max the limit for the number of channels for the + * connection. + * 0 means no limit, and is a good default + * (AMQP_DEFAULT_MAX_CHANNELS) + * Note that the maximum number of channels the protocol supports + * is 65535 (2^16, with the 0-channel reserved). The server can + * set a lower channel_max and then the client will use the lowest + * of the two + * \param [in] frame_max the maximum size of an AMQP frame ont he wire to + * request of the broker for this connection. 4096 is the minimum + * size, 2^31-1 is the maximum, a good default is 131072 (128KB), + * or AMQP_DEFAULT_FRAME_SIZE + * \param [in] heartbeat the number of seconds between heartbeat frame to + * request of the broker. A value of 0 disables heartbeats. + * Note rabbitmq-c only has partial support for hearts, as of + * v0.4.0 heartbeats are only serviced during amqp_basic_publish(), + * and amqp_simple_wait_frame()/amqp_simple_wait_frame_noblock() + * \param [in] properties a table of properties to send the broker. + * \param [in] sasl_method the SASL method to authenticate with the broker + * followed by the authentication information. The following SASL + * methods are implemented: + * - AMQP_SASL_METHOD_PLAIN, the AMQP_SASL_METHOD_PLAIN argument + * should be followed by two arguments in this order: + * const char* username, and const char* password. + * - AMQP_SASL_METHOD_EXTERNAL, the AMQP_SASL_METHOD_EXTERNAL + * argument should be followed one argument: + * const char* identity. + * \return amqp_rpc_reply_t indicating success or failure. + * - r.reply_type == AMQP_RESPONSE_NORMAL. Login completed successfully + * - r.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION. In most cases errors + * from the broker when logging in will be represented by the broker closing + * the socket. In this case r.library_error will be set to + * AMQP_STATUS_CONNECTION_CLOSED. This error can represent a number of + * error conditions including: invalid vhost, authentication failure. + * - r.reply_type == AMQP_RESPONSE_SERVER_EXCEPTION. The broker returned an + * exception: + * - If r.reply.id == AMQP_CHANNEL_CLOSE_METHOD a channel exception + * occurred, cast r.reply.decoded to amqp_channel_close_t* to see details + * of the exception. The client should amqp_send_method() a + * amqp_channel_close_ok_t. The channel must be re-opened before it + * can be used again. Any resources associated with the channel + * (auto-delete exchanges, auto-delete queues, consumers) are invalid + * and must be recreated before attempting to use them again. + * - If r.reply.id == AMQP_CONNECTION_CLOSE_METHOD a connection exception + * occurred, cast r.reply.decoded to amqp_connection_close_t* to see + * details of the exception. The client amqp_send_method() a + * amqp_connection_close_ok_t and disconnect from the broker. + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_login_with_properties( + amqp_connection_state_t state, char const *vhost, int channel_max, + int frame_max, int heartbeat, const amqp_table_t *properties, + amqp_sasl_method_enum sasl_method, ...); + +struct amqp_basic_properties_t_; + +/** + * Publish a message to the broker + * + * Publish a message on an exchange with a routing key. + * + * Note that at the AMQ protocol level basic.publish is an async method: + * this means error conditions that occur on the broker (such as publishing to + * a non-existent exchange) will not be reflected in the return value of this + * function. + * + * \param [in] state the connection object + * \param [in] channel the channel identifier + * \param [in] exchange the exchange on the broker to publish to + * \param [in] routing_key the routing key to use when publishing the message + * \param [in] mandatory indicate to the broker that the message MUST be routed + * to a queue. If the broker cannot do this it should respond with + * a basic.return method. + * \param [in] immediate indicate to the broker that the message MUST be + * delivered to a consumer immediately. If the broker cannot do this + * it should respond with a basic.return method. + * \param [in] properties the properties associated with the message + * \param [in] body the message body + * \return AMQP_STATUS_OK on success, amqp_status_enum value on failure. Note + * that basic.publish is an async method, the return value from this + * function only indicates that the message data was successfully + * transmitted to the broker. It does not indicate failures that occur + * on the broker, such as publishing to a non-existent exchange. + * Possible error values: + * - AMQP_STATUS_TIMER_FAILURE: system timer facility returned an error + * the message was not sent. + * - AMQP_STATUS_HEARTBEAT_TIMEOUT: connection timed out waiting for a + * heartbeat from the broker. The message was not sent. + * - AMQP_STATUS_NO_MEMORY: memory allocation failed. The message was + * not sent. + * - AMQP_STATUS_TABLE_TOO_BIG: a table in the properties was too large + * to fit in a single frame. Message was not sent. + * - AMQP_STATUS_CONNECTION_CLOSED: the connection was closed. + * - AMQP_STATUS_SSL_ERROR: a SSL error occurred. + * - AMQP_STATUS_TCP_ERROR: a TCP error occurred. errno or + * WSAGetLastError() may provide more information + * + * Note: this function does heartbeat processing as of v0.4.0 + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_basic_publish( + amqp_connection_state_t state, amqp_channel_t channel, + amqp_bytes_t exchange, amqp_bytes_t routing_key, amqp_boolean_t mandatory, + amqp_boolean_t immediate, struct amqp_basic_properties_t_ const *properties, + amqp_bytes_t body); + +/** + * Closes an channel + * + * \param [in] state the connection object + * \param [in] channel the channel identifier + * \param [in] code the reason for closing the channel, AMQP_REPLY_SUCCESS is a + * good default + * \return amqp_rpc_reply_t indicating success or failure + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_channel_close(amqp_connection_state_t state, + amqp_channel_t channel, int code); + +/** + * Closes the entire connection + * + * Implicitly closes all channels and informs the broker the connection + * is being closed, after receiving acknowledgment from the broker it closes + * the socket. + * + * \param [in] state the connection object + * \param [in] code the reason code for closing the connection. + * AMQP_REPLY_SUCCESS is a good default. + * \return amqp_rpc_reply_t indicating the result + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_connection_close(amqp_connection_state_t state, + int code); + +/** + * Acknowledges a message + * + * Does a basic.ack on a received message + * + * \param [in] state the connection object + * \param [in] channel the channel identifier + * \param [in] delivery_tag the delivery tag of the message to be ack'd + * \param [in] multiple if true, ack all messages up to this delivery tag, if + * false ack only this delivery tag + * \return 0 on success, 0 > on failing to send the ack to the broker. + * this will not indicate failure if something goes wrong on the + * broker + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_basic_ack(amqp_connection_state_t state, + amqp_channel_t channel, uint64_t delivery_tag, + amqp_boolean_t multiple); + +/** + * Do a basic.get + * + * Synchonously polls the broker for a message in a queue, and + * retrieves the message if a message is in the queue. + * + * \param [in] state the connection object + * \param [in] channel the channel identifier to use + * \param [in] queue the queue name to retrieve from + * \param [in] no_ack if true the message is automatically ack'ed + * if false amqp_basic_ack should be called once the message + * retrieved has been processed + * \return amqp_rpc_reply indicating success or failure + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_basic_get(amqp_connection_state_t state, + amqp_channel_t channel, + amqp_bytes_t queue, + amqp_boolean_t no_ack); + +/** + * Do a basic.reject + * + * Actively reject a message that has been delivered + * + * \param [in] state the connection object + * \param [in] channel the channel identifier + * \param [in] delivery_tag the delivery tag of the message to reject + * \param [in] requeue indicate to the broker whether it should requeue the + * message or just discard it. + * \return 0 on success, 0 > on failing to send the reject method to the broker. + * This will not indicate failure if something goes wrong on the + * broker. + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_basic_reject(amqp_connection_state_t state, + amqp_channel_t channel, uint64_t delivery_tag, + amqp_boolean_t requeue); + +/** + * Do a basic.nack + * + * Actively reject a message, this has the same effect as amqp_basic_reject() + * however, amqp_basic_nack() can negatively acknowledge multiple messages with + * one call much like amqp_basic_ack() can acknowledge mutliple messages with + * one call. + * + * \param [in] state the connection object + * \param [in] channel the channel identifier + * \param [in] delivery_tag the delivery tag of the message to reject + * \param [in] multiple if set to 1 negatively acknowledge all unacknowledged + * messages on this channel. + * \param [in] requeue indicate to the broker whether it should requeue the + * message or dead-letter it. + * \return AMQP_STATUS_OK on success, an amqp_status_enum value otherwise. + * + * \since v0.5.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_basic_nack(amqp_connection_state_t state, + amqp_channel_t channel, uint64_t delivery_tag, + amqp_boolean_t multiple, amqp_boolean_t requeue); +/** + * Check to see if there is data left in the receive buffer + * + * Can be used to see if there is data still in the buffer, if so + * calling amqp_simple_wait_frame will not immediately enter a + * blocking read. + * + * \param [in] state the connection object + * \return true if there is data in the recieve buffer, false otherwise + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +amqp_boolean_t AMQP_CALL amqp_data_in_buffer(amqp_connection_state_t state); + +/** + * Get the error string for the given error code. + * + * \deprecated This function has been deprecated in favor of + * \ref amqp_error_string2() which returns statically allocated + * string which do not need to be freed by the caller. + * + * The returned string resides on the heap; the caller is responsible + * for freeing it. + * + * \param [in] err return error code + * \return the error string + * + * \since v0.1 + */ +AMQP_DEPRECATED( + AMQP_PUBLIC_FUNCTION char *AMQP_CALL amqp_error_string(int err)); + +/** + * Get the error string for the given error code. + * + * Get an error string associated with an error code. The string is statically + * allocated and does not need to be freed + * + * \param [in] err the error code + * \return the error string + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +const char *AMQP_CALL amqp_error_string2(int err); + +/** + * Deserialize an amqp_table_t from AMQP wireformat + * + * This is an internal function and is not typically used by + * client applications + * + * \param [in] encoded the buffer containing the serialized data + * \param [in] pool memory pool used to allocate the table entries from + * \param [in] output the amqp_table_t structure to fill in. Any existing + * entries will be erased + * \param [in,out] offset The offset into the encoded buffer to start + * reading the serialized table. It will be updated + * by this function to end of the table + * \return AMQP_STATUS_OK on success, an amqp_status_enum value on failure + * Possible error codes: + * - AMQP_STATUS_NO_MEMORY out of memory + * - AMQP_STATUS_BAD_AMQP_DATA invalid wireformat + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_decode_table(amqp_bytes_t encoded, amqp_pool_t *pool, + amqp_table_t *output, size_t *offset); + +/** + * Serializes an amqp_table_t to the AMQP wireformat + * + * This is an internal function and is not typically used by + * client applications + * + * \param [in] encoded the buffer where to serialize the table to + * \param [in] input the amqp_table_t to serialize + * \param [in,out] offset The offset into the encoded buffer to start + * writing the serialized table. It will be updated + * by this function to where writing left off + * \return AMQP_STATUS_OK on success, an amqp_status_enum value on failure + * Possible error codes: + * - AMQP_STATUS_TABLE_TOO_BIG the serialized form is too large for the + * buffer + * - AMQP_STATUS_BAD_AMQP_DATA invalid table + * + * \since v0.1 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_encode_table(amqp_bytes_t encoded, amqp_table_t *input, + size_t *offset); + +/** + * Create a deep-copy of an amqp_table_t object + * + * Creates a deep-copy of an amqp_table_t object, using the provided pool + * object to allocate the necessary memory. This memory can be freed later by + * call recycle_amqp_pool(), or empty_amqp_pool() + * + * \param [in] original the table to copy + * \param [in,out] clone the table to copy to + * \param [in] pool the initialized memory pool to do allocations for the table + * from + * \return AMQP_STATUS_OK on success, amqp_status_enum value on failure. + * Possible error values: + * - AMQP_STATUS_NO_MEMORY - memory allocation failure. + * - AMQP_STATUS_INVALID_PARAMETER - invalid table (e.g., no key name) + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_table_clone(const amqp_table_t *original, + amqp_table_t *clone, amqp_pool_t *pool); + +/** + * A message object + * + * \since v0.4.0 + */ +typedef struct amqp_message_t_ { + amqp_basic_properties_t properties; /**< message properties */ + amqp_bytes_t body; /**< message body */ + amqp_pool_t pool; /**< pool used to allocate properties */ +} amqp_message_t; + +/** + * Reads the next message on a channel + * + * Reads a complete message (header + body) on a specified channel. This + * function is intended to be used with amqp_basic_get() or when an + * AMQP_BASIC_DELIVERY_METHOD method is received. + * + * \param [in,out] state the connection object + * \param [in] channel the channel on which to read the message from + * \param [in,out] message a pointer to a amqp_message_t object. Caller should + * call amqp_message_destroy() when it is done using the + * fields in the message object. The caller is responsible for + * allocating/destroying the amqp_message_t object itself. + * \param [in] flags pass in 0. Currently unused. + * \returns a amqp_rpc_reply_t object. ret.reply_type == AMQP_RESPONSE_NORMAL on + * success. + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_read_message(amqp_connection_state_t state, + amqp_channel_t channel, + amqp_message_t *message, + int flags); + +/** + * Frees memory associated with a amqp_message_t allocated in amqp_read_message + * + * \param [in] message + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_destroy_message(amqp_message_t *message); + +/** + * Envelope object + * + * \since v0.4.0 + */ +typedef struct amqp_envelope_t_ { + amqp_channel_t channel; /**< channel message was delivered on */ + amqp_bytes_t + consumer_tag; /**< the consumer tag the message was delivered to */ + uint64_t delivery_tag; /**< the messages delivery tag */ + amqp_boolean_t redelivered; /**< flag indicating whether this message is being + redelivered */ + amqp_bytes_t exchange; /**< exchange this message was published to */ + amqp_bytes_t + routing_key; /**< the routing key this message was published with */ + amqp_message_t message; /**< the message */ +} amqp_envelope_t; + +/** + * Wait for and consume a message + * + * Waits for a basic.deliver method on any channel, upon receipt of + * basic.deliver it reads that message, and returns. If any other method is + * received before basic.deliver, this function will return an amqp_rpc_reply_t + * with ret.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION, and + * ret.library_error == AMQP_STATUS_UNEXPECTED_STATE. The caller should then + * call amqp_simple_wait_frame() to read this frame and take appropriate action. + * + * This function should be used after starting a consumer with the + * amqp_basic_consume() function + * + * \param [in,out] state the connection object + * \param [in,out] envelope a pointer to a amqp_envelope_t object. Caller + * should call #amqp_destroy_envelope() when it is done using + * the fields in the envelope object. The caller is responsible + * for allocating/destroying the amqp_envelope_t object itself. + * \param [in] timeout a timeout to wait for a message delivery. Passing in + * NULL will result in blocking behavior. + * \param [in] flags pass in 0. Currently unused. + * \returns a amqp_rpc_reply_t object. ret.reply_type == AMQP_RESPONSE_NORMAL + * on success. If ret.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION, + * and ret.library_error == AMQP_STATUS_UNEXPECTED_STATE, a frame other + * than AMQP_BASIC_DELIVER_METHOD was received, the caller should call + * amqp_simple_wait_frame() to read this frame and take appropriate + * action. + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +amqp_rpc_reply_t AMQP_CALL amqp_consume_message(amqp_connection_state_t state, + amqp_envelope_t *envelope, + struct timeval *timeout, + int flags); + +/** + * Frees memory associated with a amqp_envelope_t allocated in + * amqp_consume_message() + * + * \param [in] envelope + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_destroy_envelope(amqp_envelope_t *envelope); + +/** + * Parameters used to connect to the RabbitMQ broker + * + * \since v0.2 + */ +struct amqp_connection_info { + char *user; /**< the username to authenticate with the broker, default on most + broker is 'guest' */ + char *password; /**< the password to authenticate with the broker, default on + most brokers is 'guest' */ + char *host; /**< the hostname of the broker */ + char *vhost; /**< the virtual host on the broker to connect to, a good default + is "/" */ + int port; /**< the port that the broker is listening on, default on most + brokers is 5672 */ + amqp_boolean_t ssl; +}; + +/** + * Initialze an amqp_connection_info to default values + * + * The default values are: + * - user: "guest" + * - password: "guest" + * - host: "localhost" + * - vhost: "/" + * - port: 5672 + * + * \param [out] parsed the connection info to set defaults on + * + * \since v0.2 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL + amqp_default_connection_info(struct amqp_connection_info *parsed); + +/** + * Parse a connection URL + * + * An amqp connection url takes the form: + * + * amqp://[$USERNAME[:$PASSWORD]\@]$HOST[:$PORT]/[$VHOST] + * + * Examples: + * amqp://guest:guest\@localhost:5672// + * amqp://guest:guest\@localhost/myvhost + * + * Any missing parts of the URL will be set to the defaults specified in + * amqp_default_connection_info. For amqps: URLs the default port will be set + * to 5671 instead of 5672 for non-SSL URLs. + * + * \note This function modifies url parameter. + * + * \param [in] url URI to parse, note that this parameter is modified by the + * function. + * \param [out] parsed the connection info gleaned from the URI. The char* + * members will point to parts of the url input parameter. + * Memory management will depend on how the url is allocated. + * \returns AMQP_STATUS_OK on success, AMQP_STATUS_BAD_URL on failure + * + * \since v0.2 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_parse_url(char *url, struct amqp_connection_info *parsed); + +/* socket API */ + +/** + * Open a socket connection. + * + * This function opens a socket connection returned from amqp_tcp_socket_new() + * or amqp_ssl_socket_new(). This function should be called after setting + * socket options and prior to assigning the socket to an AMQP connection with + * amqp_set_socket(). + * + * \param [in,out] self A socket object. + * \param [in] host Connect to this host. + * \param [in] port Connect on this remote port. + * + * \return AMQP_STATUS_OK on success, an amqp_status_enum on failure + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_socket_open(amqp_socket_t *self, const char *host, int port); + +/** + * Open a socket connection. + * + * This function opens a socket connection returned from amqp_tcp_socket_new() + * or amqp_ssl_socket_new(). This function should be called after setting + * socket options and prior to assigning the socket to an AMQP connection with + * amqp_set_socket(). + * + * \param [in,out] self A socket object. + * \param [in] host Connect to this host. + * \param [in] port Connect on this remote port. + * \param [in] timeout Max allowed time to spent on opening. If NULL - run in + * blocking mode + * + * \return AMQP_STATUS_OK on success, an amqp_status_enum on failure. + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_socket_open_noblock(amqp_socket_t *self, const char *host, + int port, struct timeval *timeout); + +/** + * Get the socket descriptor in use by a socket object. + * + * Retrieve the underlying socket descriptor. This function can be used to + * perform low-level socket operations that aren't supported by the socket + * interface. Use with caution! + * + * \param [in,out] self A socket object. + * + * \return The underlying socket descriptor, or -1 if there is no socket + * descriptor associated with + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_socket_get_sockfd(amqp_socket_t *self); + +/** + * Get the socket object associated with a amqp_connection_state_t + * + * \param [in] state the connection object to get the socket from + * \return a pointer to the socket object, or NULL if one has not been assigned + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +amqp_socket_t *AMQP_CALL amqp_get_socket(amqp_connection_state_t state); + +/** + * Get the broker properties table + * + * \param [in] state the connection object + * \return a pointer to an amqp_table_t containing the properties advertised + * by the broker on connection. The connection object owns the table, it + * should not be modified. + * + * \since v0.5.0 + */ +AMQP_PUBLIC_FUNCTION +amqp_table_t *AMQP_CALL + amqp_get_server_properties(amqp_connection_state_t state); + +/** + * Get the client properties table + * + * Get the properties that were passed to the broker on connection. + * + * \param [in] state the connection object + * \return a pointer to an amqp_table_t containing the properties advertised + * by the client on connection. The connection object owns the table, it + * should not be modified. + * + * \since v0.7.0 + */ +AMQP_PUBLIC_FUNCTION +amqp_table_t *AMQP_CALL + amqp_get_client_properties(amqp_connection_state_t state); + +/** + * Get the login handshake timeout. + * + * amqp_login and amqp_login_with_properties perform the login handshake with + * the broker. This function returns the timeout associated with completing + * this operation from the client side. This value can be set by using the + * amqp_set_handshake_timeout. + * + * Note that the RabbitMQ broker has configurable timeout for completing the + * login handshake, the default is 10 seconds. rabbitmq-c has a default of 12 + * seconds. + * + * \param [in] state the connection object + * \return a struct timeval representing the current login timeout for the state + * object. A NULL value represents an infinite timeout. The memory returned is + * owned by the connection object. + * + * \since v0.9.0 + */ +AMQP_PUBLIC_FUNCTION +struct timeval *AMQP_CALL + amqp_get_handshake_timeout(amqp_connection_state_t state); + +/** + * Set the login handshake timeout. + * + * amqp_login and amqp_login_with_properties perform the login handshake with + * the broker. This function sets the timeout associated with completing this + * operation from the client side. + * + * The timeout must be set before amqp_login or amqp_login_with_properties is + * called to change from the default timeout. + * + * Note that the RabbitMQ broker has a configurable timeout for completing the + * login handshake, the default is 10 seconds. rabbitmq-c has a default of 12 + * seconds. + * + * \param [in] state the connection object + * \param [in] timeout a struct timeval* representing new login timeout for the + * state object. NULL represents an infinite timeout. The value of timeout is + * copied internally, the caller is responsible for ownership of the passed in + * pointer, it does not need to remain valid after this function is called. + * \return AMQP_STATUS_OK on success. + * + * \since v0.9.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_set_handshake_timeout(amqp_connection_state_t state, + struct timeval *timeout); + +/** + * Get the RPC timeout + * + * Gets the timeout for any RPC-style AMQP command (e.g., amqp_queue_declare). + * This timeout may be changed at any time by calling \amqp_set_rpc_timeout + * function with a new timeout. The timeout applies individually to each RPC + * that is made. + * + * The default value is NULL, or an infinite timeout. + * + * When an RPC times out, the function will return an error AMQP_STATUS_TIMEOUT, + * and the connection will be closed. + * + *\warning RPC-timeouts are an advanced feature intended to be used to detect + * dead connections quickly when the rabbitmq-c implementation of heartbeats + * does not work. Do not use RPC timeouts unless you understand the implications + * of doing so. + * + * \param [in] state the connection object + * \return a struct timeval representing the current RPC timeout for the state + * object. A NULL value represents an infinite timeout. The memory returned is + * owned by the connection object. + * + * \since v0.9.0 + */ +AMQP_PUBLIC_FUNCTION +struct timeval *AMQP_CALL amqp_get_rpc_timeout(amqp_connection_state_t state); + +/** + * Set the RPC timeout + * + * Sets the timeout for any RPC-style AMQP command (e.g., amqp_queue_declare). + * This timeout may be changed at any time by calling this function with a new + * timeout. The timeout applies individually to each RPC that is made. + * + * The default value is NULL, or an infinite timeout. + * + * When an RPC times out, the function will return an error AMQP_STATUS_TIMEOUT, + * and the connection will be closed. + * + *\warning RPC-timeouts are an advanced feature intended to be used to detect + * dead connections quickly when the rabbitmq-c implementation of heartbeats + * does not work. Do not use RPC timeouts unless you understand the implications + * of doing so. + * + * \param [in] state the connection object + * \param [in] timeout a struct timeval* representing new RPC timeout for the + * state object. NULL represents an infinite timeout. The value of timeout is + * copied internally, the caller is responsible for ownership of the passed + * pointer, it does not need to remain valid after this function is called. + * \return AMQP_STATUS_SUCCESS on success. + * + * \since v0.9.0 + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_set_rpc_timeout(amqp_connection_state_t state, + struct timeval *timeout); + +AMQP_END_DECLS + +#endif /* AMQP_H */ diff --git a/ext/librabbitmq/centos_x64/include/amqp_framing.h b/ext/librabbitmq/centos_x64/include/amqp_framing.h new file mode 100644 index 00000000..fb20acc1 --- /dev/null +++ b/ext/librabbitmq/centos_x64/include/amqp_framing.h @@ -0,0 +1,1144 @@ +/* Generated code. Do not edit. Edit and re-run codegen.py instead. + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MIT + * + * Portions created by Alan Antonuk are Copyright (c) 2012-2013 + * Alan Antonuk. All Rights Reserved. + * + * Portions created by VMware are Copyright (c) 2007-2012 VMware, Inc. + * All Rights Reserved. + * + * Portions created by Tony Garnock-Jones are Copyright (c) 2009-2010 + * VMware, Inc. and Tony Garnock-Jones. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * ***** END LICENSE BLOCK ***** + */ + +/** @file amqp_framing.h */ +#ifndef AMQP_FRAMING_H +#define AMQP_FRAMING_H + +#include <amqp.h> + +AMQP_BEGIN_DECLS + +#define AMQP_PROTOCOL_VERSION_MAJOR 0 /**< AMQP protocol version major */ +#define AMQP_PROTOCOL_VERSION_MINOR 9 /**< AMQP protocol version minor */ +#define AMQP_PROTOCOL_VERSION_REVISION \ + 1 /**< AMQP protocol version revision \ + */ +#define AMQP_PROTOCOL_PORT 5672 /**< Default AMQP Port */ +#define AMQP_FRAME_METHOD 1 /**< Constant: FRAME-METHOD */ +#define AMQP_FRAME_HEADER 2 /**< Constant: FRAME-HEADER */ +#define AMQP_FRAME_BODY 3 /**< Constant: FRAME-BODY */ +#define AMQP_FRAME_HEARTBEAT 8 /**< Constant: FRAME-HEARTBEAT */ +#define AMQP_FRAME_MIN_SIZE 4096 /**< Constant: FRAME-MIN-SIZE */ +#define AMQP_FRAME_END 206 /**< Constant: FRAME-END */ +#define AMQP_REPLY_SUCCESS 200 /**< Constant: REPLY-SUCCESS */ +#define AMQP_CONTENT_TOO_LARGE 311 /**< Constant: CONTENT-TOO-LARGE */ +#define AMQP_NO_ROUTE 312 /**< Constant: NO-ROUTE */ +#define AMQP_NO_CONSUMERS 313 /**< Constant: NO-CONSUMERS */ +#define AMQP_ACCESS_REFUSED 403 /**< Constant: ACCESS-REFUSED */ +#define AMQP_NOT_FOUND 404 /**< Constant: NOT-FOUND */ +#define AMQP_RESOURCE_LOCKED 405 /**< Constant: RESOURCE-LOCKED */ +#define AMQP_PRECONDITION_FAILED 406 /**< Constant: PRECONDITION-FAILED */ +#define AMQP_CONNECTION_FORCED 320 /**< Constant: CONNECTION-FORCED */ +#define AMQP_INVALID_PATH 402 /**< Constant: INVALID-PATH */ +#define AMQP_FRAME_ERROR 501 /**< Constant: FRAME-ERROR */ +#define AMQP_SYNTAX_ERROR 502 /**< Constant: SYNTAX-ERROR */ +#define AMQP_COMMAND_INVALID 503 /**< Constant: COMMAND-INVALID */ +#define AMQP_CHANNEL_ERROR 504 /**< Constant: CHANNEL-ERROR */ +#define AMQP_UNEXPECTED_FRAME 505 /**< Constant: UNEXPECTED-FRAME */ +#define AMQP_RESOURCE_ERROR 506 /**< Constant: RESOURCE-ERROR */ +#define AMQP_NOT_ALLOWED 530 /**< Constant: NOT-ALLOWED */ +#define AMQP_NOT_IMPLEMENTED 540 /**< Constant: NOT-IMPLEMENTED */ +#define AMQP_INTERNAL_ERROR 541 /**< Constant: INTERNAL-ERROR */ + +/* Function prototypes. */ + +/** + * Get constant name string from constant + * + * @param [in] constantNumber constant to get the name of + * @returns string describing the constant. String is managed by + * the library and should not be free()'d by the program + */ +AMQP_PUBLIC_FUNCTION +char const *AMQP_CALL amqp_constant_name(int constantNumber); + +/** + * Checks to see if a constant is a hard error + * + * A hard error occurs when something severe enough + * happens that the connection must be closed. + * + * @param [in] constantNumber the error constant + * @returns true if its a hard error, false otherwise + */ +AMQP_PUBLIC_FUNCTION +amqp_boolean_t AMQP_CALL amqp_constant_is_hard_error(int constantNumber); + +/** + * Get method name string from method number + * + * @param [in] methodNumber the method number + * @returns method name string. String is managed by the library + * and should not be freed()'d by the program + */ +AMQP_PUBLIC_FUNCTION +char const *AMQP_CALL amqp_method_name(amqp_method_number_t methodNumber); + +/** + * Check whether a method has content + * + * A method that has content will receive the method frame + * a properties frame, then 1 to N body frames + * + * @param [in] methodNumber the method number + * @returns true if method has content, false otherwise + */ +AMQP_PUBLIC_FUNCTION +amqp_boolean_t AMQP_CALL + amqp_method_has_content(amqp_method_number_t methodNumber); + +/** + * Decodes a method from AMQP wireformat + * + * @param [in] methodNumber the method number for the decoded parameter + * @param [in] pool the memory pool to allocate the decoded method from + * @param [in] encoded the encoded byte string buffer + * @param [out] decoded pointer to the decoded method struct + * @returns 0 on success, an error code otherwise + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_decode_method(amqp_method_number_t methodNumber, + amqp_pool_t *pool, amqp_bytes_t encoded, + void **decoded); + +/** + * Decodes a header frame properties structure from AMQP wireformat + * + * @param [in] class_id the class id for the decoded parameter + * @param [in] pool the memory pool to allocate the decoded properties from + * @param [in] encoded the encoded byte string buffer + * @param [out] decoded pointer to the decoded properties struct + * @returns 0 on success, an error code otherwise + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_decode_properties(uint16_t class_id, amqp_pool_t *pool, + amqp_bytes_t encoded, void **decoded); + +/** + * Encodes a method structure in AMQP wireformat + * + * @param [in] methodNumber the method number for the decoded parameter + * @param [in] decoded the method structure (e.g., amqp_connection_start_t) + * @param [in] encoded an allocated byte buffer for the encoded method + * structure to be written to. If the buffer isn't large enough + * to hold the encoded method, an error code will be returned. + * @returns 0 on success, an error code otherwise. + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_encode_method(amqp_method_number_t methodNumber, + void *decoded, amqp_bytes_t encoded); + +/** + * Encodes a properties structure in AMQP wireformat + * + * @param [in] class_id the class id for the decoded parameter + * @param [in] decoded the properties structure (e.g., amqp_basic_properties_t) + * @param [in] encoded an allocated byte buffer for the encoded properties to + * written to. + * If the buffer isn't large enough to hold the encoded method, an + * an error code will be returned + * @returns 0 on success, an error code otherwise. + */ +AMQP_PUBLIC_FUNCTION +int AMQP_CALL amqp_encode_properties(uint16_t class_id, void *decoded, + amqp_bytes_t encoded); + +/* Method field records. */ + +#define AMQP_CONNECTION_START_METHOD \ + ((amqp_method_number_t)0x000A000A) /**< connection.start method id \ + @internal 10, 10; 655370 */ +/** connection.start method fields */ +typedef struct amqp_connection_start_t_ { + uint8_t version_major; /**< version-major */ + uint8_t version_minor; /**< version-minor */ + amqp_table_t server_properties; /**< server-properties */ + amqp_bytes_t mechanisms; /**< mechanisms */ + amqp_bytes_t locales; /**< locales */ +} amqp_connection_start_t; + +#define AMQP_CONNECTION_START_OK_METHOD \ + ((amqp_method_number_t)0x000A000B) /**< connection.start-ok method id \ + @internal 10, 11; 655371 */ +/** connection.start-ok method fields */ +typedef struct amqp_connection_start_ok_t_ { + amqp_table_t client_properties; /**< client-properties */ + amqp_bytes_t mechanism; /**< mechanism */ + amqp_bytes_t response; /**< response */ + amqp_bytes_t locale; /**< locale */ +} amqp_connection_start_ok_t; + +#define AMQP_CONNECTION_SECURE_METHOD \ + ((amqp_method_number_t)0x000A0014) /**< connection.secure method id \ + @internal 10, 20; 655380 */ +/** connection.secure method fields */ +typedef struct amqp_connection_secure_t_ { + amqp_bytes_t challenge; /**< challenge */ +} amqp_connection_secure_t; + +#define AMQP_CONNECTION_SECURE_OK_METHOD \ + ((amqp_method_number_t)0x000A0015) /**< connection.secure-ok method id \ + @internal 10, 21; 655381 */ +/** connection.secure-ok method fields */ +typedef struct amqp_connection_secure_ok_t_ { + amqp_bytes_t response; /**< response */ +} amqp_connection_secure_ok_t; + +#define AMQP_CONNECTION_TUNE_METHOD \ + ((amqp_method_number_t)0x000A001E) /**< connection.tune method id \ + @internal 10, 30; 655390 */ +/** connection.tune method fields */ +typedef struct amqp_connection_tune_t_ { + uint16_t channel_max; /**< channel-max */ + uint32_t frame_max; /**< frame-max */ + uint16_t heartbeat; /**< heartbeat */ +} amqp_connection_tune_t; + +#define AMQP_CONNECTION_TUNE_OK_METHOD \ + ((amqp_method_number_t)0x000A001F) /**< connection.tune-ok method id \ + @internal 10, 31; 655391 */ +/** connection.tune-ok method fields */ +typedef struct amqp_connection_tune_ok_t_ { + uint16_t channel_max; /**< channel-max */ + uint32_t frame_max; /**< frame-max */ + uint16_t heartbeat; /**< heartbeat */ +} amqp_connection_tune_ok_t; + +#define AMQP_CONNECTION_OPEN_METHOD \ + ((amqp_method_number_t)0x000A0028) /**< connection.open method id \ + @internal 10, 40; 655400 */ +/** connection.open method fields */ +typedef struct amqp_connection_open_t_ { + amqp_bytes_t virtual_host; /**< virtual-host */ + amqp_bytes_t capabilities; /**< capabilities */ + amqp_boolean_t insist; /**< insist */ +} amqp_connection_open_t; + +#define AMQP_CONNECTION_OPEN_OK_METHOD \ + ((amqp_method_number_t)0x000A0029) /**< connection.open-ok method id \ + @internal 10, 41; 655401 */ +/** connection.open-ok method fields */ +typedef struct amqp_connection_open_ok_t_ { + amqp_bytes_t known_hosts; /**< known-hosts */ +} amqp_connection_open_ok_t; + +#define AMQP_CONNECTION_CLOSE_METHOD \ + ((amqp_method_number_t)0x000A0032) /**< connection.close method id \ + @internal 10, 50; 655410 */ +/** connection.close method fields */ +typedef struct amqp_connection_close_t_ { + uint16_t reply_code; /**< reply-code */ + amqp_bytes_t reply_text; /**< reply-text */ + uint16_t class_id; /**< class-id */ + uint16_t method_id; /**< method-id */ +} amqp_connection_close_t; + +#define AMQP_CONNECTION_CLOSE_OK_METHOD \ + ((amqp_method_number_t)0x000A0033) /**< connection.close-ok method id \ + @internal 10, 51; 655411 */ +/** connection.close-ok method fields */ +typedef struct amqp_connection_close_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_connection_close_ok_t; + +#define AMQP_CONNECTION_BLOCKED_METHOD \ + ((amqp_method_number_t)0x000A003C) /**< connection.blocked method id \ + @internal 10, 60; 655420 */ +/** connection.blocked method fields */ +typedef struct amqp_connection_blocked_t_ { + amqp_bytes_t reason; /**< reason */ +} amqp_connection_blocked_t; + +#define AMQP_CONNECTION_UNBLOCKED_METHOD \ + ((amqp_method_number_t)0x000A003D) /**< connection.unblocked method id \ + @internal 10, 61; 655421 */ +/** connection.unblocked method fields */ +typedef struct amqp_connection_unblocked_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_connection_unblocked_t; + +#define AMQP_CHANNEL_OPEN_METHOD \ + ((amqp_method_number_t)0x0014000A) /**< channel.open method id @internal \ + 20, 10; 1310730 */ +/** channel.open method fields */ +typedef struct amqp_channel_open_t_ { + amqp_bytes_t out_of_band; /**< out-of-band */ +} amqp_channel_open_t; + +#define AMQP_CHANNEL_OPEN_OK_METHOD \ + ((amqp_method_number_t)0x0014000B) /**< channel.open-ok method id \ + @internal 20, 11; 1310731 */ +/** channel.open-ok method fields */ +typedef struct amqp_channel_open_ok_t_ { + amqp_bytes_t channel_id; /**< channel-id */ +} amqp_channel_open_ok_t; + +#define AMQP_CHANNEL_FLOW_METHOD \ + ((amqp_method_number_t)0x00140014) /**< channel.flow method id @internal \ + 20, 20; 1310740 */ +/** channel.flow method fields */ +typedef struct amqp_channel_flow_t_ { + amqp_boolean_t active; /**< active */ +} amqp_channel_flow_t; + +#define AMQP_CHANNEL_FLOW_OK_METHOD \ + ((amqp_method_number_t)0x00140015) /**< channel.flow-ok method id \ + @internal 20, 21; 1310741 */ +/** channel.flow-ok method fields */ +typedef struct amqp_channel_flow_ok_t_ { + amqp_boolean_t active; /**< active */ +} amqp_channel_flow_ok_t; + +#define AMQP_CHANNEL_CLOSE_METHOD \ + ((amqp_method_number_t)0x00140028) /**< channel.close method id @internal \ + 20, 40; 1310760 */ +/** channel.close method fields */ +typedef struct amqp_channel_close_t_ { + uint16_t reply_code; /**< reply-code */ + amqp_bytes_t reply_text; /**< reply-text */ + uint16_t class_id; /**< class-id */ + uint16_t method_id; /**< method-id */ +} amqp_channel_close_t; + +#define AMQP_CHANNEL_CLOSE_OK_METHOD \ + ((amqp_method_number_t)0x00140029) /**< channel.close-ok method id \ + @internal 20, 41; 1310761 */ +/** channel.close-ok method fields */ +typedef struct amqp_channel_close_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_channel_close_ok_t; + +#define AMQP_ACCESS_REQUEST_METHOD \ + ((amqp_method_number_t)0x001E000A) /**< access.request method id @internal \ + 30, 10; 1966090 */ +/** access.request method fields */ +typedef struct amqp_access_request_t_ { + amqp_bytes_t realm; /**< realm */ + amqp_boolean_t exclusive; /**< exclusive */ + amqp_boolean_t passive; /**< passive */ + amqp_boolean_t active; /**< active */ + amqp_boolean_t write; /**< write */ + amqp_boolean_t read; /**< read */ +} amqp_access_request_t; + +#define AMQP_ACCESS_REQUEST_OK_METHOD \ + ((amqp_method_number_t)0x001E000B) /**< access.request-ok method id \ + @internal 30, 11; 1966091 */ +/** access.request-ok method fields */ +typedef struct amqp_access_request_ok_t_ { + uint16_t ticket; /**< ticket */ +} amqp_access_request_ok_t; + +#define AMQP_EXCHANGE_DECLARE_METHOD \ + ((amqp_method_number_t)0x0028000A) /**< exchange.declare method id \ + @internal 40, 10; 2621450 */ +/** exchange.declare method fields */ +typedef struct amqp_exchange_declare_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t exchange; /**< exchange */ + amqp_bytes_t type; /**< type */ + amqp_boolean_t passive; /**< passive */ + amqp_boolean_t durable; /**< durable */ + amqp_boolean_t auto_delete; /**< auto-delete */ + amqp_boolean_t internal; /**< internal */ + amqp_boolean_t nowait; /**< nowait */ + amqp_table_t arguments; /**< arguments */ +} amqp_exchange_declare_t; + +#define AMQP_EXCHANGE_DECLARE_OK_METHOD \ + ((amqp_method_number_t)0x0028000B) /**< exchange.declare-ok method id \ + @internal 40, 11; 2621451 */ +/** exchange.declare-ok method fields */ +typedef struct amqp_exchange_declare_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_exchange_declare_ok_t; + +#define AMQP_EXCHANGE_DELETE_METHOD \ + ((amqp_method_number_t)0x00280014) /**< exchange.delete method id \ + @internal 40, 20; 2621460 */ +/** exchange.delete method fields */ +typedef struct amqp_exchange_delete_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t exchange; /**< exchange */ + amqp_boolean_t if_unused; /**< if-unused */ + amqp_boolean_t nowait; /**< nowait */ +} amqp_exchange_delete_t; + +#define AMQP_EXCHANGE_DELETE_OK_METHOD \ + ((amqp_method_number_t)0x00280015) /**< exchange.delete-ok method id \ + @internal 40, 21; 2621461 */ +/** exchange.delete-ok method fields */ +typedef struct amqp_exchange_delete_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_exchange_delete_ok_t; + +#define AMQP_EXCHANGE_BIND_METHOD \ + ((amqp_method_number_t)0x0028001E) /**< exchange.bind method id @internal \ + 40, 30; 2621470 */ +/** exchange.bind method fields */ +typedef struct amqp_exchange_bind_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t destination; /**< destination */ + amqp_bytes_t source; /**< source */ + amqp_bytes_t routing_key; /**< routing-key */ + amqp_boolean_t nowait; /**< nowait */ + amqp_table_t arguments; /**< arguments */ +} amqp_exchange_bind_t; + +#define AMQP_EXCHANGE_BIND_OK_METHOD \ + ((amqp_method_number_t)0x0028001F) /**< exchange.bind-ok method id \ + @internal 40, 31; 2621471 */ +/** exchange.bind-ok method fields */ +typedef struct amqp_exchange_bind_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_exchange_bind_ok_t; + +#define AMQP_EXCHANGE_UNBIND_METHOD \ + ((amqp_method_number_t)0x00280028) /**< exchange.unbind method id \ + @internal 40, 40; 2621480 */ +/** exchange.unbind method fields */ +typedef struct amqp_exchange_unbind_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t destination; /**< destination */ + amqp_bytes_t source; /**< source */ + amqp_bytes_t routing_key; /**< routing-key */ + amqp_boolean_t nowait; /**< nowait */ + amqp_table_t arguments; /**< arguments */ +} amqp_exchange_unbind_t; + +#define AMQP_EXCHANGE_UNBIND_OK_METHOD \ + ((amqp_method_number_t)0x00280033) /**< exchange.unbind-ok method id \ + @internal 40, 51; 2621491 */ +/** exchange.unbind-ok method fields */ +typedef struct amqp_exchange_unbind_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_exchange_unbind_ok_t; + +#define AMQP_QUEUE_DECLARE_METHOD \ + ((amqp_method_number_t)0x0032000A) /**< queue.declare method id @internal \ + 50, 10; 3276810 */ +/** queue.declare method fields */ +typedef struct amqp_queue_declare_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t queue; /**< queue */ + amqp_boolean_t passive; /**< passive */ + amqp_boolean_t durable; /**< durable */ + amqp_boolean_t exclusive; /**< exclusive */ + amqp_boolean_t auto_delete; /**< auto-delete */ + amqp_boolean_t nowait; /**< nowait */ + amqp_table_t arguments; /**< arguments */ +} amqp_queue_declare_t; + +#define AMQP_QUEUE_DECLARE_OK_METHOD \ + ((amqp_method_number_t)0x0032000B) /**< queue.declare-ok method id \ + @internal 50, 11; 3276811 */ +/** queue.declare-ok method fields */ +typedef struct amqp_queue_declare_ok_t_ { + amqp_bytes_t queue; /**< queue */ + uint32_t message_count; /**< message-count */ + uint32_t consumer_count; /**< consumer-count */ +} amqp_queue_declare_ok_t; + +#define AMQP_QUEUE_BIND_METHOD \ + ((amqp_method_number_t)0x00320014) /**< queue.bind method id @internal 50, \ + 20; 3276820 */ +/** queue.bind method fields */ +typedef struct amqp_queue_bind_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t queue; /**< queue */ + amqp_bytes_t exchange; /**< exchange */ + amqp_bytes_t routing_key; /**< routing-key */ + amqp_boolean_t nowait; /**< nowait */ + amqp_table_t arguments; /**< arguments */ +} amqp_queue_bind_t; + +#define AMQP_QUEUE_BIND_OK_METHOD \ + ((amqp_method_number_t)0x00320015) /**< queue.bind-ok method id @internal \ + 50, 21; 3276821 */ +/** queue.bind-ok method fields */ +typedef struct amqp_queue_bind_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_queue_bind_ok_t; + +#define AMQP_QUEUE_PURGE_METHOD \ + ((amqp_method_number_t)0x0032001E) /**< queue.purge method id @internal \ + 50, 30; 3276830 */ +/** queue.purge method fields */ +typedef struct amqp_queue_purge_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t queue; /**< queue */ + amqp_boolean_t nowait; /**< nowait */ +} amqp_queue_purge_t; + +#define AMQP_QUEUE_PURGE_OK_METHOD \ + ((amqp_method_number_t)0x0032001F) /**< queue.purge-ok method id @internal \ + 50, 31; 3276831 */ +/** queue.purge-ok method fields */ +typedef struct amqp_queue_purge_ok_t_ { + uint32_t message_count; /**< message-count */ +} amqp_queue_purge_ok_t; + +#define AMQP_QUEUE_DELETE_METHOD \ + ((amqp_method_number_t)0x00320028) /**< queue.delete method id @internal \ + 50, 40; 3276840 */ +/** queue.delete method fields */ +typedef struct amqp_queue_delete_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t queue; /**< queue */ + amqp_boolean_t if_unused; /**< if-unused */ + amqp_boolean_t if_empty; /**< if-empty */ + amqp_boolean_t nowait; /**< nowait */ +} amqp_queue_delete_t; + +#define AMQP_QUEUE_DELETE_OK_METHOD \ + ((amqp_method_number_t)0x00320029) /**< queue.delete-ok method id \ + @internal 50, 41; 3276841 */ +/** queue.delete-ok method fields */ +typedef struct amqp_queue_delete_ok_t_ { + uint32_t message_count; /**< message-count */ +} amqp_queue_delete_ok_t; + +#define AMQP_QUEUE_UNBIND_METHOD \ + ((amqp_method_number_t)0x00320032) /**< queue.unbind method id @internal \ + 50, 50; 3276850 */ +/** queue.unbind method fields */ +typedef struct amqp_queue_unbind_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t queue; /**< queue */ + amqp_bytes_t exchange; /**< exchange */ + amqp_bytes_t routing_key; /**< routing-key */ + amqp_table_t arguments; /**< arguments */ +} amqp_queue_unbind_t; + +#define AMQP_QUEUE_UNBIND_OK_METHOD \ + ((amqp_method_number_t)0x00320033) /**< queue.unbind-ok method id \ + @internal 50, 51; 3276851 */ +/** queue.unbind-ok method fields */ +typedef struct amqp_queue_unbind_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_queue_unbind_ok_t; + +#define AMQP_BASIC_QOS_METHOD \ + ((amqp_method_number_t)0x003C000A) /**< basic.qos method id @internal 60, \ + 10; 3932170 */ +/** basic.qos method fields */ +typedef struct amqp_basic_qos_t_ { + uint32_t prefetch_size; /**< prefetch-size */ + uint16_t prefetch_count; /**< prefetch-count */ + amqp_boolean_t global; /**< global */ +} amqp_basic_qos_t; + +#define AMQP_BASIC_QOS_OK_METHOD \ + ((amqp_method_number_t)0x003C000B) /**< basic.qos-ok method id @internal \ + 60, 11; 3932171 */ +/** basic.qos-ok method fields */ +typedef struct amqp_basic_qos_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_basic_qos_ok_t; + +#define AMQP_BASIC_CONSUME_METHOD \ + ((amqp_method_number_t)0x003C0014) /**< basic.consume method id @internal \ + 60, 20; 3932180 */ +/** basic.consume method fields */ +typedef struct amqp_basic_consume_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t queue; /**< queue */ + amqp_bytes_t consumer_tag; /**< consumer-tag */ + amqp_boolean_t no_local; /**< no-local */ + amqp_boolean_t no_ack; /**< no-ack */ + amqp_boolean_t exclusive; /**< exclusive */ + amqp_boolean_t nowait; /**< nowait */ + amqp_table_t arguments; /**< arguments */ +} amqp_basic_consume_t; + +#define AMQP_BASIC_CONSUME_OK_METHOD \ + ((amqp_method_number_t)0x003C0015) /**< basic.consume-ok method id \ + @internal 60, 21; 3932181 */ +/** basic.consume-ok method fields */ +typedef struct amqp_basic_consume_ok_t_ { + amqp_bytes_t consumer_tag; /**< consumer-tag */ +} amqp_basic_consume_ok_t; + +#define AMQP_BASIC_CANCEL_METHOD \ + ((amqp_method_number_t)0x003C001E) /**< basic.cancel method id @internal \ + 60, 30; 3932190 */ +/** basic.cancel method fields */ +typedef struct amqp_basic_cancel_t_ { + amqp_bytes_t consumer_tag; /**< consumer-tag */ + amqp_boolean_t nowait; /**< nowait */ +} amqp_basic_cancel_t; + +#define AMQP_BASIC_CANCEL_OK_METHOD \ + ((amqp_method_number_t)0x003C001F) /**< basic.cancel-ok method id \ + @internal 60, 31; 3932191 */ +/** basic.cancel-ok method fields */ +typedef struct amqp_basic_cancel_ok_t_ { + amqp_bytes_t consumer_tag; /**< consumer-tag */ +} amqp_basic_cancel_ok_t; + +#define AMQP_BASIC_PUBLISH_METHOD \ + ((amqp_method_number_t)0x003C0028) /**< basic.publish method id @internal \ + 60, 40; 3932200 */ +/** basic.publish method fields */ +typedef struct amqp_basic_publish_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t exchange; /**< exchange */ + amqp_bytes_t routing_key; /**< routing-key */ + amqp_boolean_t mandatory; /**< mandatory */ + amqp_boolean_t immediate; /**< immediate */ +} amqp_basic_publish_t; + +#define AMQP_BASIC_RETURN_METHOD \ + ((amqp_method_number_t)0x003C0032) /**< basic.return method id @internal \ + 60, 50; 3932210 */ +/** basic.return method fields */ +typedef struct amqp_basic_return_t_ { + uint16_t reply_code; /**< reply-code */ + amqp_bytes_t reply_text; /**< reply-text */ + amqp_bytes_t exchange; /**< exchange */ + amqp_bytes_t routing_key; /**< routing-key */ +} amqp_basic_return_t; + +#define AMQP_BASIC_DELIVER_METHOD \ + ((amqp_method_number_t)0x003C003C) /**< basic.deliver method id @internal \ + 60, 60; 3932220 */ +/** basic.deliver method fields */ +typedef struct amqp_basic_deliver_t_ { + amqp_bytes_t consumer_tag; /**< consumer-tag */ + uint64_t delivery_tag; /**< delivery-tag */ + amqp_boolean_t redelivered; /**< redelivered */ + amqp_bytes_t exchange; /**< exchange */ + amqp_bytes_t routing_key; /**< routing-key */ +} amqp_basic_deliver_t; + +#define AMQP_BASIC_GET_METHOD \ + ((amqp_method_number_t)0x003C0046) /**< basic.get method id @internal 60, \ + 70; 3932230 */ +/** basic.get method fields */ +typedef struct amqp_basic_get_t_ { + uint16_t ticket; /**< ticket */ + amqp_bytes_t queue; /**< queue */ + amqp_boolean_t no_ack; /**< no-ack */ +} amqp_basic_get_t; + +#define AMQP_BASIC_GET_OK_METHOD \ + ((amqp_method_number_t)0x003C0047) /**< basic.get-ok method id @internal \ + 60, 71; 3932231 */ +/** basic.get-ok method fields */ +typedef struct amqp_basic_get_ok_t_ { + uint64_t delivery_tag; /**< delivery-tag */ + amqp_boolean_t redelivered; /**< redelivered */ + amqp_bytes_t exchange; /**< exchange */ + amqp_bytes_t routing_key; /**< routing-key */ + uint32_t message_count; /**< message-count */ +} amqp_basic_get_ok_t; + +#define AMQP_BASIC_GET_EMPTY_METHOD \ + ((amqp_method_number_t)0x003C0048) /**< basic.get-empty method id \ + @internal 60, 72; 3932232 */ +/** basic.get-empty method fields */ +typedef struct amqp_basic_get_empty_t_ { + amqp_bytes_t cluster_id; /**< cluster-id */ +} amqp_basic_get_empty_t; + +#define AMQP_BASIC_ACK_METHOD \ + ((amqp_method_number_t)0x003C0050) /**< basic.ack method id @internal 60, \ + 80; 3932240 */ +/** basic.ack method fields */ +typedef struct amqp_basic_ack_t_ { + uint64_t delivery_tag; /**< delivery-tag */ + amqp_boolean_t multiple; /**< multiple */ +} amqp_basic_ack_t; + +#define AMQP_BASIC_REJECT_METHOD \ + ((amqp_method_number_t)0x003C005A) /**< basic.reject method id @internal \ + 60, 90; 3932250 */ +/** basic.reject method fields */ +typedef struct amqp_basic_reject_t_ { + uint64_t delivery_tag; /**< delivery-tag */ + amqp_boolean_t requeue; /**< requeue */ +} amqp_basic_reject_t; + +#define AMQP_BASIC_RECOVER_ASYNC_METHOD \ + ((amqp_method_number_t)0x003C0064) /**< basic.recover-async method id \ + @internal 60, 100; 3932260 */ +/** basic.recover-async method fields */ +typedef struct amqp_basic_recover_async_t_ { + amqp_boolean_t requeue; /**< requeue */ +} amqp_basic_recover_async_t; + +#define AMQP_BASIC_RECOVER_METHOD \ + ((amqp_method_number_t)0x003C006E) /**< basic.recover method id @internal \ + 60, 110; 3932270 */ +/** basic.recover method fields */ +typedef struct amqp_basic_recover_t_ { + amqp_boolean_t requeue; /**< requeue */ +} amqp_basic_recover_t; + +#define AMQP_BASIC_RECOVER_OK_METHOD \ + ((amqp_method_number_t)0x003C006F) /**< basic.recover-ok method id \ + @internal 60, 111; 3932271 */ +/** basic.recover-ok method fields */ +typedef struct amqp_basic_recover_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_basic_recover_ok_t; + +#define AMQP_BASIC_NACK_METHOD \ + ((amqp_method_number_t)0x003C0078) /**< basic.nack method id @internal 60, \ + 120; 3932280 */ +/** basic.nack method fields */ +typedef struct amqp_basic_nack_t_ { + uint64_t delivery_tag; /**< delivery-tag */ + amqp_boolean_t multiple; /**< multiple */ + amqp_boolean_t requeue; /**< requeue */ +} amqp_basic_nack_t; + +#define AMQP_TX_SELECT_METHOD \ + ((amqp_method_number_t)0x005A000A) /**< tx.select method id @internal 90, \ + 10; 5898250 */ +/** tx.select method fields */ +typedef struct amqp_tx_select_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_tx_select_t; + +#define AMQP_TX_SELECT_OK_METHOD \ + ((amqp_method_number_t)0x005A000B) /**< tx.select-ok method id @internal \ + 90, 11; 5898251 */ +/** tx.select-ok method fields */ +typedef struct amqp_tx_select_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_tx_select_ok_t; + +#define AMQP_TX_COMMIT_METHOD \ + ((amqp_method_number_t)0x005A0014) /**< tx.commit method id @internal 90, \ + 20; 5898260 */ +/** tx.commit method fields */ +typedef struct amqp_tx_commit_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_tx_commit_t; + +#define AMQP_TX_COMMIT_OK_METHOD \ + ((amqp_method_number_t)0x005A0015) /**< tx.commit-ok method id @internal \ + 90, 21; 5898261 */ +/** tx.commit-ok method fields */ +typedef struct amqp_tx_commit_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_tx_commit_ok_t; + +#define AMQP_TX_ROLLBACK_METHOD \ + ((amqp_method_number_t)0x005A001E) /**< tx.rollback method id @internal \ + 90, 30; 5898270 */ +/** tx.rollback method fields */ +typedef struct amqp_tx_rollback_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_tx_rollback_t; + +#define AMQP_TX_ROLLBACK_OK_METHOD \ + ((amqp_method_number_t)0x005A001F) /**< tx.rollback-ok method id @internal \ + 90, 31; 5898271 */ +/** tx.rollback-ok method fields */ +typedef struct amqp_tx_rollback_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_tx_rollback_ok_t; + +#define AMQP_CONFIRM_SELECT_METHOD \ + ((amqp_method_number_t)0x0055000A) /**< confirm.select method id @internal \ + 85, 10; 5570570 */ +/** confirm.select method fields */ +typedef struct amqp_confirm_select_t_ { + amqp_boolean_t nowait; /**< nowait */ +} amqp_confirm_select_t; + +#define AMQP_CONFIRM_SELECT_OK_METHOD \ + ((amqp_method_number_t)0x0055000B) /**< confirm.select-ok method id \ + @internal 85, 11; 5570571 */ +/** confirm.select-ok method fields */ +typedef struct amqp_confirm_select_ok_t_ { + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_confirm_select_ok_t; + +/* Class property records. */ +#define AMQP_CONNECTION_CLASS \ + (0x000A) /**< connection class id @internal 10 \ + */ +/** connection class properties */ +typedef struct amqp_connection_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_connection_properties_t; + +#define AMQP_CHANNEL_CLASS (0x0014) /**< channel class id @internal 20 */ +/** channel class properties */ +typedef struct amqp_channel_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_channel_properties_t; + +#define AMQP_ACCESS_CLASS (0x001E) /**< access class id @internal 30 */ +/** access class properties */ +typedef struct amqp_access_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_access_properties_t; + +#define AMQP_EXCHANGE_CLASS (0x0028) /**< exchange class id @internal 40 */ +/** exchange class properties */ +typedef struct amqp_exchange_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_exchange_properties_t; + +#define AMQP_QUEUE_CLASS (0x0032) /**< queue class id @internal 50 */ +/** queue class properties */ +typedef struct amqp_queue_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_queue_properties_t; + +#define AMQP_BASIC_CLASS (0x003C) /**< basic class id @internal 60 */ +#define AMQP_BASIC_CONTENT_TYPE_FLAG (1 << 15) +#define AMQP_BASIC_CONTENT_ENCODING_FLAG (1 << 14) +#define AMQP_BASIC_HEADERS_FLAG (1 << 13) +#define AMQP_BASIC_DELIVERY_MODE_FLAG (1 << 12) +#define AMQP_BASIC_PRIORITY_FLAG (1 << 11) +#define AMQP_BASIC_CORRELATION_ID_FLAG (1 << 10) +#define AMQP_BASIC_REPLY_TO_FLAG (1 << 9) +#define AMQP_BASIC_EXPIRATION_FLAG (1 << 8) +#define AMQP_BASIC_MESSAGE_ID_FLAG (1 << 7) +#define AMQP_BASIC_TIMESTAMP_FLAG (1 << 6) +#define AMQP_BASIC_TYPE_FLAG (1 << 5) +#define AMQP_BASIC_USER_ID_FLAG (1 << 4) +#define AMQP_BASIC_APP_ID_FLAG (1 << 3) +#define AMQP_BASIC_CLUSTER_ID_FLAG (1 << 2) +/** basic class properties */ +typedef struct amqp_basic_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + amqp_bytes_t content_type; /**< content-type */ + amqp_bytes_t content_encoding; /**< content-encoding */ + amqp_table_t headers; /**< headers */ + uint8_t delivery_mode; /**< delivery-mode */ + uint8_t priority; /**< priority */ + amqp_bytes_t correlation_id; /**< correlation-id */ + amqp_bytes_t reply_to; /**< reply-to */ + amqp_bytes_t expiration; /**< expiration */ + amqp_bytes_t message_id; /**< message-id */ + uint64_t timestamp; /**< timestamp */ + amqp_bytes_t type; /**< type */ + amqp_bytes_t user_id; /**< user-id */ + amqp_bytes_t app_id; /**< app-id */ + amqp_bytes_t cluster_id; /**< cluster-id */ +} amqp_basic_properties_t; + +#define AMQP_TX_CLASS (0x005A) /**< tx class id @internal 90 */ +/** tx class properties */ +typedef struct amqp_tx_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_tx_properties_t; + +#define AMQP_CONFIRM_CLASS (0x0055) /**< confirm class id @internal 85 */ +/** confirm class properties */ +typedef struct amqp_confirm_properties_t_ { + amqp_flags_t _flags; /**< bit-mask of set fields */ + char dummy; /**< Dummy field to avoid empty struct */ +} amqp_confirm_properties_t; + +/* API functions for methods */ + +/** + * amqp_channel_open + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @returns amqp_channel_open_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_channel_open_ok_t *AMQP_CALL + amqp_channel_open(amqp_connection_state_t state, amqp_channel_t channel); +/** + * amqp_channel_flow + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] active active + * @returns amqp_channel_flow_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_channel_flow_ok_t *AMQP_CALL + amqp_channel_flow(amqp_connection_state_t state, amqp_channel_t channel, + amqp_boolean_t active); +/** + * amqp_exchange_declare + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] exchange exchange + * @param [in] type type + * @param [in] passive passive + * @param [in] durable durable + * @param [in] auto_delete auto_delete + * @param [in] internal internal + * @param [in] arguments arguments + * @returns amqp_exchange_declare_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_exchange_declare_ok_t *AMQP_CALL amqp_exchange_declare( + amqp_connection_state_t state, amqp_channel_t channel, + amqp_bytes_t exchange, amqp_bytes_t type, amqp_boolean_t passive, + amqp_boolean_t durable, amqp_boolean_t auto_delete, amqp_boolean_t internal, + amqp_table_t arguments); +/** + * amqp_exchange_delete + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] exchange exchange + * @param [in] if_unused if_unused + * @returns amqp_exchange_delete_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_exchange_delete_ok_t *AMQP_CALL + amqp_exchange_delete(amqp_connection_state_t state, amqp_channel_t channel, + amqp_bytes_t exchange, amqp_boolean_t if_unused); +/** + * amqp_exchange_bind + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] destination destination + * @param [in] source source + * @param [in] routing_key routing_key + * @param [in] arguments arguments + * @returns amqp_exchange_bind_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_exchange_bind_ok_t *AMQP_CALL + amqp_exchange_bind(amqp_connection_state_t state, amqp_channel_t channel, + amqp_bytes_t destination, amqp_bytes_t source, + amqp_bytes_t routing_key, amqp_table_t arguments); +/** + * amqp_exchange_unbind + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] destination destination + * @param [in] source source + * @param [in] routing_key routing_key + * @param [in] arguments arguments + * @returns amqp_exchange_unbind_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_exchange_unbind_ok_t *AMQP_CALL + amqp_exchange_unbind(amqp_connection_state_t state, amqp_channel_t channel, + amqp_bytes_t destination, amqp_bytes_t source, + amqp_bytes_t routing_key, amqp_table_t arguments); +/** + * amqp_queue_declare + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] queue queue + * @param [in] passive passive + * @param [in] durable durable + * @param [in] exclusive exclusive + * @param [in] auto_delete auto_delete + * @param [in] arguments arguments + * @returns amqp_queue_declare_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_queue_declare_ok_t *AMQP_CALL amqp_queue_declare( + amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue, + amqp_boolean_t passive, amqp_boolean_t durable, amqp_boolean_t exclusive, + amqp_boolean_t auto_delete, amqp_table_t arguments); +/** + * amqp_queue_bind + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] queue queue + * @param [in] exchange exchange + * @param [in] routing_key routing_key + * @param [in] arguments arguments + * @returns amqp_queue_bind_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_queue_bind_ok_t *AMQP_CALL amqp_queue_bind( + amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue, + amqp_bytes_t exchange, amqp_bytes_t routing_key, amqp_table_t arguments); +/** + * amqp_queue_purge + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] queue queue + * @returns amqp_queue_purge_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_queue_purge_ok_t *AMQP_CALL amqp_queue_purge(amqp_connection_state_t state, + amqp_channel_t channel, + amqp_bytes_t queue); +/** + * amqp_queue_delete + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] queue queue + * @param [in] if_unused if_unused + * @param [in] if_empty if_empty + * @returns amqp_queue_delete_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_queue_delete_ok_t *AMQP_CALL amqp_queue_delete( + amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue, + amqp_boolean_t if_unused, amqp_boolean_t if_empty); +/** + * amqp_queue_unbind + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] queue queue + * @param [in] exchange exchange + * @param [in] routing_key routing_key + * @param [in] arguments arguments + * @returns amqp_queue_unbind_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_queue_unbind_ok_t *AMQP_CALL amqp_queue_unbind( + amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue, + amqp_bytes_t exchange, amqp_bytes_t routing_key, amqp_table_t arguments); +/** + * amqp_basic_qos + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] prefetch_size prefetch_size + * @param [in] prefetch_count prefetch_count + * @param [in] global global + * @returns amqp_basic_qos_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_basic_qos_ok_t *AMQP_CALL amqp_basic_qos(amqp_connection_state_t state, + amqp_channel_t channel, + uint32_t prefetch_size, + uint16_t prefetch_count, + amqp_boolean_t global); +/** + * amqp_basic_consume + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] queue queue + * @param [in] consumer_tag consumer_tag + * @param [in] no_local no_local + * @param [in] no_ack no_ack + * @param [in] exclusive exclusive + * @param [in] arguments arguments + * @returns amqp_basic_consume_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_basic_consume_ok_t *AMQP_CALL amqp_basic_consume( + amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue, + amqp_bytes_t consumer_tag, amqp_boolean_t no_local, amqp_boolean_t no_ack, + amqp_boolean_t exclusive, amqp_table_t arguments); +/** + * amqp_basic_cancel + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] consumer_tag consumer_tag + * @returns amqp_basic_cancel_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_basic_cancel_ok_t *AMQP_CALL + amqp_basic_cancel(amqp_connection_state_t state, amqp_channel_t channel, + amqp_bytes_t consumer_tag); +/** + * amqp_basic_recover + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @param [in] requeue requeue + * @returns amqp_basic_recover_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_basic_recover_ok_t *AMQP_CALL + amqp_basic_recover(amqp_connection_state_t state, amqp_channel_t channel, + amqp_boolean_t requeue); +/** + * amqp_tx_select + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @returns amqp_tx_select_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_tx_select_ok_t *AMQP_CALL amqp_tx_select(amqp_connection_state_t state, + amqp_channel_t channel); +/** + * amqp_tx_commit + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @returns amqp_tx_commit_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_tx_commit_ok_t *AMQP_CALL amqp_tx_commit(amqp_connection_state_t state, + amqp_channel_t channel); +/** + * amqp_tx_rollback + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @returns amqp_tx_rollback_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_tx_rollback_ok_t *AMQP_CALL amqp_tx_rollback(amqp_connection_state_t state, + amqp_channel_t channel); +/** + * amqp_confirm_select + * + * @param [in] state connection state + * @param [in] channel the channel to do the RPC on + * @returns amqp_confirm_select_ok_t + */ +AMQP_PUBLIC_FUNCTION +amqp_confirm_select_ok_t *AMQP_CALL + amqp_confirm_select(amqp_connection_state_t state, amqp_channel_t channel); + +AMQP_END_DECLS + +#endif /* AMQP_FRAMING_H */ diff --git a/ext/librabbitmq/centos_x64/include/amqp_tcp_socket.h b/ext/librabbitmq/centos_x64/include/amqp_tcp_socket.h new file mode 100644 index 00000000..3e9d82f5 --- /dev/null +++ b/ext/librabbitmq/centos_x64/include/amqp_tcp_socket.h @@ -0,0 +1,68 @@ +/** \file */ +/* + * Portions created by Alan Antonuk are Copyright (c) 2013-2014 Alan Antonuk. + * All Rights Reserved. + * + * Portions created by Michael Steinert are Copyright (c) 2012-2013 Michael + * Steinert. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/** + * A TCP socket connection. + */ + +#ifndef AMQP_TCP_SOCKET_H +#define AMQP_TCP_SOCKET_H + +#include <amqp.h> + +AMQP_BEGIN_DECLS + +/** + * Create a new TCP socket. + * + * Call amqp_connection_close() to release socket resources. + * + * \return A new socket object or NULL if an error occurred. + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +amqp_socket_t *AMQP_CALL amqp_tcp_socket_new(amqp_connection_state_t state); + +/** + * Assign an open file descriptor to a socket object. + * + * This function must not be used in conjunction with amqp_socket_open(), i.e. + * the socket connection should already be open(2) when this function is + * called. + * + * \param [in,out] self A TCP socket object. + * \param [in] sockfd An open socket descriptor. + * + * \since v0.4.0 + */ +AMQP_PUBLIC_FUNCTION +void AMQP_CALL amqp_tcp_socket_set_sockfd(amqp_socket_t *self, int sockfd); + +AMQP_END_DECLS + +#endif /* AMQP_TCP_SOCKET_H */ diff --git a/ext/librabbitmq/centos_x64/lib/librabbitmq.a b/ext/librabbitmq/centos_x64/lib/librabbitmq.a Binary files differnew file mode 100644 index 00000000..d5c3e8b4 --- /dev/null +++ b/ext/librabbitmq/centos_x64/lib/librabbitmq.a diff --git a/ext/librethinkdbxx/.travis.yml b/ext/librethinkdbxx/.travis.yml deleted file mode 100644 index b306a410..00000000 --- a/ext/librethinkdbxx/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -sudo: required -dist: trusty - -python: - - "3.4.3" - -addons: - rethinkdb: "2.3" - -script: - - make test diff --git a/ext/librethinkdbxx/COPYRIGHT b/ext/librethinkdbxx/COPYRIGHT deleted file mode 100644 index c25145d5..00000000 --- a/ext/librethinkdbxx/COPYRIGHT +++ /dev/null @@ -1,16 +0,0 @@ -RethinkDB Language Drivers - -Copyright 2010-2012 RethinkDB - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this product except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/ext/librethinkdbxx/Makefile b/ext/librethinkdbxx/Makefile deleted file mode 100644 index ba319c16..00000000 --- a/ext/librethinkdbxx/Makefile +++ /dev/null @@ -1,126 +0,0 @@ -# Customisable build settings - -CXX ?= clang++ -CXXFLAGS ?= -INCLUDE_PYTHON_DOCS ?= no -DEBUG ?= no -PYTHON ?= python3 - -# Required build settings - -ifneq (no,$(DEBUG)) - CXXFLAGS += -ggdb -else - CXXFLAGS += -O3 # -flto -endif - -CXXFLAGS += -std=c++11 -I'build/gen' -Wall -pthread -fPIC - -prefix ?= /usr -DESTDIR ?= - -.DELETE_ON_ERROR: -SHELL := /bin/bash - -modules := connection datum json term cursor types utils -headers := utils error exceptions types datum connection cursor term - -o_files := $(patsubst %, build/obj/%.o, $(modules)) -d_files := $(patsubst %, build/dep/%.d, $(modules)) - -skip_tests := regression/1133 regression/767 regression/1005 # python-only -skip_tests += arity # arity errors are compile-time -skip_tests += geo # geo types not implemented yet -skip_tests += limits # possibly broken tests: https://github.com/rethinkdb/rethinkdb/issues/5940 - -upstream_tests := \ - $(filter-out %.rb.%, \ - $(filter-out $(patsubst %,test/upstream/%%, $(skip_tests)), \ - $(filter test/upstream/$(test_filter)%, \ - $(shell find test/upstream -name '*.yaml' | egrep -v '.(rb|js).yaml$$')))) -upstream_tests_cc := $(patsubst %.yaml, build/tests/%.cc, $(upstream_tests)) -upstream_tests_o := $(patsubst %.cc, %.o, $(upstream_tests_cc)) - -.PRECIOUS: $(upstream_tests_cc) $(upstream_tests_o) - -default: build/librethinkdb++.a build/include/rethinkdb.h build/librethinkdb++.so - -all: default build/test - -build/librethinkdb++.a: $(o_files) - ar rcs $@ $^ - -build/librethinkdb++.so: $(o_files) - $(CXX) -o $@ $(CXXFLAGS) -shared $^ - -build/obj/%.o: src/%.cc build/gen/protocol_defs.h - @mkdir -p $(dir $@) - @mkdir -p $(dir build/dep/$*.d) - $(CXX) -o $@ $(CXXFLAGS) -c $< -MP -MQ $@ -MD -MF build/dep/$*.d - -build/gen/protocol_defs.h: reql/ql2.proto reql/gen.py | build/gen/. - $(PYTHON) reql/gen.py $< > $@ - -clean: - rm -rf build - -ifneq (no,$(INCLUDE_PYTHON_DOCS)) -build/include/rethinkdb.h: build/rethinkdb.nodocs.h reql/add_docs.py reql/python_docs.txt | build/include/. - $(PYTHON) reql/add_docs.py reql/python_docs.txt < $< > $@ -else -build/include/rethinkdb.h: build/rethinkdb.nodocs.h | build/include/. - cp $< $@ -endif - -build/rethinkdb.nodocs.h: build/gen/protocol_defs.h $(patsubst %, src/%.h, $(headers)) - ( echo "// Auto-generated file, built from $^"; \ - echo '#pragma once'; \ - cat $^ | \ - grep -v '^#pragma once' | \ - grep -v '^#include "'; \ - ) > $@ - -build/tests/%.cc: %.yaml test/yaml_to_cxx.py - @mkdir -p $(dir $@) - $(PYTHON) test/yaml_to_cxx.py $< > $@ - -build/tests/upstream_tests.cc: $(upstream_tests) test/gen_index_cxx.py FORCE | build/tests/. - @echo '$(PYTHON) test/gen_index_cxx.py $(wordlist 1,5,$(upstream_tests)) ... > $@' - @$(PYTHON) test/gen_index_cxx.py $(upstream_tests) > $@ - -build/tests/%.o: build/tests/%.cc build/include/rethinkdb.h test/testlib.h | build/tests/. - $(CXX) -o $@ $(CXXFLAGS) -isystem build/include -I test -c $< -Wno-unused-variable - -build/tests/%.o: test/%.cc test/testlib.h build/include/rethinkdb.h | build/tests/. - $(CXX) -o $@ $(CXXFLAGS) -isystem build/include -I test -c $< - -build/test: build/tests/testlib.o build/tests/test.o build/tests/upstream_tests.o $(upstream_tests_o) build/librethinkdb++.a - @echo $(CXX) -o $@ $(CXXFLAGS) $(wordlist 1,5,$^) ... - @$(CXX) -o $@ $(CXXFLAGS) build/librethinkdb++.a $^ - -.PHONY: test -test: build/test - build/test - -build/bench: build/tests/bench.o build/librethinkdb++.a - @$(CXX) -o $@ $(CXXFLAGS) -isystem build/include build/librethinkdb++.a $^ - -.PHONY: bench -bench: build/bench - build/bench - -.PHONY: install -install: build/librethinkdb++.a build/include/rethinkdb.h build/librethinkdb++.so - install -m755 -d $(DESTDIR)$(prefix)/lib - install -m755 -d $(DESTDIR)$(prefix)/include - install -m644 build/librethinkdb++.a $(DESTDIR)$(prefix)/lib/librethinkdb++.a - install -m644 build/librethinkdb++.so $(DESTDIR)$(prefix)/lib/librethinkdb++.so - install -m644 build/include/rethinkdb.h $(DESTDIR)$(prefix)/include/rethinkdb.h - -%/.: - mkdir -p $* - -.PHONY: FORCE -FORCE: - --include $(d_files) diff --git a/ext/librethinkdbxx/README.md b/ext/librethinkdbxx/README.md deleted file mode 100644 index 92fa9136..00000000 --- a/ext/librethinkdbxx/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# RethinkDB driver for C++ - -This driver is compatible with RethinkDB 2.0. It is based on the -official RethinkDB Python driver. - -* [RethinkDB server](http://rethinkdb.com/) -* [RethinkDB API docs](http://rethinkdb.com/api/python/) - -## Example - -``` -#include <memory> -#include <cstdio> -#include <rethinkdb.h> - -namespace R = RethinkDB; - -int main() { - std::unique_ptr<R::Connection> conn = R::connect("localhost", 28015); - R::Cursor cursor = R::table("users").filter(R::row["age"] > 14).run(*conn); - for (R::Datum& user : cursor) { - printf("%s\n", user.as_json().c_str()); - } -} -``` - -## Build - -Requires a modern C++ compiler. to build and install, run: - -``` -make -make install -``` - -Will build `include/rethinkdb.h`, `librethinkdb++.a` and -`librethinkdb++.so` into the `build/` directory. - -To include documentation from the Python driver in the header file, -pass the following argument to make. - -``` -make INCLUDE_PYTHON_DOCS=yes -``` - -To build in debug mode: - -``` -make DEBUG=yes -``` - -To install to a specific location: - -``` -make install prefix=/usr/local DESTDIR= -``` - -## Status - -Still in early stages of development. - -## Tests - -This driver is tested against the upstream ReQL tests from the -RethinkDB repo, which are programmatically translated from Python to -C++. As of 34dc13c, all tests pass: - -``` -$ make test -... -SUCCESS: 2053 tests passed -``` diff --git a/ext/librethinkdbxx/reql/add_docs.py b/ext/librethinkdbxx/reql/add_docs.py deleted file mode 100644 index 67f08df8..00000000 --- a/ext/librethinkdbxx/reql/add_docs.py +++ /dev/null @@ -1,80 +0,0 @@ -from sys import stdin, stderr, stdout, argv -from re import match, sub - -docs = {} - -for line in open(argv[1]): - res = match('^\t\(([^,]*), (.*)\),$', line) - if res: - fullname = res.group(1) - docs[fullname.split('.')[-1]] = eval(res.group(2)).decode('utf-8') - -translate_name = { - 'name': None, - 'delete_': 'delete', - 'union_': 'union', - 'operator[]': '__getitem__', - 'operator+': '__add__', - 'operator-': '__sub__', - 'operator*': '__mul__', - 'operator/': '__div__', - 'operator%': '__mod__', - 'operator&&': 'and_', - 'operator||': 'or_', - 'operator==': '__eq__', - 'operator!=': '__ne__', - 'operator>': '__gt__', - 'operator>=': '__ge__', - 'operator<': '__lt__', - 'operator<=': '__le__', - 'operator!': 'not_', - 'default_': 'default', - 'array': None, - 'desc': None, - 'asc': None, - 'maxval': None, - 'minval': None, - 'january': None, - 'february': None, - 'march': None, - 'april': None, - 'may': None, - 'june': None, - 'july': None, - 'august': None, - 'september': None, - 'october': None, - 'november': None, - 'december': None, - 'monday': None, - 'tuesday': None, - 'wednesday': None, - 'thursday': None, - 'friday': None, - 'saturday': None, - 'sunday': None, -} - -def print_docs(name, line): - py_name = translate_name.get(name, name) - if py_name in docs: - indent = match("^( *)", line).group(1) - stdout.write('\n') - # TODO: convert the examples to C++ - for line in docs[py_name].split('\n'): - stdout.write(indent + "// " + line + '\n') - elif py_name: - stderr.write('Warning: no docs for ' + py_name + ': ' + line) - -stdout.write('// Contains documentation copied as-is from the Python driver') - -for line in stdin: - res = match("^ *CO?[0-9_]+\(([^,)]+)|extern Query (\w+)|^ *// *(\$)doc\((\w+)\) *$", line) - if res: - name = res.group(1) or res.group(2) or res.group(4) - print_docs(name, line) - if not res.group(3): - stdout.write(line) - else: - stdout.write(line) - diff --git a/ext/librethinkdbxx/reql/gen.py b/ext/librethinkdbxx/reql/gen.py deleted file mode 100644 index 2b1fe9fc..00000000 --- a/ext/librethinkdbxx/reql/gen.py +++ /dev/null @@ -1,33 +0,0 @@ -from sys import argv -from re import sub, finditer, VERBOSE - -def gen(defs): - indent = 0 - enum = False - def p(s): print(" " * (indent * 4) + s) - for item in finditer(""" - (?P<type> message|enum) \\s+ (?P<name> \\w+) \\s* \\{ | - (?P<var> \\w+) \\s* = \\s* (?P<val> \\w+) \\s* ; | - \\} - """, defs, flags=VERBOSE): - if item.group(0) == "}": - indent = indent - 1 - p("};" if enum else "}") - enum = False; - elif item.group('type') == 'enum': - p("enum class %s {" % item.group('name')) - indent = indent + 1 - enum = True - elif item.group('type') == 'message': - p("namespace %s {" % item.group('name')) - indent = indent + 1 - enum = False - else: - if enum: - p("%s = %s," % (item.group('var'), item.group('val'))) - -print("// Auto-generated by reql/gen.py") -print("#pragma once") -print("namespace RethinkDB { namespace Protocol {") -gen(sub("//.*", "", open(argv[1]).read())) -print("} }") diff --git a/ext/librethinkdbxx/reql/python_docs.txt b/ext/librethinkdbxx/reql/python_docs.txt deleted file mode 100644 index 41a5c84e..00000000 --- a/ext/librethinkdbxx/reql/python_docs.txt +++ /dev/null @@ -1,189 +0,0 @@ -# This file was generated by _scripts/gen_python.py from the rethinkdb documentation in http://github.com/rethinkdb/docs -# hash: "3d13a937cfdacb7ffa3dab2ce3ebdf25bd3c192e" - -import rethinkdb - -docsSource = [ - - (rethinkdb.net.Connection.close, b'conn.close(noreply_wait=True)\n\nClose an open connection.\n\nClosing a connection normally waits until all outstanding requests have finished and then frees any open resources associated with the connection. By passing `False` to the `noreply_wait` optional argument, the connection will be closed immediately, possibly aborting any outstanding noreply writes.\n\nA noreply query is executed by passing the `noreply` option to the [run](http://rethinkdb.com/api/python/run/) command, indicating that `run()` should not wait for the query to complete before returning. You may also explicitly wait for a noreply query to complete by using the [noreply_wait](http://rethinkdb.com/api/python/noreply_wait) command.\n\n*Example* Close an open connection, waiting for noreply writes to finish.\n\n conn.close()\n\n*Example* Close an open connection immediately.\n\n conn.close(noreply_wait=False)\n'), - (rethinkdb.connect, b'r.connect(host="localhost", port=28015, db="test", auth_key="", timeout=20) -> connection\nr.connect(host) -> connection\n\nCreate a new connection to the database server. The keyword arguments are:\n\n- `host`: host of the RethinkDB instance. The default value is `localhost`.\n- `port`: the driver port, by default `28015`.\n- `db`: the database used if not explicitly specified in a query, by default `test`.\n- `auth_key`: the authentication key, by default the empty string.\n- `timeout`: timeout period in seconds for the connection to be opened (default `20`).\n\nIf the connection cannot be established, a `RqlDriverError` exception will be thrown.\n\nThe authentication key can be set from the RethinkDB command line tool. Once set, client connections must provide the key as an option to `run` in order to make the connection. For more information, read "Using the RethinkDB authentication system" in the documentation on [securing your cluster](http://rethinkdb.com/docs/security/).\n\n__Note:__ Currently, the Python driver is not thread-safe. Each thread or multiprocessing PID should be given its own connection object. (This is likely to change in a future release of RethinkDB; you can track issue [#2427](https://github.com/rethinkdb/rethinkdb/issues/2427) for progress.)\n\n*Example* Opens a connection using the default host and port but specifying the default database.\n\n conn = r.connect(db=\'marvel\')\n\n*Example* Opens a new connection to the database.\n\n conn = r.connect(host = \'localhost\',\n port = 28015,\n db = \'heroes\',\n auth_key = \'hunter2\')\n\n'), - (rethinkdb.net.Connection.noreply_wait, b'conn.noreply_wait()\n\n`noreply_wait` ensures that previous queries with the `noreply` flag have been processed\nby the server. Note that this guarantee only applies to queries run on the given connection.\n\n*Example* We have previously run queries with the `noreply` argument set to `True`. Now\nwait until the server has processed them.\n\n conn.noreply_wait()\n\n'), - (rethinkdb, b'r -> r\n\nThe top-level ReQL namespace.\n\n*Example* Setup your top-level namespace.\n\n import rethinkdb as r\n\n'), - (rethinkdb.net.Connection.reconnect, b'conn.reconnect(noreply_wait=True)\n\nClose and reopen a connection.\n\nClosing a connection normally waits until all outstanding requests have finished and then frees any open resources associated with the connection. By passing `False` to the `noreply_wait` optional argument, the connection will be closed immediately, possibly aborting any outstanding noreply writes.\n\nA noreply query is executed by passing the `noreply` option to the [run](http://rethinkdb.com/api/python/run/) command, indicating that `run()` should not wait for the query to complete before returning. You may also explicitly wait for a noreply query to complete by using the [noreply_wait](http://rethinkdb.com/api/python/noreply_wait) command.\n\n*Example* Cancel outstanding requests/queries that are no longer needed.\n\n conn.reconnect(noreply_wait=False)\n'), - (rethinkdb.net.Connection.repl, b"conn.repl()\n\nSet the default connection to make REPL use easier. Allows calling\n`.run()` on queries without specifying a connection.\n\n__Note:__ Avoid using `repl` in application code. RethinkDB connection objects are not thread-safe, and calls to `connect` from multiple threads may change the global connection object used by `repl`. Applications should specify connections explicitly.\n\n*Example* Set the default connection for the REPL, then call\n`run()` without specifying the connection.\n\n r.connect(db='marvel').repl()\n r.table('heroes').run()\n"), - (rethinkdb.ast.RqlQuery.run, b'query.run(conn, use_outdated=False, time_format=\'native\', profile=False, durability="hard") -> cursor\nquery.run(conn, use_outdated=False, time_format=\'native\', profile=False, durability="hard") -> object\n\nRun a query on a connection, returning either a single JSON result or\na cursor, depending on the query.\n\nThe optional arguments are:\n\n- `use_outdated`: whether or not outdated reads are OK (default: `False`).\n- `time_format`: what format to return times in (default: `\'native\'`).\n Set this to `\'raw\'` if you want times returned as JSON objects for exporting.\n- `profile`: whether or not to return a profile of the query\'s\n execution (default: `False`).\n- `durability`: possible values are `\'hard\'` and `\'soft\'`. In soft durability mode RethinkDB\nwill acknowledge the write immediately after receiving it, but before the write has\nbeen committed to disk.\n- `group_format`: what format to return `grouped_data` and `grouped_streams` in (default: `\'native\'`).\n Set this to `\'raw\'` if you want the raw pseudotype.\n- `noreply`: set to `True` to not receive the result object or cursor and return immediately.\n- `db`: the database to run this query against as a string. The default is the database specified in the `db` parameter to [connect](http://rethinkdb.com/api/python/connect/) (which defaults to `test`). The database may also be specified with the [db](http://rethinkdb.com/api/python/db/) command.\n- `array_limit`: the maximum numbers of array elements that can be returned by a query (default: 100,000). This affects all ReQL commands that return arrays. Note that it has no effect on the size of arrays being _written_ to the database; those always have an upper limit of 100,000 elements.\n- `binary_format`: what format to return binary data in (default: `\'native\'`). Set this to `\'raw\'` if you want the raw pseudotype.\n- `min_batch_rows`: minimum number of rows to wait for before batching a result set (default: 8). This is an integer.\n- `max_batch_rows`: maximum number of rows to wait for before batching a result set (default: unlimited). This is an integer.\n- `max_batch_bytes`: maximum number of bytes to wait for before batching a result set (default: 1024). This is an integer.\n- `max_batch_seconds`: maximum number of seconds to wait before batching a result set (default: 0.5). This is a float (not an integer) and may be specified to the microsecond.\n- `first_batch_scaledown_factor`: factor to scale the other parameters down by on the first batch (default: 4). For example, with this set to 8 and `max_batch_rows` set to 80, on the first batch `max_batch_rows` will be adjusted to 10 (80 / 8). This allows the first batch to return faster.\n\n*Example* Run a query on the connection `conn` and print out every\nrow in the result.\n\n for doc in r.table(\'marvel\').run(conn):\n print doc\n\n*Example* If you are OK with potentially out of date data from all\nthe tables involved in this query and want potentially faster reads,\npass a flag allowing out of date data in an options object. Settings\nfor individual tables will supercede this global setting for all\ntables in the query.\n\n r.table(\'marvel\').run(conn, use_outdated=True)\n\n*Example* If you just want to send a write and forget about it, you\ncan set `noreply` to true in the options. In this case `run` will\nreturn immediately.\n\n r.table(\'marvel\').run(conn, noreply=True)\n\n*Example* If you want to specify whether to wait for a write to be\nwritten to disk (overriding the table\'s default settings), you can set\n`durability` to `\'hard\'` or `\'soft\'` in the options.\n\n r.table(\'marvel\')\n .insert({ \'superhero\': \'Iron Man\', \'superpower\': \'Arc Reactor\' })\n .run(conn, noreply=True, durability=\'soft\')\n\n*Example* If you do not want a time object to be converted to a\nnative date object, you can pass a `time_format` flag to prevent it\n(valid flags are "raw" and "native"). This query returns an object\nwith two fields (`epoch_time` and `$reql_type$`) instead of a native date\nobject.\n\n r.now().run(conn, time_format="raw")\n\n*Example* Specify the database to use for the query.\n\n for doc in r.table(\'marvel\').run(conn, db=\'heroes\'):\n print doc\n\nThis is equivalent to using the `db` command to specify the database:\n\n r.db(\'heroes\').table(\'marvel\').run(conn) ...\n\n*Example* Change the batching parameters for this query.\n\n r.table(\'marvel\').run(conn, max_batch_rows=16, max_batch_bytes=2048)\n'), - (rethinkdb.set_loop_type, b'r.set_loop_type(string)\n\nSet an asynchronous event loop model. Currently, the only event loop model RethinkDB supports is `"tornado"`, for use with the [Tornado web framework](http://www.tornadoweb.org). After setting the event loop to `"tornado"`, the [connect](http://rethinkdb.com/api/python/connect) and [run](http://rethinkdb.com/api/python/run) commands will return Tornado `Future` objects.\n\n*Example* Read a table\'s data using Tornado.\n\n r.set_loop_type("tornado")\n conn = r.connect(host=\'localhost\', port=28015)\n \n @gen.coroutine\n def use_cursor(conn):\n # Print every row in the table.\n cursor = yield r.table(\'test\').order_by(index="id").run(yield conn)\n while (yield cursor.fetch_next()):\n item = yield cursor.next()\n print(item)\n\nFor a longer discussion with Tornado examples, see the documentation article on [Asynchronous connections][ac].\n\n[ac]: /docs/async-connections/\n'), - (rethinkdb.net.Connection.use, b"conn.use(db_name)\n\nChange the default database on this connection.\n\n*Example* Change the default database so that we don't need to\nspecify the database when referencing a table.\n\n conn.use('marvel')\n r.table('heroes').run(conn) # refers to r.db('marvel').table('heroes')\n"), - (rethinkdb.ast.Table.config, b'table.config() -> selection<object>\ndatabase.config() -> selection<object>\n\nQuery (read and/or update) the configurations for individual tables or databases.\n\nThe `config` command is a shorthand way to access the `table_config` or `db_config` [System tables](http://rethinkdb.com/docs/system-tables/). It will return the single row from the system that corresponds to the database or table configuration, as if [get](http://rethinkdb.com/api/python/get) had been called on the system table with the UUID of the database or table in question.\n\n*Example* Get the configuration for the `users` table.\n\n r.table(\'users\').config().run(conn)\n \n {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "users",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "a", "replicas": ["a", "b"]},\n {"primary_replica": "d", "replicas": ["c", "d"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n\n*Example* Change the write acknowledgement requirement of the `users` table.\n\n r.table(\'users\').config().update({\'write_acks\': \'single\'}).run(conn)\n'), - (rethinkdb.ast.Table.rebalance, b'table.rebalance() -> object\ndatabase.rebalance() -> object\n\nRebalances the shards of a table. When called on a database, all the tables in that database will be rebalanced.\n\nThe `rebalance` command operates by measuring the distribution of primary keys within a table and picking split points that will give each shard approximately the same number of documents. It won\'t change the number of shards within a table, or change any other configuration aspect for the table or the database.\n\nA table will lose availability temporarily after `rebalance` is called; use the [wait](http://rethinkdb.com/api/python/wait) command to wait for the table to become available again, or [status](http://rethinkdb.com/api/python/status) to check if the table is available for writing.\n\nRethinkDB automatically rebalances tables when the number of shards are increased, and as long as your documents have evenly distributed primary keys—such as the default UUIDs—it is rarely necessary to call `rebalance` manually. Cases where `rebalance` may need to be called include:\n\n* Tables with unevenly distributed primary keys, such as incrementing integers\n* Changing a table\'s primary key type\n* Increasing the number of shards on an empty table, then using non-UUID primary keys in that table\n\nThe [web UI](http://rethinkdb.com/docs/administration-tools/) (and the [info](http://rethinkdb.com/api/python/info) command) can be used to tell you when a table\'s shards need to be rebalanced.\n\nThe return value of `rebalance` is an object with two fields:\n\n* `rebalanced`: the number of tables rebalanced.\n* `status_changes`: a list of new and old table status values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `rebalance` was executed. \n * `new_val`: The table\'s `status` value after `rebalance` was executed. (This value will almost always indicate the table is unavailable.)\n\nSee the [status](http://rethinkdb.com/api/python/status) command for an explanation of the objects returned in the `old_val` and `new_val` fields.\n\n*Example* Rebalance a table.\n\n r.table(\'superheroes\').rebalance().run(conn)\n \n {\n "rebalanced": 1,\n "status_changes": [\n {\n "old_val": {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": True,\n "ready_for_outdated_reads": True,\n "ready_for_reads": True,\n "ready_for_writes": True\n }\n },\n "new_val": {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "transitioning"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "transitioning"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": False,\n "ready_for_outdated_reads": False,\n "ready_for_reads": False,\n "ready_for_writes": False\n }\n }\n \n }\n ]\n }\n'), - (rethinkdb.ast.Table.reconfigure, b'table.reconfigure(shards=<s>, replicas=<r>[, primary_replica_tag=<t>, dry_run=False]) -> object\ndatabase.reconfigure(shards=<s>, replicas=<r>[, primary_replica_tag=<t>, dry_run=False]) -> object\n\nReconfigure a table\'s sharding and replication.\n\n* `shards`: the number of shards, an integer from 1-32. Required.\n* `replicas`: either an integer or a mapping object. Required.\n * If `replicas` is an integer, it specifies the number of replicas per shard. Specifying more replicas than there are servers will return an error.\n * If `replicas` is an object, it specifies key-value pairs of server tags and the number of replicas to assign to those servers: `{"tag1": 2, "tag2": 4, "tag3": 2, ...}`. For more information about server tags, read [Administration tools](http://rethinkdb.com/docs/administration-tools/).\n* `primary_replica_tag`: the primary server specified by its server tag. Required if `replicas` is an object; the tag must be in the object. This must *not* be specified if `replicas` is an integer.\n* `dry_run`: if `True` the generated configuration will not be applied to the table, only returned.\n\nThe return value of `reconfigure` is an object with three fields:\n\n* `reconfigured`: the number of tables reconfigured. This will be `0` if `dry_run` is `True`.\n* `config_changes`: a list of new and old table configuration values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [config](http://rethinkdb.com/api/python/config) value before `reconfigure` was executed. \n * `new_val`: The table\'s `config` value after `reconfigure` was executed.\n* `status_changes`: a list of new and old table status values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `reconfigure` was executed. \n * `new_val`: The table\'s `status` value after `reconfigure` was executed.\n\nFor `config_changes` and `status_changes`, see the [config](http://rethinkdb.com/api/python/config) and [status](http://rethinkdb.com/api/python/status) commands for an explanation of the objects returned in the `old_val` and `new_val` fields.\n\nA table will lose availability temporarily after `reconfigure` is called; use the [table_status](http://rethinkdb.com/api/python/table_status) command to determine when the table is available again.\n\n**Note:** Whenever you call `reconfigure`, the write durability will be set to `hard` and the write acknowledgments will be set to `majority`; these can be changed by using the `config` command on the table.\n\nIf `reconfigure` is called on a database, all the tables in the database will have their configurations affected. The return value will be an array of the objects described above, one per table.\n\nRead [Sharding and replication](http://rethinkdb.com/docs/sharding-and-replication/) for a complete discussion of the subject, including advanced topics.\n\n*Example* Reconfigure a table.\n\n r.table(\'superheroes\').reconfigure(shards=2, replicas=1).run(conn)\n \n {\n "reconfigured": 1,\n "config_changes": [\n {\n "new_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "jeeves", "replicas": ["jeeves"]},\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n },\n "old_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n }\n ],\n "status_changes": [\n {\n "new_val": (status object),\n "old_val": (status object)\n }\n ]\n }\n\n*Example* Reconfigure a table, specifying replicas by server tags.\n\n r.table(\'superheroes\').reconfigure(shards=2, replicas={\'wooster\': 1, \'wayne\': 1}, primary_replica_tag=\'wooster\').run(conn)\n \n {\n "reconfigured": 1,\n "config_changes": [\n {\n "new_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "jeeves", "replicas": ["jeeves", "alfred"]},\n {"primary_replica": "jeeves", "replicas": ["jeeves", "alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n },\n "old_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n }\n ],\n "status_changes": [\n {\n "new_val": (status object),\n "old_val": (status object)\n }\n ]\n }\n'), - (rethinkdb.ast.Table.status, b'table.status() -> selection<object>\n\nReturn the status of a table.\n\nThe return value is an object providing information about the table\'s shards, replicas and replica readiness states. For a more complete discussion of the object fields, read about the `table_status` table in [System tables](http://rethinkdb.com/docs/system-tables/).\n\n* `db`: database name.\n* `name`: table name.\n* `id`: table UUID.\n* `shards`: an array of objects, one for each shard, with the following keys per object:\n * `primary_replica`: name of the shard\'s primary server.\n * `replicas`: an array of objects showing the status of each replica, with the following keys:\n * `server`: name of the replica server.\n * `state`: one of `ready`, `disconnected`, `backfilling_data`, `offloading_data`, `erasing_data`, `looking_for_primary_replica` or `transitioning`.\n* `status`: an object with the following boolean keys:\n * `all_replicas_ready`: `True` if all backfills have finished.\n * `ready_for_outdated_reads`: `True` if the table is ready for read queries with the `use_outdated` flag set to `True`.\n * `ready_for_reads`: `True` if the table is ready for read queries with current data (with the `use_outdated` flag set to `False` or unspecified).\n * `ready_for_writes`: `True` if the table is ready for write queries.\n\n*Example* Get a table\'s status.\n\n r.table(\'superheroes\').status().run(conn)\n \n {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": True,\n "ready_for_outdated_reads": True,\n "ready_for_reads": True,\n "ready_for_writes": True\n }\n }\n'), - (rethinkdb.ast.Table.wait, b'table.wait([wait_for=\'ready_for_writes\', timeout=<sec>]) -> object\ndatabase.wait([wait_for=\'ready_for_writes\', timeout=<sec>]) -> object\nr.wait([wait_for=\'ready_for_writes\', timeout=<sec>]) -> object\n\nWait for a table or all the tables in a database to be ready. A table may be temporarily unavailable after creation, rebalancing or reconfiguring. The `wait` command blocks until the given table (or database) is fully up to date.\n\nThe `wait` command takes two optional arguments:\n\n* `wait_for`: a string indicating a table [status](http://rethinkdb.com/api/python/status) to wait on before returning, one of `ready_for_outdated_reads`, `ready_for_reads`, `ready_for_writes`, or `all_replicas_ready`. The default is `ready_for_writes`. \n* `timeout`: a number indicating maximum time to wait for in seconds before returning. The default is no timeout.\n\nThe return value is an object consisting of two key/value pairs:\n\n* `ready`: an integer indicating the number of tables waited for. It will always be `1` when `wait` is called on a table, and the total number of tables when called on a database.\n* `status_changes`: a list with one entry for each of the tables. Each member of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `wait` was executed. \n * `new_val`: The table\'s `status` value after `wait` finished.\n\nSee [status](http://rethinkdb.com/api/python/status) and [System tables](http://rethinkdb.com/docs/system-tables/) for a description of the fields within `status_changes`.\n\nIf `wait` is called with no table or database specified (the `r.wait()` form), it will wait on all the tables in the default database (set with the [connect](http://rethinkdb.com/api/python/connect/) command\'s `db` parameter, which defaults to `test`).\n\n*Example* Wait on a table to be ready.\n\n r.table(\'superheroes\').wait().run(conn)\n \n {\n "ready": 1,\n "status_changes": [\n \t{\n \t "old_val": {\n \t\t"db": "database",\n \t\t"id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n \t\t"name": "superheroes",\n \t\t"shards": [\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t },\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t }\n \t\t],\n \t\t"status": {\n \t\t "all_replicas_ready": True,\n \t\t "ready_for_outdated_reads": True,\n \t\t "ready_for_reads": True,\n \t\t "ready_for_writes": True\n \t\t}\n \t },\n \t "new_val": {\n \t\t"db": "database",\n \t\t"id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n \t\t"name": "superheroes",\n \t\t"shards": [\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t },\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t }\n \t\t],\n \t\t"status": {\n \t\t "all_replicas_ready": True,\n \t\t "ready_for_outdated_reads": True,\n \t\t "ready_for_reads": True,\n \t\t "ready_for_writes": True\n \t\t}\n \t }\n \t}\n ]\n }\n'), - (rethinkdb.ast.RqlQuery.avg, b"sequence.avg([field_or_function]) -> number\n\nAverages all the elements of a sequence. If called with a field name,\naverages all the values of that field in the sequence, skipping\nelements of the sequence that lack that field. If called with a\nfunction, calls that function on every element of the sequence and\naverages the results, skipping elements of the sequence where that\nfunction returns `None` or a non-existence error.\n\nProduces a non-existence error when called on an empty sequence. You\ncan handle this case with `default`.\n\n*Example* What's the average of 3, 5, and 7?\n\n r.expr([3, 5, 7]).avg().run(conn)\n\n*Example* What's the average number of points scored in a game?\n\n r.table('games').avg('points').run(conn)\n\n*Example* What's the average number of points scored in a game,\ncounting bonus points?\n\n r.table('games').avg(lambda game:\n game['points'] + game['bonus_points']\n ).run(conn)\n\n*Example* What's the average number of points scored in a game?\n(But return `None` instead of raising an error if there are no games where\npoints have been scored.)\n\n r.table('games').avg('points').default(None).run(conn)\n"), - (rethinkdb.ast.RqlQuery.contains, b"sequence.contains(value|predicate[, value|predicate, ...]) -> bool\n\nWhen called with values, returns `True` if a sequence contains all the\nspecified values. When called with predicate functions, returns `True`\nif for each predicate there exists at least one element of the stream\nwhere that predicate returns `True`.\n\nValues and predicates may be mixed freely in the argument list.\n\n*Example* Has Iron Man ever fought Superman?\n\n r.table('marvel').get('ironman')['opponents'].contains('superman').run(conn)\n\n*Example* Has Iron Man ever defeated Superman in battle?\n\n r.table('marvel').get('ironman')['battles'].contains(lambda battle:\n (battle['winner'] == 'ironman') & (battle['loser'] == 'superman')\n ).run(conn)\n\n*Example* Use `contains` with a predicate function to simulate an `or`. Return the Marvel superheroes who live in Detroit, Chicago or Hoboken.\n\n r.table('marvel').filter(\n lambda hero: r.expr(['Detroit', 'Chicago', 'Hoboken']).contains(hero['city'])\n ).run(conn)\n"), - (rethinkdb.ast.RqlQuery.count, b"sequence.count([value_or_predicate]) -> number\nbinary.count() -> number\n\nCounts the number of elements in a sequence. If called with a value,\ncounts the number of times that value occurs in the sequence. If\ncalled with a predicate function, counts the number of elements in the\nsequence where that function returns `True`.\n\nIf `count` is called on a [binary](http://rethinkdb.com/api/python/binary) object, it will return the size of the object in bytes.\n\n*Example* Count the number of users.\n\n r.table('users').count().run(conn)\n\n*Example* Count the number of 18 year old users.\n\n r.table('users')['age'].count(18).run(conn)\n\n*Example* Count the number of users over 18.\n\n r.table('users')['age'].count(lambda age: age > 18).run(conn)\n\n r.table('users').count(lambda user: user['age'] > 18).run(conn)\n"), - (rethinkdb.ast.RqlQuery.distinct, b"sequence.distinct() -> array\ntable.distinct([index=<indexname>]) -> stream\n\nRemoves duplicate elements from a sequence.\n\nThe `distinct` command can be called on any sequence or table with an index.\n\n{% infobox %}\nWhile `distinct` can be called on a table without an index, the only effect will be to convert the table into a stream; the content of the stream will not be affected.\n{% endinfobox %}\n\n*Example* Which unique villains have been vanquished by Marvel heroes?\n\n r.table('marvel').concat_map(\n lambda hero: hero['villain_list']).distinct().run(conn)\n\n*Example* Topics in a table of messages have a secondary index on them, and more than one message can have the same topic. What are the unique topics in the table?\n\n r.table('messages').distinct(index='topics').run(conn)\n\nThe above structure is functionally identical to:\n\n r.table('messages')['topics'].distinct().run(conn)\n\nHowever, the first form (passing the index as an argument to `distinct`) is faster, and won't run into array limit issues since it's returning a stream.\n"), - (rethinkdb.ast.RqlQuery.group, b'sequence.group(field_or_function..., [index=\'index_name\', multi=False]) -> grouped_stream\n\nTakes a stream and partitions it into multiple groups based on the\nfields or functions provided.\n\nWith the `multi` flag single documents can be assigned to multiple groups, similar to the behavior of [multi-indexes](http://rethinkdb.com/docs/secondary-indexes/python). When `multi` is `True` and the grouping value is an array, documents will be placed in each group that corresponds to the elements of the array. If the array is empty the row will be ignored.\n\n*Example* Grouping games by player.\n\nSuppose that the table `games` has the following data:\n\n [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ]\n\nGrouping games by player can be done with:\n\n > r.table(\'games\').group(\'player\').run(conn)\n \n {\n "Alice": [\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ],\n "Bob": [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n\nCommands chained after `group` will be called on each of these grouped\nsub-streams, producing grouped data.\n\n*Example* What is each player\'s best game?\n\n > r.table(\'games\').group(\'player\').max(\'points\').run(conn)\n \n {\n "Alice": {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n "Bob": {"id": 2, "player": "Bob", "points": 15, "type": "ranked"}\n }\n\nCommands chained onto grouped data will operate on each grouped datum,\nproducing more grouped data.\n\n*Example* What is the maximum number of points scored by each player?\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].run(conn)\n \n {\n "Alice": 7,\n "Bob": 15\n }\n\nYou can also group by more than one field.\n\n*Example* What is the maximum number of points scored by each\nplayer for each game type?\n\n > r.table(\'games\').group(\'player\', \'type\').max(\'points\')[\'points\'].run(conn)\n \n {\n ("Alice", "free"): 7,\n ("Bob", "free"): 10,\n ("Bob", "ranked"): 15\n }\n\nYou can also group by a function.\n\n*Example* What is the maximum number of points scored by each\nplayer for each game type?\n\n > r.table(\'games\')\n .group(lambda game:\n game.pluck(\'player\', \'type\')\n ).max(\'points\')[\'points\'].run(conn)\n \n {\n frozenset([(\'player\', \'Alice\'), (\'type\', \'free\')]): 7,\n frozenset([(\'player\', \'Bob\'), (\'type\', \'free\')]): 10,\n frozenset([(\'player\', \'Bob\'), (\'type\', \'ranked\')]): 15,\n }\n\nUsing a function, you can also group by date on a ReQL [date field](http://rethinkdb.com/docs/dates-and-times/javascript/).\n\n*Example* How many matches have been played this year by month?\n\n > r.table(\'matches\').group(\n lambda match: [match[\'date\'].year(), match[\'date\'].month()]\n ).count().run(conn)\n \n {\n (2014, 2): 2,\n (2014, 3): 2,\n (2014, 4): 1,\n (2014, 5): 3\n }\n\nYou can also group on an index (primary key or secondary).\n\n*Example* What is the maximum number of points scored by game type?\n\n > r.table(\'games\').group(index=\'type\').max(\'points\')[\'points\'].run(conn)\n \n {\n "free": 10,\n "ranked": 15\n }\n\nSuppose that the table `games2` has the following data:\n\n [\n { \'id\': 1, \'matches\': {\'a\': [1, 2, 3], \'b\': [4, 5, 6]} },\n { \'id\': 2, \'matches\': {\'b\': [100], \'c\': [7, 8, 9]} },\n { \'id\': 3, \'matches\': {\'a\': [10, 20], \'c\': [70, 80]} }\n ]\n\nUsing the `multi` option we can group data by match A, B or C.\n\n > r.table(\'games2\').group(r.row[\'matches\'].keys(), multi=True).run(conn)\n \n [\n {\n \'group\': \'a\',\n \'reduction\': [ <id 1>, <id 3> ]\n },\n {\n \'group\': \'b\',\n \'reduction\': [ <id 1>, <id 2> ]\n },\n {\n \'group\': \'c\',\n \'reduction\': [ <id 2>, <id 3> ]\n }\n ]\n\n(The full result set is abbreviated in the figure; `<id 1>, <id 2>` and `<id 3>` would be the entire documents matching those keys.)\n\n*Example* Use [map](http://rethinkdb.com/api/python/map) and [sum](http://rethinkdb.com/api/python/sum) to get the total points scored for each match.\n\n r.table(\'games2\').group(r.row[\'matches\'].keys(), multi=True).ungroup().map(\n lambda doc: { \'match\': doc[\'group\'], \'total\': doc[\'reduction\'].sum(\n lambda set: set[\'matches\'][doc[\'group\']].sum()\n )}).run(conn)\n \n [\n { \'match\': \'a\', \'total\': 36 },\n { \'match\': \'b\', \'total\': 115 },\n { \'match\': \'c\', \'total\': 174 }\n ]\n\nThe inner `sum` adds the scores by match within each document; the outer `sum` adds those results together for a total across all the documents.\n\nIf you want to operate on all the groups rather than operating on each\ngroup (e.g. if you want to order the groups by their reduction), you\ncan use [ungroup](http://rethinkdb.com/api/python/ungroup/) to turn a grouped stream or\ngrouped data into an array of objects representing the groups.\n\n*Example* Ungrouping grouped data.\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].ungroup().run(conn)\n \n [\n {\n "group": "Alice",\n "reduction": 7\n },\n {\n "group": "Bob",\n "reduction": 15\n }\n ]\n\nUngrouping is useful e.g. for ordering grouped data, or for inserting\ngrouped data into a table.\n\n*Example* What is the maximum number of points scored by each\nplayer, with the highest scorers first?\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].ungroup().order_by(\n r.desc(\'reduction\')).run(conn)\n \n [\n {\n "group": "Bob",\n "reduction": 15\n },\n {\n "group": "Alice",\n "reduction": 7\n }\n ]\n\nWhen grouped data are returned to the client, they are transformed\ninto a client-specific native type. (Something similar is done with\n[times](http://rethinkdb.com/docs/dates-and-times/).) In Python, grouped data are\ntransformed into a `dictionary`. If the group value is an `array`, the\nkey is converted to a `tuple`. If the group value is a `dictionary`,\nit will be converted to a `frozenset`.\n\nIf you instead want to receive the raw\npseudotype from the server (e.g. if you\'re planning to serialize the\nresult as JSON), you can specify `group_format: \'raw\'` as an optional\nargument to `run`:\n\n*Example* Get back the raw `GROUPED_DATA` pseudotype.\n\n > r.table(\'games\').group(\'player\').avg(\'points\').run(conn, group_format=\'raw\')\n \n {\n "$reql_type$": "GROUPED_DATA",\n "data": [\n ["Alice", 4.5],\n ["Bob", 12.5]\n ]\n }\n\nNot passing the `group_format` flag would return:\n\n {\n "Alice": 4.5,\n "Bob": 12.5\n }\n\nYou might also want to use the [ungroup](http://rethinkdb.com/api/python/ungroup/)\ncommand (see above), which will turn the grouped data into an array of\nobjects on the server.\n\nIf you run a query that returns a grouped stream, it will be\nautomatically converted to grouped data before being sent back to you\n(there is currently no efficient way to stream groups from RethinkDB).\nThis grouped data is subject to the array size limit (see [run](http://rethinkdb.com/api/python/run)).\n\nIn general, operations on grouped streams will be efficiently\ndistributed, and operations on grouped data won\'t be. You can figure\nout what you\'re working with by putting `type_of` on the end of your\nquery. Below are efficient and inefficient examples.\n\n*Example* Efficient operation.\n\n # r.table(\'games\').group(\'player\').type_of().run(conn)\n # Returns "GROUPED_STREAM"\n r.table(\'games\').group(\'player\').min(\'points\').run(conn) # EFFICIENT\n\n*Example* Inefficient operation.\n\n # r.table(\'games\').group(\'player\').order_by(\'score\').type_of().run(conn)\n # Returns "GROUPED_DATA"\n r.table(\'games\').group(\'player\').order_by(\'score\').nth(0).run(conn) # INEFFICIENT\n\nWhat does it mean to be inefficient here? When operating on grouped\ndata rather than a grouped stream, *all* of the data has to be\navailable on the node processing the query. This means that the\noperation will only use one server\'s resources, and will require\nmemory proportional to the size of the grouped data it\'s operating\non. (In the case of the `order_by` in the inefficient example, that\nmeans memory proportional **to the size of the table**.) The array\nlimit is also enforced for grouped data, so the `order_by` example\nwould fail for tables with more than 100,000 rows unless you used the `array_limit` option with `run`.\n\n*Example* What is the maximum number of points scored by each\nplayer in free games?\n\n > r.table(\'games\').filter(lambda game:\n game[\'type\'] = \'free\'\n ).group(\'player\').max(\'points\')[\'points\'].run(conn)\n \n {\n "Alice": 7,\n "Bob": 10\n }\n\n*Example* What is each player\'s highest even and odd score?\n\n > r.table(\'games\')\n .group(\'name\', lambda game:\n game[\'points\'] % 2\n ).max(\'points\')[\'points\'].run(conn)\n \n {\n ("Alice", 1): 7,\n ("Bob", 0): 10,\n ("Bob", 1): 15\n }\n'), - (rethinkdb.ast.RqlQuery.max, b"sequence.max(field_or_function) -> element\nsequence.max(index='index') -> element\n\nFinds the maximum element of a sequence. The `max` command can be called with:\n\n* a **field name**, to return the element of the sequence with the largest value in that field;\n* an **index** (the primary key or a secondary index), to return the element of the sequence with the largest value in that index;\n* a **function**, to apply the function to every element within the sequence and return the element which returns the largest value from the function, ignoring any elements where the function produces a non-existence error.\n\nFor more information on RethinkDB's sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nCalling `max` on an empty sequence will throw a non-existence error; this can be handled using the [default](http://rethinkdb.com/api/python/default/) command.\n\n*Example* Return the maximum value in the list `[3, 5, 7]`.\n\n r.expr([3, 5, 7]).max().run(conn)\n\n*Example* Return the user who has scored the most points.\n\n r.table('users').max('points').run(conn)\n\n*Example* The same as above, but using a secondary index on the `points` field.\n\n r.table('users').max(index='points').run(conn)\n\n*Example* Return the user who has scored the most points, adding in bonus points from a separate field using a function.\n\n r.table('users').max(lambda user:\n user['points'] + user['bonus_points']\n ).run(conn)\n\n*Example* Return the highest number of points any user has ever scored. This returns the value of that `points` field, not a document.\n\n r.table('users').max('points')['points'].run(conn)\n\n*Example* Return the user who has scored the most points, but add a default `None` return value to prevent an error if no user has ever scored points.\n\n r.table('users').max('points').default(None).run(conn)\n"), - (rethinkdb.ast.RqlQuery.min, b"sequence.min(field_or_function) -> element\nsequence.min(index='index') -> element\n\nFinds the minimum element of a sequence. The `min` command can be called with:\n\n* a **field name**, to return the element of the sequence with the smallest value in that field;\n* an **index** (the primary key or a secondary index), to return the element of the sequence with the smallest value in that index;\n* a **function**, to apply the function to every element within the sequence and return the element which returns the smallest value from the function, ignoring any elements where the function produces a non-existence error.\n\nFor more information on RethinkDB's sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nCalling `min` on an empty sequence will throw a non-existence error; this can be handled using the [default](http://rethinkdb.com/api/python/default/) command.\n\n*Example* Return the minimum value in the list `[3, 5, 7]`.\n\n r.expr([3, 5, 7]).min().run(conn)\n\n*Example* Return the user who has scored the fewest points.\n\n r.table('users').min('points').run(conn)\n\n*Example* The same as above, but using a secondary index on the `points` field.\n\n r.table('users').min(index='points').run(conn)\n\n*Example* Return the user who has scored the fewest points, adding in bonus points from a separate field using a function.\n\n r.table('users').min(lambda user:\n user['points'] + user['bonus_points']\n ).run(conn)\n\n*Example* Return the smallest number of points any user has ever scored. This returns the value of that `points` field, not a document.\n\n r.table('users').min('points')['points'].run(conn)\n\n*Example* Return the user who has scored the fewest points, but add a default `None` return value to prevent an error if no user has ever scored points.\n\n r.table('users').min('points').default(None).run(conn)\n"), - (rethinkdb.ast.RqlQuery.reduce, b'sequence.reduce(reduction_function) -> value\n\nProduce a single value from a sequence through repeated application of a reduction\nfunction. \nThe reduction function can be called on:\n\n- two elements of the sequence\n- one element of the sequence and one result of a previous reduction\n- two results of previous reductions\n\nThe reduction function can be called on the results of two previous reductions because the\n`reduce` command is distributed and parallelized across shards and CPU cores. A common\nmistaken when using the `reduce` command is to suppose that the reduction is executed\nfrom left to right. Read the [map-reduce in RethinkDB](http://rethinkdb.com/docs/map-reduce/) article to\nsee an example.\n\nIf the sequence is empty, the server will produce a `RqlRuntimeError` that can be\ncaught with `default`. \nIf the sequence has only one element, the first element will be returned.\n\n*Example* Return the number of documents in the table `posts`.\n\n r.table("posts").map(lambda doc: 1)\n .reduce(lambda left, right: left+right)\n .default(0).run(conn)\n\nA shorter way to execute this query is to use [count](http://rethinkdb.com/api/python/count).\n\n*Example* Suppose that each `post` has a field `comments` that is an array of\ncomments. \nReturn the number of comments for all posts.\n\n r.table("posts").map(lambda doc:\n doc["comments"].count()\n ).reduce(lambda left, right:\n left+right\n ).default(0).run(conn)\n\n*Example* Suppose that each `post` has a field `comments` that is an array of\ncomments. \nReturn the maximum number comments per post.\n\n r.table("posts").map(lambda doc:\n doc["comments"].count()\n ).reduce(lambda left, right:\n r.branch(\n left > right,\n left,\n right\n )\n ).default(0).run(conn)\n\nA shorter way to execute this query is to use [max](http://rethinkdb.com/api/python/max).\n'), - (rethinkdb.ast.RqlQuery.sum, b"sequence.sum([field_or_function]) -> number\n\nSums all the elements of a sequence. If called with a field name,\nsums all the values of that field in the sequence, skipping elements\nof the sequence that lack that field. If called with a function,\ncalls that function on every element of the sequence and sums the\nresults, skipping elements of the sequence where that function returns\n`None` or a non-existence error.\n\nReturns `0` when called on an empty sequence.\n\n*Example* What's 3 + 5 + 7?\n\n r.expr([3, 5, 7]).sum().run(conn)\n\n*Example* How many points have been scored across all games?\n\n r.table('games').sum('points').run(conn)\n\n*Example* How many points have been scored across all games,\ncounting bonus points?\n\n r.table('games').sum(lambda game:\n game['points'] + game['bonus_points']\n ).run(conn)\n"), - (rethinkdb.ast.RqlQuery.ungroup, b'grouped_stream.ungroup() -> array\ngrouped_data.ungroup() -> array\n\nTakes a grouped stream or grouped data and turns it into an array of\nobjects representing the groups. Any commands chained after `ungroup`\nwill operate on this array, rather than operating on each group\nindividually. This is useful if you want to e.g. order the groups by\nthe value of their reduction.\n\nThe format of the array returned by `ungroup` is the same as the\ndefault native format of grouped data in the JavaScript driver and\ndata explorer.\n\n*Example* What is the maximum number of points scored by each\nplayer, with the highest scorers first?\n\nSuppose that the table `games` has the following data:\n\n [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ]\n\nWe can use this query:\n\n r.table(\'games\')\n .group(\'player\').max(\'points\')[\'points\']\n .ungroup().order_by(r.desc(\'reduction\')).run(conn)\n\nResult: \n\n [\n {\n "group": "Bob",\n "reduction": 15\n },\n {\n "group": "Alice",\n "reduction": 7\n }\n ]\n\n*Example* Select one random player and all their games.\n\n r.table(\'games\').group(\'player\').ungroup().sample(1).run(conn)\n\nResult:\n\n [\n {\n "group": "Bob",\n "reduction": [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n ]\n\nNote that if you didn\'t call `ungroup`, you would instead select one\nrandom game from each player:\n\n r.table(\'games\').group(\'player\').sample(1).run(conn)\n\nResult:\n\n {\n "Alice": [\n {"id": 5, "player": "Alice", "points": 7, "type": "free"}\n ],\n "Bob": [\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n\n*Example* Types!\n\n r.table(\'games\').group(\'player\').type_of().run(conn) # Returns "GROUPED_STREAM"\n r.table(\'games\').group(\'player\').ungroup().type_of().run(conn) # Returns "ARRAY"\n r.table(\'games\').group(\'player\').avg(\'points\').run(conn) # Returns "GROUPED_DATA"\n r.table(\'games\').group(\'player\').avg(\'points\').ungroup().run(conn) #Returns "ARRAY"\n'), - (rethinkdb.args, b"r.args(array) -> special\n\n`r.args` is a special term that's used to splice an array of arguments\ninto another term. This is useful when you want to call a variadic\nterm such as `get_all` with a set of arguments produced at runtime.\n\nThis is analogous to unpacking argument lists in Python.\n\n*Example* Get Alice and Bob from the table `people`.\n\n r.table('people').get_all('Alice', 'Bob').run(conn)\n # or\n r.table('people').get_all(r.args(['Alice', 'Bob'])).run(conn)\n\n*Example* Get all of Alice's children from the table `people`.\n\n # r.table('people').get('Alice') returns {'id': 'Alice', 'children': ['Bob', 'Carol']}\n r.table('people').get_all(r.args(r.table('people').get('Alice')['children'])).run(conn)\n"), - (rethinkdb.binary, b'r.binary(data) -> binary\n\nEncapsulate binary data within a query.\n\nThe type of data `binary` accepts depends on the client language. In Python, it expects a parameter of `bytes` type. Using a `bytes` object within a query implies the use of `binary` and the ReQL driver will automatically perform the coercion (in Python 3 only).\n\nBinary objects returned to the client in JavaScript will also be of the `bytes` type. This can be changed with the `binary_format` option provided to [run](http://rethinkdb.com/api/python/run) to return "raw" objects.\n\nOnly a limited subset of ReQL commands may be chained after `binary`:\n\n* [coerce_to](http://rethinkdb.com/api/python/coerce_to/) can coerce `binary` objects to `string` types\n* [count](http://rethinkdb.com/api/python/count/) will return the number of bytes in the object\n* [slice](http://rethinkdb.com/api/python/slice/) will treat bytes like array indexes (i.e., `slice(10,20)` will return bytes 10–19)\n* [type_of](http://rethinkdb.com/api/python/type_of) returns `PTYPE<BINARY>`\n* [info](http://rethinkdb.com/api/python/info) will return information on a binary object.\n\n*Example* Save an avatar image to a existing user record.\n\n f = open(\'./default_avatar.png\', \'rb\')\n avatar_image = f.read()\n f.close()\n r.table(\'users\').get(100).update({\'avatar\': r.binary(avatar_image)}).run(conn)\n\n*Example* Get the size of an existing avatar image.\n\n r.table(\'users\').get(100)[\'avatar\'].count().run(conn)\n \n 14156\n\nRead more details about RethinkDB\'s binary object support: [Storing binary objects](http://rethinkdb.com/docs/storing-binary/).\n'), - (rethinkdb.branch, b'r.branch(test, true_branch, false_branch) -> any\n\nIf the `test` expression returns `False` or `None`, the `false_branch` will be evaluated.\nOtherwise, the `true_branch` will be evaluated.\n \nThe `branch` command is effectively an `if` renamed due to language constraints.\n\n*Example* Return heroes and superheroes.\n\n r.table(\'marvel\').map(\n r.branch(\n r.row[\'victories\'] > 100,\n r.row[\'name\'] + \' is a superhero\',\n r.row[\'name\'] + \' is a hero\'\n )\n ).run(conn)\n\nIf the documents in the table `marvel` are:\n\n [{\n "name": "Iron Man",\n "victories": 214\n },\n {\n "name": "Jubilee",\n "victories": 9\n }]\n\nThe results will be:\n\n [\n "Iron Man is a superhero",\n "Jubilee is a hero"\n ]\n'), - (rethinkdb.ast.RqlQuery.coerce_to, b"sequence.coerce_to('array') -> array\nvalue.coerce_to('string') -> string\nstring.coerce_to('number') -> number\narray.coerce_to('object') -> object\nobject.coerce_to('array') -> array\nbinary.coerce_to('string') -> string\nstring.coerce_to('binary') -> binary\n\nConvert a value of one type into another.\n\n* a sequence, selection or object can be coerced to an array\n* an array of key-value pairs can be coerced to an object\n* a string can be coerced to a number\n* any datum (single value) can be coerced to a string\n* a binary object can be coerced to a string and vice-versa\n\n*Example* Coerce a stream to an array to store its output in a field. (A stream cannot be stored in a field directly.)\n\n r.table('posts').map(lambda post: post.merge(\n { 'comments': r.table('comments').get_all(post['id'], index='post_id').coerce_to('array') }\n )).run(conn)\n\n*Example* Coerce an array of pairs into an object.\n\n r.expr([['name', 'Ironman'], ['victories', 2000]]).coerce_to('object').run(conn)\n\n__Note:__ To coerce a list of key-value pairs like `['name', 'Ironman', 'victories', 2000]` to an object, use the [object](http://rethinkdb.com/api/python/object) command.\n\n*Example* Coerce a number to a string.\n\n r.expr(1).coerce_to('string').run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.default, b'value.default(default_value) -> any\nsequence.default(default_value) -> any\n\nHandle non-existence errors. Tries to evaluate and return its first argument. If an\nerror related to the absence of a value is thrown in the process, or if its first\nargument returns `None`, returns its second argument. (Alternatively, the second argument\nmay be a function which will be called with either the text of the non-existence error\nor `None`.)\n\n*Example* Suppose we want to retrieve the titles and authors of the table `posts`.\nIn the case where the author field is missing or `None`, we want to retrieve the string\n`Anonymous`.\n\n r.table("posts").map(lambda post:\n {\n "title": post["title"],\n "author": post["author"].default("Anonymous")\n }\n ).run(conn)\n\nWe can rewrite the previous query with `r.branch` too.\n\n r.table("posts").map(lambda post:\n r.branch(\n post.has_fields("author"),\n {\n "title": post["title"],\n "author": post["author"]\n },\n {\n "title": post["title"],\n "author": "Anonymous" \n }\n )\n ).run(conn)\n\n*Example* The `default` command can be useful to filter documents too. Suppose\nwe want to retrieve all our users who are not grown-ups or whose age is unknown\n(i.e the field `age` is missing or equals `None`). We can do it with this query:\n\n r.table("users").filter(lambda user:\n (user["age"] < 18).default(True)\n ).run(conn)\n\nOne more way to write the previous query is to set the age to be `-1` when the\nfield is missing.\n\n r.table("users").filter(lambda user:\n user["age"].default(-1) < 18\n ).run(conn)\n\nOne last way to do the same query is to use `has_fields`.\n\n r.table("users").filter(lambda user:\n user.has_fields("age").not_() | (user["age"] < 18)\n ).run(conn)\n\nThe body of every `filter` is wrapped in an implicit `.default(False)`. You can overwrite\nthe value `False` by passing an option in filter, so the previous query can also be\nwritten like this.\n\n r.table("users").filter(\n lambda user: (user["age"] < 18).default(True),\n default=True\n ).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.do, b"any.do(function) -> any\nr.do([args]*, function) -> any\nany.do(expr) -> any\nr.do([args]*, expr) -> any\n\nCall an anonymous function using return values from other ReQL commands or queries as arguments.\n\nThe last argument to `do` (or, in some forms, the only argument) is an expression or an anonymous function which receives values from either the previous arguments or from prefixed commands chained before `do`. The `do` command is essentially a single-element [map](http://rethinkdb.com/api/python/map/), letting you map a function over just one document. This allows you to bind a query result to a local variable within the scope of `do`, letting you compute the result just once and reuse it in a complex expression or in a series of ReQL commands.\n\nArguments passed to the `do` function must be basic data types, and cannot be streams or selections. (Read about [ReQL data types](http://rethinkdb.com/docs/data-types/).) While the arguments will all be evaluated before the function is executed, they may be evaluated in any order, so their values should not be dependent on one another. The type of `do`'s result is the type of the value returned from the function or last expression.\n\n*Example* Compute a golfer's net score for a game.\n\n r.table('players').get('86be93eb-a112-48f5-a829-15b2cb49de1d').do(\n lambda player: player['gross_score'] - player['course_handicap']\n ).run(conn)\n\n*Example* Return the name of the best scoring player in a two-player golf match.\n\n r.do(r.table('players').get(id1), r.table('players').get(id2),\n (lambda player1, player2:\n r.branch(player1['gross_score'].lt(player2['gross_score']),\n player1, player2))\n ).run(conn)\n\nNote that `branch`, the ReQL conditional command, must be used instead of `if`. See the `branch` [documentation](http://rethinkdb.com/api/python/branch) for more.\n\n*Example* Take different actions based on the result of a ReQL [insert](http://rethinkdb.com/api/python/insert) command.\n\n new_data = {\n 'id': 100,\n 'name': 'Agatha',\n 'gross_score': 57,\n 'course_handicap': 4\n }\n r.table('players').insert(new_data).do(lambda doc:\n r.branch((doc['inserted'] != 0),\n r.table('log').insert({'time': r.now(), 'response': doc, 'result': 'ok'}),\n r.table('log').insert({'time': r.now(), 'response': doc, 'result': 'error'}))\n ).run(conn)\n"), - (rethinkdb.error, b"r.error(message) -> error\n\nThrow a runtime error. If called with no arguments inside the second argument to `default`, re-throw the current error.\n\n*Example* Iron Man can't possibly have lost a battle:\n\n r.table('marvel').get('IronMan').do(\n lambda ironman: r.branch(ironman['victories'] < ironman['battles'],\n r.error('impossible code path'),\n ironman)\n ).run(conn)\n\n"), - (rethinkdb.expr, b"r.expr(value) -> value\n\nConstruct a ReQL JSON object from a native object.\n\nIf the native object is of the `bytes` type, then `expr` will return a binary object. See [binary](http://rethinkdb.com/api/python/binary) for more information.\n\n*Example* Objects wrapped with expr can then be manipulated by ReQL API functions.\n\n r.expr({'a':'b'}).merge({'b':[1,2,3]}).run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.for_each, b"sequence.for_each(write_query) -> object\n\nLoop over a sequence, evaluating the given write query for each element.\n\n*Example* Now that our heroes have defeated their villains, we can safely remove them from the villain table.\n\n r.table('marvel').for_each(\n lambda hero: r.table('villains').get(hero['villainDefeated']).delete()\n ).run(conn)\n\n"), - (rethinkdb.http, b'r.http(url[, options]) -> value\nr.http(url[, options]) -> stream\n\nRetrieve data from the specified URL over HTTP. The return type depends on the `result_format` option, which checks the `Content-Type` of the response by default.\n\n*Example* Perform an HTTP `GET` and store the result in a table.\n\n r.table(\'posts\').insert(r.http(\'http://httpbin.org/get\')).run(conn)\n\nSee [the tutorial](http://rethinkdb.com/docs/external-api-access/) on `r.http` for more examples on how to use this command.\n\n* `timeout`: Number of seconds to wait before timing out and aborting the operation. Default: 30.\n\n* `reattempts`: An integer giving the number of attempts to make in cast of connection errors or potentially-temporary HTTP errors. Default: 5.\n\n* `redirects`: An integer giving the number of redirects and location headers to follow. Default: 1.\n\n* `verify`: Verify the server\'s SSL certificate, specified as a boolean. Default: True.\n\n* `result_format`: The format the result should be returned in. The values can be `\'text\'` (always return as a string), `\'json\'` (parse the result as JSON, raising an error if the parsing fails), `\'jsonp\'` (parse the result as [padded JSON](http://www.json-p.org/)), `\'binary\'` (return a binary object), or `\'auto\'` . The default is `\'auto\'`.\n\n When `result_format` is `\'auto\'`, the response body will be parsed according to the `Content-Type` of the response:\n * `application/json`: parse as `\'json\'`\n * `application/json-p`, `text/json-p`, `text/javascript`: parse as `\'jsonp\'`\n * `audio/*`, `video/*`, `image/*`, `application/octet-stream`: return a binary object\n * Anything else: parse as `\'text\'`\n\n* `method`: HTTP method to use for the request. One of `GET`, `POST`, `PUT`, `PATCH`, `DELETE` or `HEAD`. Default: `GET`.\n\n* `auth`: Authentication information in the form of an object with key/value pairs indicating the authentication type (in the `type` key) and any required information. Types currently supported are `basic` and `digest` for HTTP Basic and HTTP Digest authentication respectively. If `type` is omitted, `basic` is assumed. Example:\n\n\t```py\n\tr.http(\'http://httpbin.org/basic-auth/fred/mxyzptlk\',\n auth={ \'type\': \'basic\', \'user\': \'fred\', \'pass\': \'mxyzptlk\' }).run(conn)\n\t```\n\n* `params`: URL parameters to append to the URL as encoded key/value pairs, specified as an object. For example, `{ \'query\': \'banana\', \'limit\': 2 }` will be appended as `?query=banana&limit=2`. Default: none.\n\n* `header`: Extra header lines to include. The value may be an array of strings or an object. Default: none.\n\n Unless specified otherwise, `r.http` will by default use the headers `Accept-Encoding: deflate=1;gzip=0.5` and `User-Agent: RethinkDB/VERSION`.\n\n* `data`: Data to send to the server on a `POST`, `PUT`, `PATCH`, or `DELETE` request.\n\n For `PUT`, `PATCH` and `DELETE` requests, the value will be serialized to JSON and placed in the request body, and the `Content-Type` will be set to `application/json`.\n\n\tFor `POST` requests, data may be either an object or a string. Objects will be written to the body as form-encoded key/value pairs (values must be numbers, strings, or `None`). Strings will be put directly into the body. If `data` is not a string or an object, an error will be thrown.\n\n If `data` is not specified, no data will be sent.\n\n`r.http` supports depagination, which will request multiple pages in a row and aggregate the results into a stream. The use of this feature is controlled by the optional arguments `page` and `page_limit`. Either none or both of these arguments must be provided.\n\n* `page`: This option may specify either a built-in pagination strategy (as a string), or a function to provide the next URL and/or `params` to request.\n\n At the moment, the only supported built-in is `\'link-next\'`, which is equivalent to `lambda info: info[\'header\'][\'link\'][\'rel="next"\'].default(None)`.\n\n *Example* Perform a GitHub search and collect up to 3 pages of results.\n\n ```py\n r.http("https://api.github.com/search/code?q=addClass+user:mozilla",\n page=\'link-next\', page_limit=3).run(conn)\n ```\n\n As a function, `page` takes one parameter, an object of the format:\n\n ```py\n {\n \'params\': object, # the URL parameters used in the last request\n \'header\': object, # the HTTP headers of the last response as key/value pairs\n \'body\': value # the body of the last response in the format specified by `result_format`\n }\n ```\n\n The `header` field will be a parsed version of the header with fields lowercased, like so:\n\n ```py\n {\n \'content-length\': \'1024\',\n \'content-type\': \'application/json\',\n \'date\': \'Thu, 1 Jan 1970 00:00:00 GMT\',\n \'link\': {\n \'rel="last"\': \'http://example.com/?page=34\',\n \'rel="next"\': \'http://example.com/?page=2\'\n }\n }\n ```\n\n The `page` function may return a string corresponding to the next URL to request, `None` indicating that there is no more to get, or an object of the format:\n\n ```py\n {\n \'url\': string, # the next URL to request, or None for no more pages\n \'params\': object # new URL parameters to use, will be merged with the previous request\'s params\n }\n ```\n\n* `page_limit`: An integer specifying the maximum number of requests to issue using the `page` functionality. This is to prevent overuse of API quotas, and must be specified with `page`.\n * `-1`: no limit\n * `0`: no requests will be made, an empty stream will be returned\n * `n`: `n` requests will be made\n\n# Examples\n\n*Example* Perform multiple requests with different parameters.\n\n r.expr([1, 2, 3]).map(lambda i: r.http(\'http://httpbin.org/get\',\n params={ \'user\': i })).run(conn)\n\n*Example* Perform a `PUT` request for each item in a table.\n\n r.table(\'data\').map(lambda row: r.http(\'http://httpbin.org/put\',\n method=\'PUT\', data=row)).run(conn)\n\n*Example* Perform a `POST` request with accompanying data.\n\nUsing form-encoded data:\n\n r.http(\'http://httpbin.org/post\',\n method=\'POST\',\n data={ \'player\': \'Bob\', \'game\': \'tic tac toe\' }).run(conn)\n\nUsing JSON data:\n\n r.http(\'http://httpbin.org/post\',\n method=\'POST\',\n data=r.expr(value).coerce_to(\'string\'),\n header={ \'Content-Type\': \'application/json\' }).run(conn)\n\n*Example* Perform depagination with a custom `page` function.\n\n r.http(\'example.com/pages\',\n page=lambda info: info[\'body\'][\'meta\'][\'next\'].default(None),\n page_limit=5).run(conn)\n\n# Learn more\n\nSee [the tutorial](http://rethinkdb.com/docs/external-api-access/) on `r.http` for more examples on how to use this command.\n'), - (rethinkdb.ast.RqlQuery.info, b"any.info() -> object\n\nGet information about a ReQL value.\n\n*Example* Get information about a table such as primary key, or cache size.\n\n r.table('marvel').info().run(conn)\n\n"), - (rethinkdb.js, b'r.js(js_string[, timeout=<number>]) -> value\n\nCreate a javascript expression.\n\n*Example* Concatenate two strings using JavaScript.\n\n`timeout` is the number of seconds before `r.js` times out. The default value is 5 seconds.\n\n{% infobox %}\nWhenever possible, you should use native ReQL commands rather than `r.js` for better performance.\n{% endinfobox %}\n\n r.js("\'str1\' + \'str2\'").run(conn)\n\n*Example* Select all documents where the \'magazines\' field is greater than 5 by running JavaScript on the server.\n\n r.table(\'marvel\').filter(\n r.js(\'(function (row) { return row.magazines.length > 5; })\')\n ).run(conn)\n\n*Example* You may also specify a timeout in seconds (defaults to 5).\n\n r.js(\'while(true) {}\', timeout=1.3).run(conn)\n\n'), - (rethinkdb.json, b'r.json(json_string) -> value\n\nParse a JSON string on the server.\n\n*Example* Send an array to the server\'\n\n r.json("[1,2,3]").run(conn)\n\n'), - (rethinkdb.range, b'r.range() -> stream\nr.range([start_value, ]end_value) -> stream\n\nGenerate a stream of sequential integers in a specified range. `range` takes 0, 1 or 2 arguments:\n\n* With no arguments, `range` returns an "infinite" stream from 0 up to and including the maximum integer value;\n* With one argument, `range` returns a stream from 0 up to but not including the end value;\n* With two arguments, `range` returns a stream from the start value up to but not including the end value.\n\nNote that the left bound (including the implied left bound of 0 in the 0- and 1-argument form) is always closed and the right bound is always open: the start value will always be included in the returned range and the end value will *not* be included in the returned range.\n\nAny specified arguments must be integers, or a `RqlRuntimeError` will be thrown. If the start value is equal or to higher than the end value, no error will be thrown but a zero-element stream will be returned.\n\n*Example* Return a four-element range of `[0, 1, 2, 3]`.\n\n > r.range(4).run(conn)\n \n [0, 1, 2, 3]\n\nYou can also use the [limit](http://rethinkdb.com/api/python/limit) command with the no-argument variant to achieve the same result in this case:\n\n > r.range().limit(4).run(conn)\n \n [0, 1, 2, 3]\n\n*Example* Return a range from -5 through 5.\n\n > r.range(-5, 6).run(conn)\n \n [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]\n'), - (rethinkdb.ast.RqlQuery.to_json_string, b'value.to_json_string() -> string\nvalue.to_json() -> string\n\nConvert a ReQL value or object to a JSON string. You may use either `to_json_string` or `to_json`.\n\n*Example* Get a ReQL document as a JSON string.\n\n > r.table(\'hero\').get(1).to_json()\n \n \'{"id": 1, "name": "Batman", "city": "Gotham", "powers": ["martial arts", "cinematic entrances"]}\'\n'), - (rethinkdb.ast.RqlQuery.to_json, b'value.to_json_string() -> string\nvalue.to_json() -> string\n\nConvert a ReQL value or object to a JSON string. You may use either `to_json_string` or `to_json`.\n\n*Example* Get a ReQL document as a JSON string.\n\n > r.table(\'hero\').get(1).to_json()\n \n \'{"id": 1, "name": "Batman", "city": "Gotham", "powers": ["martial arts", "cinematic entrances"]}\'\n'), - (rethinkdb.ast.RqlQuery.type_of, b'any.type_of() -> string\n\nGets the type of a value.\n\n*Example* Get the type of a string.\n\n r.expr("foo").type_of().run(conn)\n\n'), - (rethinkdb.uuid, b'r.uuid() -> string\n\nReturn a UUID (universally unique identifier), a string that can be used as a unique ID.\n\n*Example* Generate a UUID.\n\n > r.uuid().run(conn)\n \n 27961a0e-f4e8-4eb3-bf95-c5203e1d87b9\n'), - (rethinkdb.net.Cursor.close, b'cursor.close()\n\nClose a cursor. Closing a cursor cancels the corresponding query and frees the memory\nassociated with the open request.\n\n*Example* Close a cursor.\n\n cursor.close()\n'), - (rethinkdb.net.Cursor.next, b"cursor.next([wait=True])\n\nGet the next element in the cursor.\n\nThe optional `wait` argument specifies whether to wait for the next available element and how long to wait:\n\n* `True`: Wait indefinitely (the default).\n* `False`: Do not wait at all. If data is immediately available, it will be returned; if it is not available, a `RqlDriverError` will be raised.\n* number: Wait up the specified number of seconds for data to be available before raising `RqlDriverError`.\n\nThe behavior of `next` will be identical with `False`, `None` or the number `0`.\n\nCalling `next` the first time on a cursor provides the first element of the cursor. If the data set is exhausted (e.g., you have retrieved all the documents in a table), a `StopIteration` error will be raised when `next` is called.\n\n*Example* Retrieve the next element.\n\n cursor = r.table('superheroes').run(conn)\n doc = cursor.next()\n\n*Example* Retrieve the next element on a [changefeed](http://rethinkdb.com/docs/changefeeds/python), waiting up to five seconds.\n\n cursor = r.table('superheroes').changes().run(conn)\n doc = cursor.next(wait=5)\n\n__Note:__ RethinkDB sequences can be iterated through via the Python [Iterable][it] interface. The canonical way to retrieve all the results is to use a [for...in](../each/) loop or [list()](../to_array/).\n\n[it]: https://docs.python.org/3.4/library/stdtypes.html#iterator-types\n"), - (rethinkdb.ast.RqlQuery.date, b'time.date() -> time\n\nReturn a new time object only based on the day, month and year (ie. the same day at 00:00).\n\n*Example* Retrieve all the users whose birthday is today\n\n r.table("users").filter(lambda user:\n user["birthdate"].date() == r.now().date()\n ).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.day, b'time.day() -> number\n\nReturn the day of a time object as a number between 1 and 31.\n\n*Example* Return the users born on the 24th of any month.\n\n r.table("users").filter(\n r.row["birthdate"].day() == 24\n )\n\n'), - (rethinkdb.ast.RqlQuery.day_of_week, b'time.day_of_week() -> number\n\nReturn the day of week of a time object as a number between 1 and 7 (following ISO 8601 standard). For your convenience, the terms r.monday, r.tuesday etc. are defined and map to the appropriate integer.\n\n*Example* Return today\'s day of week.\n\n r.now().day_of_week().run(conn)\n\n*Example* Retrieve all the users who were born on a Tuesday.\n\n r.table("users").filter( lambda user:\n user["birthdate"].day_of_week().eq(r.tuesday)\n )\n\n'), - (rethinkdb.ast.RqlQuery.day_of_year, b'time.day_of_year() -> number\n\nReturn the day of the year of a time object as a number between 1 and 366 (following ISO 8601 standard).\n\n*Example* Retrieve all the users who were born the first day of a year.\n\n r.table("users").filter(\n r.row["birthdate"].day_of_year() == 1\n ).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.during, b'time.during(start_time, end_time[, left_bound="closed", right_bound="open"])\n -> bool\n\nReturn whether a time is between two other times. By default, this is inclusive of the start time and exclusive of the end time. Set `left_bound` and `right_bound` to explicitly include (`closed`) or exclude (`open`) that endpoint of the range.\n\n*Example* Retrieve all the posts that were posted between December 1st, 2013 (inclusive) and December 10th, 2013 (exclusive).\n\n r.table("posts").filter(\n r.row[\'date\'].during(r.time(2013, 12, 1, "Z"), r.time(2013, 12, 10, "Z"))\n ).run(conn)\n\n*Example* Retrieve all the posts that were posted between December 1st, 2013 (exclusive) and December 10th, 2013 (inclusive).\n\n r.table("posts").filter(\n r.row[\'date\'].during(r.time(2013, 12, 1, "Z"), r.time(2013, 12, 10, "Z"), left_bound="open", right_bound="closed")\n ).run(conn)\n\n'), - (rethinkdb.epoch_time, b'r.epoch_time(epoch_time) -> time\n\nCreate a time object based on seconds since epoch. The first argument is a double and\nwill be rounded to three decimal places (millisecond-precision).\n\n*Example* Update the birthdate of the user "John" to November 3rd, 1986.\n\n r.table("user").get("John").update({"birthdate": r.epoch_time(531360000)}).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.hours, b'time.hours() -> number\n\nReturn the hour in a time object as a number between 0 and 23.\n\n*Example* Return all the posts submitted after midnight and before 4am.\n\n r.table("posts").filter(lambda post:\n post["date"].hours() < 4\n ).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.in_timezone, b"time.in_timezone(timezone) -> time\n\nReturn a new time object with a different timezone. While the time stays the same, the results returned by methods such as hours() will change since they take the timezone into account. The timezone argument has to be of the ISO 8601 format.\n\n*Example* Hour of the day in San Francisco (UTC/GMT -8, without daylight saving time).\n\n r.now().in_timezone('-08:00').hours().run(conn)\n"), - (rethinkdb.iso8601, b'r.iso8601(iso8601Date[, default_timezone=\'\']) -> time\n\nCreate a time object based on an ISO 8601 date-time string (e.g. \'2013-01-01T01:01:01+00:00\'). We support all valid ISO 8601 formats except for week dates. If you pass an ISO 8601 date-time without a time zone, you must specify the time zone with the `default_timezone` argument. Read more about the ISO 8601 format at [Wikipedia](http://en.wikipedia.org/wiki/ISO_8601).\n\n*Example* Update the time of John\'s birth.\n\n r.table("user").get("John").update({"birth": r.iso8601(\'1986-11-03T08:30:00-07:00\')}).run(conn)\n'), - (rethinkdb.ast.RqlQuery.minutes, b'time.minutes() -> number\n\nReturn the minute in a time object as a number between 0 and 59.\n\n*Example* Return all the posts submitted during the first 10 minutes of every hour.\n\n r.table("posts").filter(lambda post:\n post["date"].minutes() < 10\n ).run(conn)\n'), - (rethinkdb.ast.RqlQuery.month, b'time.month() -> number\n\nReturn the month of a time object as a number between 1 and 12. For your convenience, the terms r.january, r.february etc. are defined and map to the appropriate integer.\n\n*Example* Retrieve all the users who were born in November.\n\n r.table("users").filter(\n r.row["birthdate"].month() == 11\n )\n\n*Example* Retrieve all the users who were born in November.\n\n r.table("users").filter(\n r.row["birthdate"].month() == r.november\n )\n\n'), - (rethinkdb.now, b'r.now() -> time\n\nReturn a time object representing the current time in UTC. The command now() is computed once when the server receives the query, so multiple instances of r.now() will always return the same time inside a query.\n\n*Example* Add a new user with the time at which he subscribed.\n\n r.table("users").insert({\n "name": "John",\n "subscription_date": r.now()\n }).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.seconds, b'time.seconds() -> number\n\nReturn the seconds in a time object as a number between 0 and 59.999 (double precision).\n\n*Example* Return the post submitted during the first 30 seconds of every minute.\n\n r.table("posts").filter(lambda post:\n post["date"].seconds() < 30\n ).run(conn)\n\n'), - (rethinkdb.time, b'r.time(year, month, day[, hour, minute, second], timezone)\n -> time\n\nCreate a time object for a specific time.\n\nA few restrictions exist on the arguments:\n\n- `year` is an integer between 1400 and 9,999.\n- `month` is an integer between 1 and 12.\n- `day` is an integer between 1 and 31.\n- `hour` is an integer.\n- `minutes` is an integer.\n- `seconds` is a double. Its value will be rounded to three decimal places\n(millisecond-precision).\n- `timezone` can be `\'Z\'` (for UTC) or a string with the format `\xc2\xb1[hh]:[mm]`.\n\n*Example* Update the birthdate of the user "John" to November 3rd, 1986 UTC.\n\n r.table("user").get("John").update({"birthdate": r.time(1986, 11, 3, \'Z\')}).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.time_of_day, b'time.time_of_day() -> number\n\nReturn the number of seconds elapsed since the beginning of the day stored in the time object.\n\n*Example* Retrieve posts that were submitted before noon.\n\n r.table("posts").filter(\n r.row["date"].time_of_day() <= 12*60*60\n ).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.timezone, b'time.timezone() -> string\n\nReturn the timezone of the time object.\n\n*Example* Return all the users in the "-07:00" timezone.\n\n r.table("users").filter(lambda user:\n user["subscriptionDate"].timezone() == "-07:00"\n )\n\n'), - (rethinkdb.ast.RqlQuery.to_epoch_time, b'time.to_epoch_time() -> number\n\nConvert a time object to its epoch time.\n\n*Example* Return the current time in seconds since the Unix Epoch with millisecond-precision.\n\n r.now().to_epoch_time()\n\n'), - (rethinkdb.ast.RqlQuery.to_iso8601, b'time.to_iso8601() -> string\n\nConvert a time object to a string in ISO 8601 format.\n\n*Example* Return the current ISO 8601 time.\n\n > r.now().to_iso8601().run(conn)\n \n "2015-04-20T18:37:52.690+00:00"\n\n'), - (rethinkdb.ast.RqlQuery.year, b'time.year() -> number\n\nReturn the year of a time object.\n\n*Example* Retrieve all the users born in 1986.\n\n r.table("users").filter(lambda user:\n user["birthdate"].year() == 1986\n ).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.append, b"array.append(value) -> array\n\nAppend a value to an array.\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].append('newBoots').run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.__getitem__, b"sequence[attr] -> sequence\nsingleSelection[attr] -> value\nobject[attr] -> value\narray[index] -> value\n\nGet a single field from an object. If called on a sequence, gets that field from every object in the sequence, skipping objects that lack it.\n\n*Example* What was Iron Man's first appearance in a comic?\n\n r.table('marvel').get('IronMan')['firstAppearance'].run(conn)\n\nThe `[]` command also accepts integer arguments as array offsets, like the [nth](http://rethinkdb.com/api/python/nth) command.\n\n*Example* Get the fourth element in a sequence. (The first element is position `0`, so the fourth element is position `3`.)\n\n r.expr([10, 20, 30, 40, 50])[3]\n \n 40\n"), - (rethinkdb.ast.RqlQuery.change_at, b'array.change_at(index, value) -> array\n\nChange a value in an array at a given index. Returns the modified array.\n\n*Example* Bruce Banner hulks out.\n\n r.expr(["Iron Man", "Bruce", "Spider-Man"]).change_at(1, "Hulk").run(conn)\n'), - (rethinkdb.ast.RqlQuery.delete_at, b"array.delete_at(index [,endIndex]) -> array\n\nRemove one or more elements from an array at a given index. Returns the modified array. (Note: `delete_at` operates on arrays, not documents; to delete documents, see the [delete](http://rethinkdb.com/api/python/delete) command.)\n\nIf only `index` is specified, `delete_at` removes the element at that index. If both `index` and `end_index` are specified, `delete_at` removes the range of elements between `index` and `end_index`, inclusive of `index` but not inclusive of `end_index`.\n\nIf `end_index` is specified, it must not be less than `index`. Both `index` and `end_index` must be within the array's bounds (i.e., if the array has 10 elements, an `index` or `end_index` of 10 or higher is invalid).\n\nBy using a negative `index` you can delete from the end of the array. `-1` is the last element in the array, `-2` is the second-to-last element, and so on. You may specify a negative `end_index`, although just as with a positive value, this will not be inclusive. The range `(2,-1)` specifies the third element through the next-to-last element.\n\n*Example* Delete the second element of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(1).run(conn)\n \n ['a', 'c', 'd', 'e', 'f']\n\n*Example* Delete the second and third elements of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(1,3).run(conn)\n \n ['a', 'd', 'e', 'f']\n\n*Example* Delete the next-to-last element of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(-2).run(conn)\n \n ['a', 'b', 'c', 'd', 'f']\n\n*Example* Delete a comment on a post.\n\nGiven a post document such as:\n\n{\n id: '4cf47834-b6f9-438f-9dec-74087e84eb63',\n title: 'Post title',\n author: 'Bob',\n comments: [\n { author: 'Agatha', text: 'Comment 1' },\n { author: 'Fred', text: 'Comment 2' }\n ]\n}\n\nThe second comment can be deleted by using `update` and `delete_at` together.\n\n r.table('posts').get('4cf47834-b6f9-438f-9dec-74087e84eb63').update(\n lambda post: { 'comments': post['comments'].delete_at(1) }\n ).run(conn)\n"), - (rethinkdb.ast.RqlQuery.difference, b"array.difference(array) -> array\n\nRemove the elements of one array from another array.\n\n*Example* Retrieve Iron Man's equipment list without boots.\n\n r.table('marvel').get('IronMan')['equipment'].difference(['Boots']).run(conn)\n\n*Example* Remove Iron Man's boots from his equipment.\n\n r.table('marvel').get('IronMan')[:equipment].update(lambda doc:\n {'equipment': doc['equipment'].difference(['Boots'])}\n ).run(conn)\n"), - (rethinkdb.ast.RqlQuery.get_field, b"sequence.get_field(attr) -> sequence\nsingleSelection.get_field(attr) -> value\nobject.get_field(attr) -> value\n\nGet a single field from an object. If called on a sequence, gets that field from every\nobject in the sequence, skipping objects that lack it.\n\n*Example* What was Iron Man's first appearance in a comic?\n\n r.table('marvel').get('IronMan').get_field('firstAppearance').run(conn)\n"), - (rethinkdb.ast.RqlQuery.has_fields, b'sequence.has_fields([selector1, selector2...]) -> stream\narray.has_fields([selector1, selector2...]) -> array\nobject.has_fields([selector1, selector2...]) -> boolean\n\nTest if an object has one or more fields. An object has a field if it has that key and the key has a non-null value. For instance, the object `{\'a\': 1,\'b\': 2,\'c\': null}` has the fields `a` and `b`.\n\nWhen applied to a single object, `has_fields` returns `true` if the object has the fields and `false` if it does not. When applied to a sequence, it will return a new sequence (an array or stream) containing the elements that have the specified fields.\n\n*Example* Return the players who have won games.\n\n r.table(\'players\').has_fields(\'games_won\').run(conn)\n\n*Example* Return the players who have *not* won games. To do this, use `has_fields` with [not](http://rethinkdb.com/api/python/not), wrapped with [filter](http://rethinkdb.com/api/python/filter).\n\n r.table(\'players\').filter(~r.row.has_fields(\'games_won\')).run(conn)\n\n*Example* Test if a specific player has won any games.\n\n r.table(\'players\').get(\n \'b5ec9714-837e-400c-aa74-dbd35c9a7c4c\').has_fields(\'games_won\').run(conn)\n\n**Nested Fields**\n\n`has_fields` lets you test for nested fields in objects. If the value of a field is itself a set of key/value pairs, you can test for the presence of specific keys.\n\n*Example* In the `players` table, the `games_won` field contains one or more fields for kinds of games won:\n\n {\n \'games_won\': {\n \'playoffs\': 2,\n \'championships\': 1\n }\n }\n\nReturn players who have the "championships" field.\n\n r.table(\'players\').has_fields({\'games_won\': {\'championships\': true}}).run(conn)\n\nNote that `true` in the example above is testing for the existence of `championships` as a field, not testing to see if the value of the `championships` field is set to `true`. There\'s a more convenient shorthand form available. (See [pluck](http://rethinkdb.com/api/python/pluck) for more details on this.)\n\n r.table(\'players\').has_fields({\'games_won\': \'championships\'}).run(conn)\n'), - (rethinkdb.ast.RqlQuery.insert_at, b'array.insert_at(index, value) -> array\n\nInsert a value in to an array at a given index. Returns the modified array.\n\n*Example* Hulk decides to join the avengers.\n\n r.expr(["Iron Man", "Spider-Man"]).insert_at(1, "Hulk").run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.keys, b"singleSelection.keys() -> array\nobject.keys() -> array\n\nReturn an array containing all of the object's keys.\n\n*Example* Get all the keys of a row.\n\n r.table('marvel').get('ironman').keys().run(conn)\n\n"), - (rethinkdb.literal, b'r.literal(object) -> special\n\nReplace an object in a field instead of merging it with an existing object in a `merge` or `update` operation. = Using `literal` with no arguments in a `merge` or `update` operation will remove the corresponding field.\n\n*Example* Replace one nested document with another rather than merging the fields.\n\nAssume your users table has this structure:\n\n [\n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 18,\n "city": "Dallas"\n }\n } \n ...\n ]\n\nUsing `update` to modify the `data` field will normally merge the nested documents:\n\n r.table(\'users\').get(1).update({ \'data\': { \'age\': 19, \'job\': \'Engineer\' } }).run(conn)\n \n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 19,\n "city": "Dallas",\n "job": "Engineer"\n }\n } \n\nThat will preserve `city` and other existing fields. But to replace the entire `data` document with a new object, use `literal`:\n\n r.table(\'users\').get(1).update({ \'data\': r.literal({ \'age\': 19, \'job\': \'Engineer\' }) }).run(conn)\n \n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 19,\n "job": "Engineer"\n }\n } \n\n*Example* Use `literal` to remove a field from a document.\n\n r.table(\'users\').get(1).merge({ "data": r.literal() }).run(conn)\n \n {\n "id": 1,\n "name": "Alice"\n }\n'), - (rethinkdb.ast.RqlQuery.merge, b'singleSelection.merge(object|function[, object|function, ...]) -> object\nobject.merge(object|function[, object|function, ...]) -> object\nsequence.merge(object|function[, object|function, ...]) -> stream\narray.merge(object|function[, object|function, ...]) -> array\n\nMerge two or more objects together to construct a new object with properties from all. When there is a conflict between field names, preference is given to fields in the rightmost object in the argument list `merge` also accepts a subquery function that returns an object, which will be used similarly to a [map](http://rethinkdb.com/api/python/map/) function.\n\n*Example* Equip Thor for battle.\n\n r.table(\'marvel\').get(\'thor\').merge(\n r.table(\'equipment\').get(\'hammer\'),\n r.table(\'equipment\').get(\'pimento_sandwich\')\n ).run(conn)\n\n*Example* Equip every hero for battle, using a subquery function to retrieve their weapons.\n\n r.table(\'marvel\').merge(lambda hero:\n { \'weapons\': r.table(\'weapons\').get(hero[\'weapon_id\']) }\n ).run(conn)\n\n*Example* Use `merge` to join each blog post with its comments.\n\nNote that the sequence being merged—in this example, the comments—must be coerced from a selection to an array. Without `coerce_to` the operation will throw an error ("Expected type DATUM but found SELECTION").\n\n r.table(\'posts\').merge(lambda post:\n { \'comments\': r.table(\'comments\').get_all(post[\'id\'],\n index=\'post_id\').coerce_to(\'array\') }\n ).run(conn)\n\n*Example* Merge can be used recursively to modify object within objects.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10}}}\n ).run(conn)\n\n*Example* To replace a nested object with another object you can use the literal keyword.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : r.literal({\'repulsor rays\' : {\'dmg\' : 3, \'cooldown\' : 0}})}\n ).run(conn)\n\n*Example* Literal can be used to remove keys from an object as well.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : {\'spectacular graviton beam\' : r.literal()}}\n ).run(conn)\n\n'), - (rethinkdb.object, b'r.object([key, value,]...) -> object\n\nCreates an object from a list of key-value pairs, where the keys must\nbe strings. `r.object(A, B, C, D)` is equivalent to\n`r.expr([[A, B], [C, D]]).coerce_to(\'OBJECT\')`.\n\n*Example* Create a simple object.\n\n > r.object(\'id\', 5, \'data\', [\'foo\', \'bar\']).run(conn)\n {\'data\': ["foo", "bar"], \'id\': 5}\n'), - (rethinkdb.ast.RqlQuery.pluck, b"sequence.pluck([selector1, selector2...]) -> stream\narray.pluck([selector1, selector2...]) -> array\nobject.pluck([selector1, selector2...]) -> object\nsingleSelection.pluck([selector1, selector2...]) -> object\n\nPlucks out one or more attributes from either an object or a sequence of objects\n(projection).\n\n*Example* We just need information about IronMan's reactor and not the rest of the\ndocument.\n\n r.table('marvel').get('IronMan').pluck('reactorState', 'reactorPower').run(conn)\n\n*Example* For the hero beauty contest we only care about certain qualities.\n\n r.table('marvel').pluck('beauty', 'muscleTone', 'charm').run(conn)\n\n*Example* Pluck can also be used on nested objects.\n\n r.table('marvel').pluck({'abilities' : {'damage' : True, 'mana_cost' : True}, 'weapons' : True}).run(conn)\n\n*Example* The nested syntax can quickly become overly verbose so there's a shorthand\nfor it.\n\n r.table('marvel').pluck({'abilities' : ['damage', 'mana_cost']}, 'weapons').run(conn)\n\nFor more information read the [nested field documentation](http://rethinkdb.com/docs/nested-fields/).\n"), - (rethinkdb.ast.RqlQuery.prepend, b"array.prepend(value) -> array\n\nPrepend a value to an array.\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].prepend('newBoots').run(conn)\n"), - (rethinkdb.row, b"r.row -> value\n\nReturns the currently visited document. Note that `row` does not work within subqueries to access nested documents; you should use anonymous functions to access those documents instead. (See the last example.)\n\n*Example* Get all users whose age is greater than 5.\n\n r.table('users').filter(r.row['age'] > 5).run(conn)\n\n*Example* Access the attribute 'child' of an embedded document.\n\n r.table('users').filter(r.row['embedded_doc']['child'] > 5).run(conn)\n\n*Example* Add 1 to every element of an array.\n\n r.expr([1, 2, 3]).map(r.row + 1).run(conn)\n\n*Example* For nested queries, use functions instead of `row`.\n\n r.table('users').filter(\n lambda doc: doc['name'] == r.table('prizes').get('winner')\n ).run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.set_difference, b"array.set_difference(array) -> array\n\nRemove the elements of one array from another and return them as a set (an array with\ndistinct values).\n\n*Example* Check which pieces of equipment Iron Man has, excluding a fixed list.\n\n r.table('marvel').get('IronMan')['equipment'].set_difference(['newBoots', 'arc_reactor']).run(conn)\n"), - (rethinkdb.ast.RqlQuery.set_insert, b"array.set_insert(value) -> array\n\nAdd a value to an array and return it as a set (an array with distinct values).\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].set_insert('newBoots').run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.set_intersection, b"array.set_intersection(array) -> array\n\nIntersect two arrays returning values that occur in both of them as a set (an array with\ndistinct values).\n\n*Example* Check which pieces of equipment Iron Man has from a fixed list.\n\n r.table('marvel').get('IronMan')['equipment'].set_intersection(['newBoots', 'arc_reactor']).run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.set_union, b"array.set_union(array) -> array\n\nAdd a several values to an array and return it as a set (an array with distinct values).\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots and an arc reactor.\n\n r.table('marvel').get('IronMan')['equipment'].set_union(['newBoots', 'arc_reactor']).run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.splice_at, b'array.splice_at(index, array) -> array\n\nInsert several values in to an array at a given index. Returns the modified array.\n\n*Example* Hulk and Thor decide to join the avengers.\n\n r.expr(["Iron Man", "Spider-Man"]).splice_at(1, ["Hulk", "Thor"]).run(conn)\n'), - (rethinkdb.ast.RqlQuery.without, b"sequence.without([selector1, selector2...]) -> stream\narray.without([selector1, selector2...]) -> array\nsingleSelection.without([selector1, selector2...]) -> object\nobject.without([selector1, selector2...]) -> object\n\nThe opposite of pluck; takes an object or a sequence of objects, and returns them with\nthe specified paths removed.\n\n*Example* Since we don't need it for this computation we'll save bandwidth and leave\nout the list of IronMan's romantic conquests.\n\n r.table('marvel').get('IronMan').without('personalVictoriesList').run(conn)\n\n*Example* Without their prized weapons, our enemies will quickly be vanquished.\n\n r.table('enemies').without('weapons').run(conn)\n\n*Example* Nested objects can be used to remove the damage subfield from the weapons and abilities fields.\n\n r.table('marvel').without({'weapons' : {'damage' : True}, 'abilities' : {'damage' : True}}).run(conn)\n\n*Example* The nested syntax can quickly become overly verbose so there's a shorthand for it.\n\n r.table('marvel').without({'weapons' : 'damage', 'abilities' : 'damage'}).run(conn)\n\n"), - (rethinkdb.circle, b"r.circle([longitude, latitude], radius[, num_vertices=32, geo_system='WGS84', unit='m', fill=True]) -> geometry\nr.circle(point, radius[, {num_vertices=32, geo_system='WGS84', unit='m', fill=True]) -> geometry\n\nConstruct a circular line or polygon. A circle in RethinkDB is a polygon or line *approximating* a circle of a given radius around a given center, consisting of a specified number of vertices (default 32).\n\nThe center may be specified either by two floating point numbers, the latitude (−90 to 90) and longitude (−180 to 180) of the point on a perfect sphere (see [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system), or by a point object. The radius is a floating point number whose units are meters by default, although that may be changed with the `unit` argument.\n\nOptional arguments available with `circle` are:\n\n* `num_vertices`: the number of vertices in the polygon or line. Defaults to 32.\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n* `unit`: Unit for the radius distance. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n* `fill`: if `True` (the default) the circle is filled, creating a polygon; if `False` the circle is unfilled (creating a line).\n\n*Example* Define a circle.\n\n r.table('geo').insert({\n 'id': 300,\n 'name': 'Hayes Valley',\n 'neighborhood': r.circle([-122.423246,37.779388], 1000)\n }).run(conn)\n"), - (rethinkdb.ast.RqlQuery.distance, b"geometry.distance(geometry[, geo_system='WGS84', unit='m']) -> number\n\nCompute the distance between a point and another geometry object. At least one of the geometry objects specified must be a point.\n\nOptional arguments available with `distance` are:\n\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n* `unit`: Unit to return the distance in. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n\nIf one of the objects is a polygon or a line, the point will be projected onto the line or polygon assuming a perfect sphere model before the distance is computed (using the model specified with `geo_system`). As a consequence, if the polygon or line is extremely large compared to Earth's radius and the distance is being computed with the default WGS84 model, the results of `distance` should be considered approximate due to the deviation between the ellipsoid and spherical models.\n\n*Example* Compute the distance between two points on the Earth in kilometers.\n\n > point1 = r.point(-122.423246,37.779388)\n > point2 = r.point(-117.220406,32.719464)\n > r.distance(point1, point2, unit='km').run(conn)\n \n 734.1252496021841\n"), - (rethinkdb.ast.RqlQuery.fill, b"line.fill() -> polygon\n\nConvert a Line object into a Polygon object. If the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\nIf the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them. You cannot directly construct a polygon with holes in it using `polygon`, but you can use [polygon_sub](http://rethinkdb.com/api/python/polygon_sub) to use a second polygon within the interior of the first to define a hole.\n\n*Example* Create a line object and then convert it to a polygon.\n\n r.table('geo').insert({\n 'id': 201,\n 'rectangle': r.line(\n [-122.423246,37.779388],\n [-122.423246,37.329898],\n [-121.886420,37.329898],\n [-121.886420,37.779388]\n )\n }).run(conn)\n \n r.table('geo').get(201).update({\n 'rectangle': r.row('rectangle').fill()\n }).run(conn)\n"), - (rethinkdb.geojson, b"r.geojson(geojson) -> geometry\n\nConvert a [GeoJSON][] object to a ReQL geometry object.\n\n[GeoJSON]: http://geojson.org\n\nRethinkDB only allows conversion of GeoJSON objects which have ReQL equivalents: Point, LineString, and Polygon. MultiPoint, MultiLineString, and MultiPolygon are not supported. (You could, however, store multiple points, lines and polygons in an array and use a geospatial multi index with them.)\n\nOnly longitude/latitude coordinates are supported. GeoJSON objects that use Cartesian coordinates, specify an altitude, or specify their own coordinate reference system will be rejected.\n\n*Example* Convert a GeoJSON object to a ReQL geometry object.\n\n geo_json = {\n 'type': 'Point',\n 'coordinates': [ -122.423246, 37.779388 ]\n }\n r.table('geo').insert({\n 'id': 'sfo',\n 'name': 'San Francisco',\n 'location': r.geojson(geo_json)\n }).run(conn)\n"), - (rethinkdb.ast.Table.get_intersecting, b"table.get_intersecting(geometry, index='indexname') -> selection<stream>\n\nGet all documents where the given geometry object intersects the geometry object of the requested geospatial index.\n\nThe `index` argument is mandatory. This command returns the same results as `table.filter(r.row('index').intersects(geometry))`. The total number of results is limited to the array size limit which defaults to 100,000, but can be changed with the `array_limit` option to [run](http://rethinkdb.com/api/python/run).\n\n*Example* Which of the locations in a list of parks intersect `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks').get_intersecting(circle1, index='area').run(conn)\n"), - (rethinkdb.ast.Table.get_nearest, b"table.get_nearest(point, index='indexname'[, max_results=100, max_dist=100000, unit='m', geo_system='WGS84']) -> array\n\nGet all documents where the specified geospatial index is within a certain distance of the specified point (default 100 kilometers).\n\nThe `index` argument is mandatory. Optional arguments are:\n\n* `max_results`: the maximum number of results to return (default 100).\n* `unit`: Unit for the distance. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n* `max_dist`: the maximum distance from an object to the specified point (default 100 km).\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n\nThe return value will be an array of two-item objects with the keys `dist` and `doc`, set to the distance between the specified point and the document (in the units specified with `unit`, defaulting to meters) and the document itself, respectively.\n\n*Example* Return a list of enemy hideouts within 5000 meters of the secret base.\n\n secret_base = r.point(-122.422876,37.777128)\n r.table('hideouts').get_nearest(secret_base, index='location',\n max_dist=5000).run(conn)\n"), - (rethinkdb.ast.RqlQuery.includes, b"sequence.includes(geometry) -> sequence\ngeometry.includes(geometry) -> bool\n\nTests whether a geometry object is completely contained within another. When applied to a sequence of geometry objects, `includes` acts as a [filter](http://rethinkdb.com/api/python/filter), returning a sequence of objects from the sequence that include the argument.\n\n*Example* Is `point2` included within a 2000-meter circle around `point1`?\n\n > point1 = r.point(-117.220406,32.719464)\n > point2 = r.point(-117.206201,32.725186)\n > r.circle(point1, 2000).includes(point2).run(conn)\n \n True\n\n*Example* Which of the locations in a list of parks include `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks')['area'].includes(circle1).run(conn)\n"), - (rethinkdb.ast.RqlQuery.intersects, b"sequence.intersects(geometry) -> sequence\ngeometry.intersects(geometry) -> bool\n\nTests whether two geometry objects intersect with one another. When applied to a sequence of geometry objects, `intersects` acts as a [filter](http://rethinkdb.com/api/python/filter), returning a sequence of objects from the sequence that intersect with the argument.\n\n*Example* Is `point2` within a 2000-meter circle around `point1`?\n\n > point1 = r.point(-117.220406,32.719464)\n > point2 = r.point(-117.206201,32.725186)\n > r.circle(point1, 2000).intersects(point2).run(conn)\n \n True\n\n*Example* Which of the locations in a list of parks intersect `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks')('area').intersects(circle1).run(conn)\n"), - (rethinkdb.line, b"r.line([lon1, lat1], [lon2, lat2], ...) -> line\nr.line(point1, point2, ...) -> line\n\nConstruct a geometry object of type Line. The line can be specified in one of two ways:\n\n* Two or more two-item arrays, specifying latitude and longitude numbers of the line's vertices;\n* Two or more [Point](http://rethinkdb.com/api/python/point) objects specifying the line's vertices.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\n*Example* Define a line.\n\n r.table('geo').insert({\n 'id': 101,\n 'route': r.line([-122.423246,37.779388], [-121.886420,37.329898])\n }).run(conn)\n"), - (rethinkdb.point, b"r.point(longitude, latitude) -> point\n\nConstruct a geometry object of type Point. The point is specified by two floating point numbers, the longitude (−180 to 180) and latitude (−90 to 90) of the point on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\n*Example* Define a point.\n\n r.table('geo').insert({\n 'id': 1,\n 'name': 'San Francisco',\n 'location': r.point(-122.423246,37.779388)\n }).run(conn)\n"), - (rethinkdb.polygon, b"r.polygon([lon1, lat1], [lon2, lat2], ...) -> polygon\nr.polygon(point1, point2, ...) -> polygon\n\nConstruct a geometry object of type Polygon. The Polygon can be specified in one of two ways:\n\n* Three or more two-item arrays, specifying latitude and longitude numbers of the polygon's vertices;\n* Three or more [Point](http://rethinkdb.com/api/python/point) objects specifying the polygon's vertices.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\nIf the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them. You cannot directly construct a polygon with holes in it using `polygon`, but you can use [polygon_sub](http://rethinkdb.com/api/python/polygon_sub) to use a second polygon within the interior of the first to define a hole.\n\n*Example* Define a polygon.\n\n r.table('geo').insert({\n 'id': 101,\n 'rectangle': r.polygon(\n [-122.423246,37.779388],\n [-122.423246,37.329898],\n [-121.886420,37.329898],\n [-121.886420,37.779388]\n )\n }).run(conn)\n"), - (rethinkdb.ast.RqlQuery.polygon_sub, b'polygon1.polygon_sub(polygon2) -> polygon\n\nUse `polygon2` to "punch out" a hole in `polygon1`. `polygon2` must be completely contained within `polygon1` and must have no holes itself (it must not be the output of `polygon_sub` itself).\n\n*Example* Define a polygon with a hole punched in it.\n\n outer_polygon = r.polygon(\n [-122.4,37.7],\n [-122.4,37.3],\n [-121.8,37.3],\n [-121.8,37.7]\n )\n inner_polygon = r.polygon(\n [-122.3,37.4],\n [-122.3,37.6],\n [-122.0,37.6],\n [-122.0,37.4]\n )\n outer_polygon.polygon_sub(inner_polygon).run(conn)\n'), - (rethinkdb.ast.RqlQuery.to_geojson, b"geometry.to_geojson() -> object\n\nConvert a ReQL geometry object to a [GeoJSON][] object.\n\n[GeoJSON]: http://geojson.org\n\n*Example* Convert a ReQL geometry object to a GeoJSON object.\n\n > r.table(geo).get('sfo')['location'].to_geojson().run(conn)\n \n {\n 'type': 'Point',\n 'coordinates': [ -122.423246, 37.779388 ]\n }\n"), - (rethinkdb.ast.RqlQuery.eq_join, b'sequence.eq_join(left_field, right_table[, index=\'id\']) -> sequence\n\nJoin tables using a field on the left-hand sequence matching primary keys or secondary indexes on the right-hand table. `eq_join` is more efficient than other Re_qL join types, and operates much faster. Documents in the result set consist of pairs of left-hand and right-hand documents, matched when the field on the left-hand side exists and is non-null and an entry with that field\'s value exists in the specified index on the right-hand side.\n\nThe result set of `eq_join` is a stream or array of objects. Each object in the returned set will be an object of the form `{ left: <left-document>, right: <right-document> }`, where the values of `left` and `right` will be the joined documents. Use the <code><a href="/api/python/zip/">zip</a></code> command to merge the `left` and `right` fields together.\n\n**Example:** Match players with the games they\'ve played against one another.\n\nThe players table contains these documents:\n\n [\n { \'id\': 1, \'player\': \'George\', \'gameId\': 1 },\n { \'id\': 2, \'player\': \'Agatha\', \'gameId\': 3 },\n { \'id\': 3, \'player\': \'Fred\', \'gameId\': 2 },\n { \'id\': 4, \'player\': \'Marie\', \'gameId\': 2 },\n { \'id\': 5, \'player\': \'Earnest\', \'gameId\': 1 },\n { \'id\': 6, \'player\': \'Beth\', \'gameId\': 3 }\n ]\n\nThe games table contains these documents:\n\n [\n { \'id\': 1, \'field\': \'Little Delving\' },\n { \'id\': 2, \'field\': \'Rushock Bog\' },\n { \'id\': 3, \'field\': \'Bucklebury\' }\n ]\n\nJoin these tables using `game_id` on the player table and `id` on the games table:\n\n r.table(\'players\').eq_join(\'game_id\', r.table(\'games\')).run(conn)\n\nThis will return a result set such as the following:\n\n [\n {\n "left" : { "gameId" : 3, "id" : 2, "player" : "Agatha" },\n "right" : { "id" : 3, "field" : "Bucklebury" }\n },\n {\n "left" : { "gameId" : 2, "id" : 3, "player" : "Fred" },\n "right" : { "id" : 2, "field" : "Rushock Bog" }\n },\n ...\n ]\n\nWhat you likely want is the result of using `zip` with that. For clarity, we\'ll use `without` to drop the `id` field from the games table (it conflicts with the `id` field for the players and it\'s redundant anyway), and we\'ll order it by the games.\n\n r.table(\'players\').eq_join(\'game_id\', r.table(\'games\')).without({\'right\': "id"}).zip().order_by(\'game_id\').run(conn)\n \n [\n { "field": "Little Delving", "gameId": 1, "id": 5, "player": "Earnest" },\n { "field": "Little Delving", "gameId": 1, "id": 1, "player": "George" },\n { "field": "Rushock Bog", "gameId": 2, "id": 3, "player": "Fred" },\n { "field": "Rushock Bog", "gameId": 2, "id": 4, "player": "Marie" },\n { "field": "Bucklebury", "gameId": 3, "id": 6, "player": "Beth" },\n { "field": "Bucklebury", "gameId": 3, "id": 2, "player": "Agatha" }\n ]\n\nFor more information, see [Table joins in Rethink_dB](http://rethinkdb.com/docs/table-joins/).\n\n**Example:** Use a secondary index on the right table rather than the primary key. If players have a secondary index on their cities, we can get a list of arenas with players in the same area.\n\n r.table(\'arenas\').eq_join(\'city_id\', r.table(\'arenas\'), index=\'city_id\').run(conn)\n\n**Example:** Use a nested key as the join field. Suppose the documents in the players table were structured like this:\n\n { \'id\': 1, \'player\': \'George\', \'game\': {\'id\': 1} },\n { \'id\': 2, \'player\': \'Agatha\', \'game\': {\'id\': 3} },\n ...\n\nSimply specify the field using the `row` command instead of a string.\n\n r.table(\'players\').eq_join(r.row[\'game\'][\'id\'], r.table(\'games\')).without({\'right\': \'id\'}).zip().run(conn)\n \n [\n { "field": "Little Delving", "game": { "id": 1 }, "id": 5, "player": "Earnest" },\n { "field": "Little Delving", "game": { "id": 1 }, "id": 1, "player": "George" },\n ...\n ]\n\n**Example:** Use a function instead of a field to join on a more complicated expression. Suppose the players have lists of favorite games ranked in order in a field such as `"favorites": [3, 2, 1]`. Get a list of players and their top favorite:\n\n r.table(\'players3\').eq_join(\n lambda player: player[\'favorites\'].nth(0),\n r.table(\'games\')\n ).without([{\'left\': [\'favorites\', \'game_id\', \'id\']}, {\'right\': \'id\'}]).zip()\n\nResult:\n\n [\n \t{ "field": "Rushock Bog", "name": "Fred" },\n \t{ "field": "Little Delving", "name": "George" },\n \t...\n ]\n'), - (rethinkdb.ast.RqlQuery.inner_join, b"sequence.inner_join(other_sequence, predicate) -> stream\narray.inner_join(other_sequence, predicate) -> array\n\nReturns an inner join of two sequences. The returned sequence represents an intersection of the left-hand sequence and the right-hand sequence: each row of the left-hand sequence will be compared with each row of the right-hand sequence to find all pairs of rows which satisfy the predicate. Each matched pair of rows of both sequences are combined into a result row. In most cases, you will want to follow the join with [zip](http://rethinkdb.com/api/python/zip) to combine the left and right results.\n\nNote that `inner_join` is slower and much less efficient than using [eq_join](http://rethinkdb.com/api/python/eq_join/) or [concat_map](http://rethinkdb.com/api/python/concat_map/) with [get_all](http://rethinkdb.com/api/python/get_all/). You should avoid using `inner_join` in commands when possible.\n\n*Example* Return a list of all matchups between Marvel and DC heroes in which the DC hero could beat the Marvel hero in a fight.\n\n r.table('marvel').inner_join(r.table('dc'),\n lambda marvel_row, dc_row: marvel_row['strength'] < dc_row['strength']\n ).zip().run(conn)\n\n(Compare this to an [outer_join](http://rethinkdb.com/api/python/outer_join) with the same inputs and predicate, which would return a list of *all* Marvel heroes along with any DC heroes with a higher strength.)"), - (rethinkdb.ast.RqlQuery.outer_join, b"sequence.outer_join(other_sequence, predicate) -> stream\narray.outer_join(other_sequence, predicate) -> array\n\nReturns a left outer join of two sequences. The returned sequence represents a union of the left-hand sequence and the right-hand sequence: all documents in the left-hand sequence will be returned, each matched with a document in the right-hand sequence if one satisfies the predicate condition. In most cases, you will want to follow the join with [zip](http://rethinkdb.com/api/python/zip) to combine the left and right results.\n\nNote that `outer_join` is slower and much less efficient than using [concat_map](http://rethinkdb.com/api/python/concat_map/) with [get_all](http://rethinkdb.com/api/python/get_all). You should avoid using `outer_join` in commands when possible.\n\n*Example* Return a list of all Marvel heroes, paired with any DC heroes who could beat them in a fight.\n\n r.table('marvel').outer_join(r.table('dc'),\n lambda marvel_row, dc_row: marvel_row['strength'] < dc_row['strength']\n ).zip().run(conn)\n\n(Compare this to an [inner_join](http://rethinkdb.com/api/python/inner_join) with the same inputs and predicate, which would return a list only of the matchups in which the DC hero has the higher strength.)\n"), - (rethinkdb.ast.RqlQuery.zip, b"stream.zip() -> stream\narray.zip() -> array\n\nUsed to 'zip' up the result of a join by merging the 'right' fields into 'left' fields of each member of the sequence.\n\n*Example* 'zips up' the sequence by merging the left and right fields produced by a join.\n\n r.table('marvel').eq_join('main_dc_collaborator', r.table('dc')).zip().run(conn)\n"), - (rethinkdb.db_create, b'r.db_create(db_name) -> object\n\nCreate a database. A RethinkDB database is a collection of tables, similar to\nrelational databases.\n\nIf successful, the command returns an object with two fields:\n\n* `dbs_created`: always `1`.\n* `config_changes`: a list containing one object with two fields, `old_val` and `new_val`:\n * `old_val`: always `None`.\n * `new_val`: the database\'s new [config](http://rethinkdb.com/api/python/config) value.\n\nIf a database with the same name already exists, the command throws `RqlRuntimeError`.\n\nNote: Only alphanumeric characters and underscores are valid for the database name.\n\n*Example* Create a database named \'superheroes\'.\n\n r.db_create(\'superheroes\').run(conn)\n \n {\n "config_changes": [\n {\n "new_val": {\n "id": "e4689cfc-e903-4532-a0e6-2d6797a43f07",\n "name": "superheroes"\n },\n "old_val": None\n }\n ],\n "dbs_created": 1\n }\n\n'), - (rethinkdb.db_drop, b'r.db_drop(db_name) -> object\n\nDrop a database. The database, all its tables, and corresponding data will be deleted.\n\nIf successful, the command returns an object with two fields:\n\n* `dbs_dropped`: always `1`.\n* `tables_dropped`: the number of tables in the dropped database.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: the database\'s original [config](http://rethinkdb.com/api/python/config) value.\n * `new_val`: always `None`.\n\nIf the given database does not exist, the command throws `RqlRuntimeError`.\n\n*Example* Drop a database named \'superheroes\'.\n\n r.db_drop(\'superheroes\').run(conn)\n \n {\n "config_changes": [\n {\n "old_val": {\n "id": "e4689cfc-e903-4532-a0e6-2d6797a43f07",\n "name": "superheroes"\n },\n "new_val": None\n }\n ],\n "tables_dropped": 3,\n "dbs_dropped": 1\n }\n\n'), - (rethinkdb.db_list, b'r.db_list() -> array\n\nList all database names in the system. The result is a list of strings.\n\n*Example* List all databases.\n\n r.db_list().run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.changes, b'stream.changes(squash=True, include_states=False) -> stream\nsingleSelection.changes(squash=True, include_states=False) -> stream\n\nReturn an infinite stream of objects representing changes to a query.\n\nThe `squash` optional argument controls how `changes` batches change notifications:\n\n* `True`: When multiple changes to the same document occur before a batch of notifications is sent, the changes are "squashed" into one change. The client receives a notification that will bring it fully up to date with the server. This is the default.\n* `False`: All changes will be sent to the client verbatim.\n* `n`: A numeric value (floating point). Similar to `True`, but the server will wait `n` seconds to respond in order to squash as many changes together as possible, reducing network traffic.\n\nIf the `include_states` optional argument is `True`, the changefeed stream will include special status documents consisting of the field `state` and a string indicating a change in the feed\'s state. These documents can occur at any point in the feed between the notification documents described below. There are currently two states:\n\n* `{"state": "initializing"}` indicates the following documents represent initial values on the feed rather than changes. This will be the first document of a feed that returns initial values.\n* `{"state": "ready"}` indicates the following documents represent changes. This will be the first document of a feed that does *not* return initial values; otherwise, it will indicate the initial values have all been sent.\n\nPoint changefeeds will always return initial values and have an `initializing` state; feeds that return changes on unfiltered tables will never return initial values. Feeds that return changes on more complex queries may or may not return return initial values, depending on the kind of aggregation. Read the article on [Changefeeds in RethinkDB][cfr] for a more detailed discussion. If `include_states` is `True` on a changefeed that does not return initial values, the first document on the feed will be `{"state": "ready"}`.\n\n[cfr]: /docs/changefeeds/python/\n\nIf `include_states` is `False` (the default), the status documents will not be sent on the feed.\n\nIf the table becomes unavailable, the changefeed will be disconnected, and a runtime exception will be thrown by the driver.\n\nChangefeed notifications take the form of a two-field object:\n\n {\n "old_val": <document before change>,\n "new_val": <document after change>\n }\n\nThe first notification object in the changefeed stream will contain the query\'s initial value in `new_val` and have no `old_val` field. When a document is deleted, `new_val` will be `None`; when a document is inserted, `old_val` will be `None`.\n\nCertain document transformation commands can be chained before changefeeds. For more information, read the [discussion of changefeeds][cfr] in the "Query language" documentation.\n\nThe server will buffer up to 100,000 elements. If the buffer limit is hit, early changes will be discarded, and the client will receive an object of the form `{"error": "Changefeed cache over array size limit, skipped X elements."}` where `X` is the number of elements skipped.\n\nCommands that operate on streams (such as `filter` or `map`) can usually be chained after `changes`. However, since the stream produced by `changes` has no ending, commands that need to consume the entire stream before returning (such as `reduce` or `count`) cannot.\n\nIt\'s a good idea to open changefeeds on their own connection. If you don\'t, other queries run on the same connection will experience unpredictable latency spikes while the connection blocks on more changes.\n\n*Example* Subscribe to the changes on a table.\n\nStart monitoring the changefeed in one client:\n\n for change in r.table(\'games\').changes().run(conn):\n print change\n\nAs these queries are performed in a second client, the first client would receive and print the following objects:\n\n > r.table(\'games\').insert({\'id\': 1}).run(conn)\n {\'old_val\': None, \'new_val\': {\'id\': 1}}\n \n > r.table(\'games\').get(1).update({\'player1\': \'Bob\'}).run(conn)\n {\'old_val\': {\'id\': 1}, \'new_val\': {\'id\': 1, \'player1\': \'Bob\'}}\n \n > r.table(\'games\').get(1).replace({\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}).run(conn)\n {\'old_val\': {\'id\': 1, \'player1\': \'Bob\'},\n \'new_val\': {\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}}\n \n > r.table(\'games\').get(1).delete().run(conn)\n {\'old_val\': {\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}, \'new_val\': None}\n \n > r.table_drop(\'games\').run(conn)\n RqlRuntimeError: Changefeed aborted (table unavailable)\n\n*Example* Return all the changes that increase a player\'s score.\n\n r.table(\'test\').changes().filter(\n r.row[\'new_val\'][\'score\'] > r.row[\'old_val\'][\'score\']\n ).run(conn)\n\n*Example* Return all the changes to Bob\'s score.\n\n # Note that this will have to look at and discard all the changes to\n # rows besides Bob\'s. This is currently no way to filter with an index\n # on changefeeds.\n r.table(\'test\').changes().filter(r.row[\'new_val\'][\'name\'].eq(\'Bob\')).run(conn)\n\n*Example* Return all the inserts on a table.\n\n r.table(\'test\').changes().filter(r.row[\'old_val\'].eq(None)).run(conn)\n\n*Example* Return all the changes to game 1, with state notifications.\n\n r.table(\'games\').get(1).changes(include_states=True).run(conn)\n \n # result returned on changefeed\n {"state": "initializing"}\n {"new_val": {"id": 1, "score": 12, "arena": "Hobbiton Field"}}\n {"state": "ready"}\n {\n \t"old_val": {"id": 1, "score": 12, "arena": "Hobbiton Field"},\n \t"new_val": {"id": 1, "score": 14, "arena": "Hobbiton Field"}\n }\n {\n \t"old_val": {"id": 1, "score": 14, "arena": "Hobbiton Field"},\n \t"new_val": {"id": 1, "score": 17, "arena": "Hobbiton Field", "winner": "Frodo"}\n }\n\n*Example* Return all the changes to the top 10 games. This assumes the presence of a `score` secondary index on the `games` table.\n\n r.table(\'games\').order_by(index=r.desc(\'score\')).limit(10).run(conn)\n'), - (rethinkdb.ast.Table.index_create, b'table.index_create(index_name[, index_function][, multi=False, geo=False]) -> object\n\nCreate a new secondary index on a table. Secondary indexes improve the speed of many read queries at the slight cost of increased storage space and decreased write performance. For more information about secondary indexes, read the article "[Using secondary indexes in RethinkDB](http://rethinkdb.com/docs/secondary-indexes/)."\n\nRethinkDB supports different types of secondary indexes:\n\n- *Simple indexes* based on the value of a single field.\n- *Compound indexes* based on multiple fields.\n- *Multi indexes* based on arrays of values.\n- *Geospatial indexes* based on indexes of geometry objects, created when the `geo` optional argument is true.\n- Indexes based on *arbitrary expressions*.\n\nThe `index_function` can be an anonymous function or a binary representation obtained from the `function` field of [index_status](http://rethinkdb.com/api/python/index_status).\n\nIf successful, `create_index` will return an object of the form `{"created": 1}`. If an index by that name already exists on the table, a `RqlRuntimeError` will be thrown.\n\n*Example* Create a simple index based on the field `post_id`.\n\n r.table(\'comments\').index_create(\'post_id\').run(conn)\n*Example* Create a simple index based on the nested field `author > name`.\n\n r.table(\'comments\').index_create(\'author_name\', r.row["author"]["name"]).run(conn)\n\n*Example* Create a geospatial index based on the field `location`.\n\n r.table(\'places\').index_create(\'location\', geo=True).run(conn)\n\nA geospatial index field should contain only geometry objects. It will work with geometry ReQL terms ([get_intersecting](http://rethinkdb.com/api/python/get_intersecting/) and [get_nearest](http://rethinkdb.com/api/python/get_nearest/)) as well as index-specific terms ([index_status](http://rethinkdb.com/api/python/index_status), [index_wait](http://rethinkdb.com/api/python/index_wait), [index_drop](http://rethinkdb.com/api/python/index_drop) and [index_list](http://rethinkdb.com/api/python/index_list)). Using terms that rely on non-geometric ordering such as [get_all](http://rethinkdb.com/api/python/get_all/), [order_by](http://rethinkdb.com/api/python/order_by/) and [between](http://rethinkdb.com/api/python/order_by/) will result in an error.\n\n*Example* Create a compound index based on the fields `post_id` and `date`.\n\n r.table(\'comments\').index_create(\'post_and_date\', [r.row["post_id"], r.row["date"]]).run(conn)\n\n*Example* Create a multi index based on the field `authors`.\n\n r.table(\'posts\').index_create(\'authors\', multi=True).run(conn)\n\n*Example* Create a geospatial multi index based on the field `towers`.\n\n r.table(\'networks\').index_create(\'towers\', geo=True, multi=True).run(conn)\n\n*Example* Create an index based on an arbitrary expression.\n\n r.table(\'posts\').index_create(\'authors\', lambda doc:\n r.branch(\n doc.has_fields("updated_at"),\n doc["updated_at"],\n doc["created_at"]\n )\n ).run(conn)\n\n*Example* Create a new secondary index based on an existing one.\n\n index = r.table(\'posts\').index_status(\'authors\').nth(0)[\'function\'].run(conn)\n r.table(\'new_posts\').index_create(\'authors\', index).run(conn)\n\n*Example* Rebuild an outdated secondary index on a table.\n\n old_index = r.table(\'posts\').index_status(\'old_index\').nth(0)[\'function\'].run(conn)\n r.table(\'posts\').index_create(\'new_index\', old_index).run(conn)\n r.table(\'posts\').index_wait(\'new_index\').run(conn)\n r.table(\'posts\').index_rename(\'new_index\', \'old_index\', overwrite=True).run(conn)\n'), - (rethinkdb.ast.Table.index_drop, b"table.index_drop(index_name) -> object\n\nDelete a previously created secondary index of this table.\n\n*Example* Drop a secondary index named 'code_name'.\n\n r.table('dc').index_drop('code_name').run(conn)\n\n"), - (rethinkdb.ast.Table.index_list, b"table.index_list() -> array\n\nList all the secondary indexes of this table.\n\n*Example* List the available secondary indexes for this table.\n\n r.table('marvel').index_list().run(conn)\n"), - (rethinkdb.ast.Table.index_rename, b"table.index_rename(old_index_name, new_index_name[, overwrite=False]) -> object\n\nRename an existing secondary index on a table. If the optional argument `overwrite` is specified as `True`, a previously existing index with the new name will be deleted and the index will be renamed. If `overwrite` is `False` (the default) an error will be raised if the new index name already exists.\n\nThe return value on success will be an object of the format `{'renamed': 1}`, or `{'renamed': 0}` if the old and new names are the same.\n\nAn error will be raised if the old index name does not exist, if the new index name is already in use and `overwrite` is `False`, or if either the old or new index name are the same as the primary key field name.\n\n*Example* Rename an index on the comments table.\n\n r.table('comments').index_rename('post_id', 'message_id').run(conn)\n"), - (rethinkdb.ast.Table.index_status, b'table.index_status([, index...]) -> array\n\nGet the status of the specified indexes on this table, or the status\nof all indexes on this table if no indexes are specified.\n\nThe result is an array where for each index, there will be an object like this one:\n\n {\n "index": <index_name>,\n "ready": True,\n "function": <binary>,\n "multi": <bool>,\n "outdated": <bool>\n }\n\nor this one:\n\n {\n "index": <index_name>,\n "ready": False,\n "blocks_processed": <int>,\n "blocks_total": <int>,\n "function": <binary>,\n "multi": <bool>,\n "outdated": <bool>\n }\n\nThe `multi` field will be `true` or `false` depending on whether this index was created as a multi index (see [index_create](http://rethinkdb.com/api/python/index_create/) for details). The `outdated` field will be true if the index is outdated in the current version of RethinkDB and needs to be rebuilt.\n\nThe `function` field is a binary object containing an opaque representation of the secondary index (including the `multi` argument if specified). It can be passed as the second argument to [index_create](http://rethinkdb.com/api/python/index_create/) to create a new index with the same function; see `index_create` for more information.\n\n*Example* Get the status of all the indexes on `test`:\n\n r.table(\'test\').index_status().run(conn)\n\n*Example* Get the status of the `timestamp` index:\n\n r.table(\'test\').index_status(\'timestamp\').run(conn)\n\n*Example* Save the binary representation of the index:\n\n func = r.table(\'test\').index_status(\'timestamp\').nth(0)[\'function\'].run(conn)\n'), - (rethinkdb.ast.Table.index_wait, b'table.index_wait([, index...]) -> array\n\nWait for the specified indexes on this table to be ready, or for all\nindexes on this table to be ready if no indexes are specified.\n\nThe result is an array containing one object for each table index:\n\n {\n "index": <index_name>,\n "ready": True,\n "function": <binary>,\n "multi": <bool>,\n "geo": <bool>,\n "outdated": <bool>\n }\n\nSee the [index_status](http://rethinkdb.com/api/python/index_status) documentation for a description of the field values.\n\n*Example* Wait for all indexes on the table `test` to be ready:\n\n r.table(\'test\').index_wait().run(conn)\n\n*Example* Wait for the index `timestamp` to be ready:\n\n r.table(\'test\').index_wait(\'timestamp\').run(conn)\n'), - (rethinkdb.ast.DB.table_create, b'db.table_create(table_name[, options]) -> object\n\nCreate a table. A RethinkDB table is a collection of JSON documents.\n\nIf successful, the command returns an object with two fields:\n\n* `tables_created`: always `1`.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: always `None`.\n * `new_val`: the table\'s new [config](http://rethinkdb.com/api/python/config) value.\n\nIf a table with the same name already exists, the command throws `RqlRuntimeError`.\n\nNote: Only alphanumeric characters and underscores are valid for the table name.\n\nWhen creating a table you can specify the following options:\n\n* `primary_key`: the name of the primary key. The default primary key is `id`.\n* `durability`: if set to `soft`, writes will be acknowledged by the server immediately and flushed to disk in the background. The default is `hard`: acknowledgment of writes happens after data has been written to disk.\n* `shards`: the number of shards, an integer from 1-32. Defaults to `1`.\n* `replicas`: either an integer or a mapping object. Defaults to `1`.\n * If `replicas` is an integer, it specifies the number of replicas per shard. Specifying more replicas than there are servers will return an error.\n * If `replicas` is an object, it specifies key-value pairs of server tags and the number of replicas to assign to those servers: `{\'tag1\': 2, \'tag2\': 4, \'tag3\': 2, ...}`.\n* `primary_replica_tag`: the primary server specified by its server tag. Required if `replicas` is an object; the tag must be in the object. This must *not* be specified if `replicas` is an integer.\n\nThe [data type](http://rethinkdb.com/docs/data-types/) of a primary key is usually a string (like a UUID) or a number, but it can also be a time, binary object, boolean or an array. It cannot be an object.\n\n*Example* Create a table named \'dc_universe\' with the default settings.\n\n r.db(\'test\').table_create(\'dc_universe\').run(conn)\n \n {\n "config_changes": [\n {\n "new_val": {\n "db": "test",\n "durability": "hard",\n "id": "20ea60d4-3b76-4817-8828-98a236df0297",\n "name": "dc_universe",\n "primary_key": "id",\n "shards": [\n {\n "primary_replica": "rethinkdb_srv1",\n "replicas": [\n "rethinkdb_srv1",\n "rethinkdb_srv2"\n ]\n }\n ],\n "write_acks": "majority"\n },\n "old_val": None\n }\n ],\n "tables_created": 1\n }\n\n*Example* Create a table named \'dc_universe\' using the field \'name\' as primary key.\n\n r.db(\'test\').table_create(\'dc_universe\', primary_key=\'name\').run(conn)\n\n*Example* Create a table set up for two shards and three replicas per shard. This requires three available servers.\n\n r.db(\'test\').table_create(\'dc_universe\', shards=2, replicas=3).run(conn)\n\nRead [Sharding and replication](http://rethinkdb.com/docs/sharding-and-replication/) for a complete discussion of the subject, including advanced topics.\n'), - (rethinkdb.ast.DB.table_drop, b'db.table_drop(table_name) -> object\n\nDrop a table. The table and all its data will be deleted.\n\nIf successful, the command returns an object with two fields:\n\n* `tables_dropped`: always `1`.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: the dropped table\'s [config](http://rethinkdb.com/api/python/config) value.\n * `new_val`: always `None`.\n\nIf the given table does not exist in the database, the command throws `RqlRuntimeError`.\n\n*Example* Drop a table named \'dc_universe\'.\n\n r.db(\'test\').table_drop(\'dc_universe\').run(conn)\n \n {\n "config_changes": [\n {\n "old_val": {\n "db": "test",\n "durability": "hard",\n "id": "20ea60d4-3b76-4817-8828-98a236df0297",\n "name": "dc_universe",\n "primary_key": "id",\n "shards": [\n {\n "primary_replica": "rethinkdb_srv1",\n "replicas": [\n "rethinkdb_srv1",\n "rethinkdb_srv2"\n ]\n }\n ],\n "write_acks": "majority"\n },\n "new_val": None\n }\n ],\n "tables_dropped": 1\n }\n'), - (rethinkdb.ast.DB.table_list, b"db.table_list() -> array\n\nList all table names in a database. The result is a list of strings.\n\n*Example* List all tables of the 'test' database.\n\n r.db('test').table_list().run(conn)\n \n"), - (rethinkdb.ast.RqlQuery.__add__, b'number + number -> number\nstring + string -> string\narray + array -> array\ntime + number -> time\n\nSum two numbers, concatenate two strings, or concatenate 2 arrays.\n\n*Example* It\'s as easy as 2 + 2 = 4.\n\n > (r.expr(2) + 2).run(conn)\n \n 4\n\n*Example* Strings can be concatenated too.\n\n > (r.expr("foo") + "bar").run(conn)\n \n "foobar"\n\n*Example* Arrays can be concatenated too.\n\n > (r.expr(["foo", "bar"]) + ["buzz"]).run(conn)\n \n [\'foo\', \'bar\', \'buzz\']\n\n*Example* Create a date one year from now.\n\n r.now() + 365*24*60*60\n\n*Example* Use [args](http://rethinkdb.com/api/python/args) with `add` to sum multiple values.\n\n > r.add(r.args([10, 20, 30])).run(conn)\n \n 60\n\n*Example* Concatenate an array of strings with `args`.\n\n > r.add(r.args([\'foo\', \'bar\', \'buzz\'])).run(conn)\n \n "foobarbuzz"\n'), - (rethinkdb.add, b'number + number -> number\nstring + string -> string\narray + array -> array\ntime + number -> time\n\nSum two numbers, concatenate two strings, or concatenate 2 arrays.\n\n*Example* It\'s as easy as 2 + 2 = 4.\n\n > (r.expr(2) + 2).run(conn)\n \n 4\n\n*Example* Strings can be concatenated too.\n\n > (r.expr("foo") + "bar").run(conn)\n \n "foobar"\n\n*Example* Arrays can be concatenated too.\n\n > (r.expr(["foo", "bar"]) + ["buzz"]).run(conn)\n \n [\'foo\', \'bar\', \'buzz\']\n\n*Example* Create a date one year from now.\n\n r.now() + 365*24*60*60\n\n*Example* Use [args](http://rethinkdb.com/api/python/args) with `add` to sum multiple values.\n\n > r.add(r.args([10, 20, 30])).run(conn)\n \n 60\n\n*Example* Concatenate an array of strings with `args`.\n\n > r.add(r.args([\'foo\', \'bar\', \'buzz\'])).run(conn)\n \n "foobarbuzz"\n'), - (rethinkdb.ast.RqlQuery.__and__, b'bool & bool -> bool\nr.and_(bool, bool) -> bool\nbool.and_(bool) -> bool\n\nCompute the logical "and" of two or more values. The `and_` command can be used as an infix operator after its first argument (`r.expr(True).and_(False)`) or given all of its arguments as parameters (`r.and_(True, False)`). The standard Python and operator, `&`, may also be used with ReQL.\n\n*Example* Return whether both `a` and `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) & b).run(conn)\n \n False\n*Example* Return whether all of `x`, `y` and `z` evaluate to true.\n\n > x = True\n > y = True\n > z = True\n > r.and_(x, y, z).run(conn)\n \n True\n'), - (rethinkdb.and_, b'bool & bool -> bool\nr.and_(bool, bool) -> bool\nbool.and_(bool) -> bool\n\nCompute the logical "and" of two or more values. The `and_` command can be used as an infix operator after its first argument (`r.expr(True).and_(False)`) or given all of its arguments as parameters (`r.and_(True, False)`). The standard Python and operator, `&`, may also be used with ReQL.\n\n*Example* Return whether both `a` and `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) & b).run(conn)\n \n False\n*Example* Return whether all of `x`, `y` and `z` evaluate to true.\n\n > x = True\n > y = True\n > z = True\n > r.and_(x, y, z).run(conn)\n \n True\n'), - (rethinkdb.ast.RqlQuery.__div__, b"number / number -> number\n\nDivide two numbers.\n\n*Example* It's as easy as 2 / 2 = 1.\n\n (r.expr(2) / 2).run(conn)\n"), - (rethinkdb.div, b"number / number -> number\n\nDivide two numbers.\n\n*Example* It's as easy as 2 / 2 = 1.\n\n (r.expr(2) / 2).run(conn)\n"), - (rethinkdb.ast.RqlQuery.__eq__, b'value == value -> bool\nvalue.eq(value) -> bool\n\nTest if two values are equal.\n\n*Example* Does 2 equal 2?\n\n (r.expr(2) == 2).run(conn)\n r.expr(2).eq(2).run(conn)\n'), - (rethinkdb.ast.RqlQuery.eq, b'value == value -> bool\nvalue.eq(value) -> bool\n\nTest if two values are equal.\n\n*Example* Does 2 equal 2?\n\n (r.expr(2) == 2).run(conn)\n r.expr(2).eq(2).run(conn)\n'), - (rethinkdb.ast.RqlQuery.__ge__, b'value >= value -> bool\nvalue.ge(value) -> bool\n\nTest if the first value is greater than or equal to other.\n\n*Example* Is 2 greater than or equal to 2?\n\n (r.expr(2) >= 2).run(conn)\n r.expr(2).ge(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.ge, b'value >= value -> bool\nvalue.ge(value) -> bool\n\nTest if the first value is greater than or equal to other.\n\n*Example* Is 2 greater than or equal to 2?\n\n (r.expr(2) >= 2).run(conn)\n r.expr(2).ge(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.__gt__, b'value > value -> bool\nvalue.gt(value) -> bool\n\nTest if the first value is greater than other.\n\n*Example* Is 2 greater than 2?\n\n (r.expr(2) > 2).run(conn)\n r.expr(2).gt(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.gt, b'value > value -> bool\nvalue.gt(value) -> bool\n\nTest if the first value is greater than other.\n\n*Example* Is 2 greater than 2?\n\n (r.expr(2) > 2).run(conn)\n r.expr(2).gt(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.__le__, b'value <= value -> bool\nvalue.le(value) -> bool\n\nTest if the first value is less than or equal to other.\n\n*Example* Is 2 less than or equal to 2?\n\n (r.expr(2) <= 2).run(conn)\n r.expr(2).le(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.le, b'value <= value -> bool\nvalue.le(value) -> bool\n\nTest if the first value is less than or equal to other.\n\n*Example* Is 2 less than or equal to 2?\n\n (r.expr(2) <= 2).run(conn)\n r.expr(2).le(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.__lt__, b'value < value -> bool\nvalue.lt(value) -> bool\n\nTest if the first value is less than other.\n\n*Example* Is 2 less than 2?\n\n (r.expr(2) < 2).run(conn)\n r.expr(2).lt(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.lt, b'value < value -> bool\nvalue.lt(value) -> bool\n\nTest if the first value is less than other.\n\n*Example* Is 2 less than 2?\n\n (r.expr(2) < 2).run(conn)\n r.expr(2).lt(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.__mod__, b"number % number -> number\n\nFind the remainder when dividing two numbers.\n\n*Example* It's as easy as 2 % 2 = 0.\n\n (r.expr(2) % 2).run(conn)\n\n`\n"), - (rethinkdb.mod, b"number % number -> number\n\nFind the remainder when dividing two numbers.\n\n*Example* It's as easy as 2 % 2 = 0.\n\n (r.expr(2) % 2).run(conn)\n\n`\n"), - (rethinkdb.ast.RqlQuery.__mul__, b'number * number -> number\narray * number -> array\n\nMultiply two numbers, or make a periodic array.\n\n*Example* It\'s as easy as 2 * 2 = 4.\n\n (r.expr(2) * 2).run(conn)\n\n*Example* Arrays can be multiplied by numbers as well.\n\n (r.expr(["This", "is", "the", "song", "that", "never", "ends."]) * 100).run(conn)\n\n'), - (rethinkdb.mul, b'number * number -> number\narray * number -> array\n\nMultiply two numbers, or make a periodic array.\n\n*Example* It\'s as easy as 2 * 2 = 4.\n\n (r.expr(2) * 2).run(conn)\n\n*Example* Arrays can be multiplied by numbers as well.\n\n (r.expr(["This", "is", "the", "song", "that", "never", "ends."]) * 100).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.__ne__, b'value != value -> bool\nvalue.ne(value) -> bool\n\nTest if two values are not equal.\n\n*Example* Does 2 not equal 2?\n\n (r.expr(2) != 2).run(conn)\n r.expr(2).ne(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.ne, b'value != value -> bool\nvalue.ne(value) -> bool\n\nTest if two values are not equal.\n\n*Example* Does 2 not equal 2?\n\n (r.expr(2) != 2).run(conn)\n r.expr(2).ne(2).run(conn)\n\n'), - (rethinkdb.ast.RqlQuery.__invert__, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), - (rethinkdb.ast.RqlQuery.not_, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), - (rethinkdb.not_, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), - (rethinkdb.ast.RqlQuery.__or__, b'bool | bool -> bool\nbool.or_(bool[, bool, ...]) -> bool\nr.or_(bool, bool) -> bool\n\nCompute the logical "or" of two or more values. The `or_` command can be used as an infix operator after its first argument (`r.expr(True).or_(False)`) or given all of its arguments as parameters (`r.or_(True, False)`). The standard Python or operator, `|`, may also be used with ReQL.\n\n*Example* Return whether either `a` or `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) | b).run(conn)\n \n True\n\n*Example* Return whether any of `x`, `y` or `z` evaluate to true.\n\n > x = False\n > y = False\n > z = False\n > r.or_(x, y, z).run(conn)\n \n False\n\n__Note:__ When using `or` inside a `filter` predicate to test the values of fields that may not exist on the documents being tested, you should use the `default` command with those fields so they explicitly return `False`.\n\n r.table(\'posts\').filter(lambda post:\n post[\'category\'].default(\'foo\').eq(\'article\').or(\n post[\'genre\'].default(\'foo\').eq(\'mystery\'))\n ).run(conn)\n'), - (rethinkdb.or_, b'bool | bool -> bool\nbool.or_(bool[, bool, ...]) -> bool\nr.or_(bool, bool) -> bool\n\nCompute the logical "or" of two or more values. The `or_` command can be used as an infix operator after its first argument (`r.expr(True).or_(False)`) or given all of its arguments as parameters (`r.or_(True, False)`). The standard Python or operator, `|`, may also be used with ReQL.\n\n*Example* Return whether either `a` or `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) | b).run(conn)\n \n True\n\n*Example* Return whether any of `x`, `y` or `z` evaluate to true.\n\n > x = False\n > y = False\n > z = False\n > r.or_(x, y, z).run(conn)\n \n False\n\n__Note:__ When using `or` inside a `filter` predicate to test the values of fields that may not exist on the documents being tested, you should use the `default` command with those fields so they explicitly return `False`.\n\n r.table(\'posts\').filter(lambda post:\n post[\'category\'].default(\'foo\').eq(\'article\').or(\n post[\'genre\'].default(\'foo\').eq(\'mystery\'))\n ).run(conn)\n'), - (rethinkdb.random, b"r.random() -> number\nr.random(number[, number], float=True) -> number\nr.random(integer[, integer]) -> integer\n\nGenerate a random number between given (or implied) bounds. `random` takes zero, one or two arguments.\n\n- With __zero__ arguments, the result will be a floating-point number in the range `[0,1)` (from 0 up to but not including 1).\n- With __one__ argument _x,_ the result will be in the range `[0,x)`, and will be integer unless `float=True` is given as an option. Specifying a floating point number without the `float` option will raise an error.\n- With __two__ arguments _x_ and _y,_ the result will be in the range `[x,y)`, and will be integer unless `float=True` is given as an option. If _x_ and _y_ are equal an error will occur, unless the floating-point option has been specified, in which case _x_ will be returned. Specifying a floating point number without the `float` option will raise an error.\n\nNote: The last argument given will always be the 'open' side of the range, but when\ngenerating a floating-point number, the 'open' side may be less than the 'closed' side.\n\n*Example* Generate a random number in the range `[0,1)`\n\n r.random().run(conn)\n\n*Example* Generate a random integer in the range `[0,100)`\n\n r.random(100).run(conn)\n r.random(0, 100).run(conn)\n\n*Example* Generate a random number in the range `(-2.24,1.59]`\n\n r.random(1.59, -2.24, float=True).run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.__sub__, b"number - number -> number\ntime - time -> number\ntime - number -> time\n\nSubtract two numbers.\n\n*Example* It's as easy as 2 - 2 = 0.\n\n (r.expr(2) - 2).run(conn)\n\n*Example* Create a date one year ago today.\n\n r.now() - 365*24*60*60\n\n*Example* Retrieve how many seconds elapsed between today and date\n\n r.now() - date\n\n"), - (rethinkdb.sub, b"number - number -> number\ntime - time -> number\ntime - number -> time\n\nSubtract two numbers.\n\n*Example* It's as easy as 2 - 2 = 0.\n\n (r.expr(2) - 2).run(conn)\n\n*Example* Create a date one year ago today.\n\n r.now() - 365*24*60*60\n\n*Example* Retrieve how many seconds elapsed between today and date\n\n r.now() - date\n\n"), - (rethinkdb.ast.Table.between, b'table.between(lower_key, upper_key[, index=\'id\', left_bound=\'closed\', right_bound=\'open\'])\n -> selection\n\nGet all documents between two keys. Accepts three optional arguments: `index`, `left_bound`, and `right_bound`. If `index` is set to the name of a secondary index, `between` will return all documents where that index\'s value is in the specified range (it uses the primary key by default). `left_bound` or `right_bound` may be set to `open` or `closed` to indicate whether or not to include that endpoint of the range (by default, `left_bound` is closed and `right_bound` is open).\n\nYou may also use the special constants `r.minval` and `r.maxval` for boundaries, which represent "less than any index key" and "more than any index key" respectively. For instance, if you use `r.minval` as the lower key, then `between` will return all documents whose primary keys (or indexes) are less than the specified upper key.\n\nNote that compound indexes are sorted using [lexicographical order][lo]. Take the following range as an example:\n\n\t[[1, "c"] ... [5, "e"]]\n\nThis range includes all compound keys:\n\n* whose first item is 1 and second item is equal or greater than "c";\n* whose first item is between 1 and 5, *regardless of the value of the second item*;\n* whose first item is 5 and second item is less than or equal to "e".\n\n[lo]: https://en.wikipedia.org/wiki/Lexicographical_order\n\n*Example* Find all users with primary key >= 10 and < 20 (a normal half-open interval).\n\n r.table(\'marvel\').between(10, 20).run(conn)\n\n*Example* Find all users with primary key >= 10 and <= 20 (an interval closed on both sides).\n\n r.table(\'marvel\').between(10, 20, right_bound=\'closed\').run(conn)\n\n*Example* Find all users with primary key < 20.\n\n r.table(\'marvel\').between(r.minval, 20).run(conn)\n\n*Example* Find all users with primary key > 10.\n\n r.table(\'marvel\').between(10, r.maxval, left_bound=\'open\').run(conn)\n\n*Example* Between can be used on secondary indexes too. Just pass an optional index argument giving the secondary index to query.\n\n r.table(\'dc\').between(\'dark_knight\', \'man_of_steel\', index=\'code_name\').run(conn)\n\n*Example* Get all users whose full name is between "John Smith" and "Wade Welles."\n\n r.table("users").between(["Smith", "John"], ["Welles", "Wade"],\n index="full_name").run(conn)\n\n*Example* Subscribe to a [changefeed](http://rethinkdb.com/docs/changefeeds/javascript) of teams ranked in the top 10.\n\n changes = r.table("teams").between(1, 11, index="rank").changes().run(conn)\n\n__Note:__ Between works with secondary indexes on date fields, but will not work with unindexed date fields. To test whether a date value is between two other dates, use the [during](http://rethinkdb.com/api/python/during) command, not `between`.\n\nSecondary indexes can be used in extremely powerful ways with `between` and other commands; read the full article on [secondary indexes](http://rethinkdb.com/docs/secondary-indexes) for examples using boolean operations, `contains` and more.\n\n__Note:__ RethinkDB uses byte-wise ordering for `between` and does not support Unicode collations; non-ASCII characters will be sorted by UTF-8 codepoint.\n\n__Note:__ If you chain `between` after [order_by](http://rethinkdb.com/api/python/order_by), the `between` command must use the index specified in `order_by`, and will default to that index. Trying to specify another index will result in a `RqlRuntimeError`.\n'), - (rethinkdb.db, b"r.db(db_name) -> db\n\nReference a database.\n\n*Example* Before we can query a table we have to select the correct database.\n\n r.db('heroes').table('marvel').run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.filter, b'selection.filter(predicate[, default=False]) -> selection\nstream.filter(predicate[, default=False]) -> stream\narray.filter(predicate[, default=False]) -> array\n\nReturn all the elements in a sequence for which the given predicate is true. The return value of `filter` will be the same as the input (sequence, stream, or array). Documents can be filtered in a variety of ways—ranges, nested values, boolean conditions, and the results of anonymous functions.\n\nBy default, `filter` will silently skip documents with missing fields: if the predicate tries to access a field that doesn\'t exist (for instance, the predicate `{\'age\': 30}` applied to a document with no `age` field), that document will not be returned in the result set, and no error will be generated. This behavior can be changed with the `default` optional argument.\n\n* If `default` is set to `True`, documents with missing fields will be returned rather than skipped.\n* If `default` is set to `r.error()`, an `RqlRuntimeError` will be thrown when a document with a missing field is tested.\n* If `default` is set to `False` (the default), documents with missing fields will be skipped.\n\n*Example* Get all users who are 30 years old.\n\n r.table(\'users\').filter({\'age\': 30}).run(conn)\n\nThe predicate `{\'age\': 30}` selects documents in the `users` table with an `age` field whose value is `30`. Documents with an `age` field set to any other value *or* with no `age` field present are skipped.\n\nWhile the `{\'field\': value}` style of predicate is useful for exact matches, a more general way to write a predicate is to use the [row](http://rethinkdb.com/api/python/row) command with a comparison operator such as [eq](http://rethinkdb.com/api/python/eq) (`==`) or [gt](http://rethinkdb.com/api/python/gt) (`>`), or to use a lambda function that returns `True` or `False`.\n\n r.table(\'users\').filter(r.row["age"] == 30).run(conn)\n\nIn this case, the predicate `r.row["age"] == 30` returns `True` if the field `age` is equal to 30. You can write this predicate as a lambda function instead:\n\n r.table(\'users\').filter(lambda user:\n user["age"] == 30\n ).run(conn)\n\nPredicates to `filter` are evaluated on the server, and must use ReQL expressions. Some Python comparison operators are overloaded by the RethinkDB driver and will be translated to ReQL, such as `==`, `<`/`>` and `|`/`&` (note the single character form, rather than `||`/`&&`).\n\nAlso, predicates must evaluate document fields. They cannot evaluate [secondary indexes](http://rethinkdb.com/docs/secondary-indexes/).\n\n*Example* Get all users who are more than 18 years old.\n\n r.table("users").filter(r.row["age"] > 18).run(conn)\n\n*Example* Get all users who are less than 18 years old and more than 13 years old.\n\n r.table("users").filter((r.row["age"] < 18) & (r.row["age"] > 13)).run(conn)\n\n*Example* Get all users who are more than 18 years old or have their parental consent.\n\n r.table("users").filter(\n (r.row["age"] >= 18) | (r.row["hasParentalConsent"])).run(conn)\n\n*Example* Retrieve all users who subscribed between January 1st, 2012\n(included) and January 1st, 2013 (excluded).\n\n r.table("users").filter(\n lambda user: user["subscription_date"].during(\n r.time(2012, 1, 1, \'Z\'), r.time(2013, 1, 1, \'Z\'))\n ).run(conn)\n\n*Example* Retrieve all users who have a gmail account (whose field `email` ends with `@gmail.com`).\n\n r.table("users").filter(\n lambda user: user["email"].match("@gmail.com$")\n ).run(conn)\n\n*Example* Filter based on the presence of a value in an array.\n\nGiven this schema for the `users` table:\n\n {\n "name": <type \'str\'>\n "places_visited": [<type \'str\'>]\n }\n\nRetrieve all users whose field `places_visited` contains `France`.\n\n r.table("users").filter(lambda user:\n user["places_visited"].contains("France")\n ).run(conn)\n\n*Example* Filter based on nested fields.\n\nGiven this schema for the `users` table:\n\n {\n "id": <type \'str\'>\n "name": {\n "first": <type \'str\'>,\n "middle": <type \'str\'>,\n "last": <type \'str\'>\n }\n }\n\nRetrieve all users named "William Adama" (first name "William", last name\n"Adama"), with any middle name.\n\n r.table("users").filter({\n "name": {\n "first": "William",\n "last": "Adama"\n }\n }).run(conn)\n\nIf you want an exact match for a field that is an object, you will have to use `r.literal`.\n\nRetrieve all users named "William Adama" (first name "William", last name\n"Adama"), and who do not have a middle name.\n\n r.table("users").filter(r.literal({\n "name": {\n "first": "William",\n "last": "Adama"\n }\n })).run(conn)\n\nYou may rewrite these with lambda functions.\n\n r.table("users").filter(\n lambda user:\n (user["name"]["first"] == "William")\n & (user["name"]["last"] == "Adama")\n ).run(conn)\n\n r.table("users").filter(lambda user:\n user["name"] == {\n "first": "William",\n "last": "Adama"\n }\n ).run(conn)\n\nBy default, documents missing fields tested by the `filter` predicate are skipped. In the previous examples, users without an `age` field are not returned. By passing the optional `default` argument to `filter`, you can change this behavior.\n\n*Example* Get all users less than 18 years old or whose `age` field is missing.\n\n r.table("users").filter(r.row["age"] < 18, default=True).run(conn)\n\n*Example* Get all users more than 18 years old. Throw an error if a\ndocument is missing the field `age`.\n\n r.table("users").filter(r.row["age"] > 18, default=r.error()).run(conn)\n\n*Example* Get all users who have given their phone number (all the documents whose field `phone_number` exists and is not `None`).\n\n r.table(\'users\').filter(\n lambda user: user.has_fields(\'phone_number\')\n ).run(conn)\n\n*Example* Get all users with an "editor" role or an "admin" privilege.\n\n r.table(\'users\').filter(\n lambda user: (user[\'role\'] == \'editor\').default(False) |\n (user[\'privilege\'] == \'admin\').default(False)\n ).run(conn)\n\nInstead of using the `default` optional argument to `filter`, we have to use default values on the fields within the `or` clause. Why? If the field on the left side of the `or` clause is missing from a document—in this case, if the user doesn\'t have a `role` field—the predicate will generate an error, and will return `False` (or the value the `default` argument is set to) without evaluating the right side of the `or`. By using `.default(False)` on the fields, each side of the `or` will evaluate to either the field\'s value or `False` if the field doesn\'t exist.\n'), - (rethinkdb.ast.Table.get, b"table.get(key) -> singleRowSelection\n\nGet a document by primary key.\n\nIf no document exists with that primary key, `get` will return `None`.\n\n*Example* Find a document by UUID.\n\n r.table('posts').get('a9849eef-7176-4411-935b-79a6e3c56a74').run(conn)\n\n*Example* Find a document and merge another document with it.\n\n r.table('heroes').get(3).merge(\n { 'powers': ['invisibility', 'speed'] }\n ).run(conn)\n\n_*Example* Subscribe to a document's [changefeed](http://rethinkdb.com/docs/changefeeds/python).\n\n changes = r.table('heroes').get(3).changes().run(conn)\n"), - (rethinkdb.ast.Table.get_all, b"table.get_all(key1[, key2...], [, index='id']) -> selection\n\nGet all documents where the given value matches the value of the requested index.\n\n*Example* Secondary index keys are not guaranteed to be unique so we cannot query via [get](http://rethinkdb.com/api/python/get/) when using a secondary index.\n\n r.table('marvel').get_all('man_of_steel', index='code_name').run(conn)\n\n*Example* Without an index argument, we default to the primary index. While `get` will either return the document or `None` when no document with such a primary key value exists, this will return either a one or zero length stream.\n\n r.table('dc').get_all('superman').run(conn)\n\n*Example* You can get multiple documents in a single call to `get_all`.\n\n r.table('dc').get_all('superman', 'ant man').run(conn)\n\n*Example* You can use [args](http://rethinkdb.com/api/python/args/) with `get_all` to retrieve multiple documents whose keys are in a list. This uses `get_all` to get a list of female superheroes, coerces that to an array, and then gets a list of villains who have those superheroes as enemies.\n\n r.do(\n r.table('heroes').get_all('f', {'index': 'gender'})['id'].coerce_to('array'), \n lamdba heroines: r.table('villains').get_all(r.args(heroines))\n ).run(conn)\n\nSecondary indexes can be used in extremely powerful ways with `get_all` and other commands; read the full article on [secondary indexes](http://rethinkdb.com/docs/secondary-indexes) for examples using boolean operations, `contains` and more.\n"), - (rethinkdb.ast.DB.table, b"db.table(name[, use_outdated=False, identifier_format='name']) -> table\n\nReturn all documents in a table. Other commands may be chained after `table` to return a subset of documents (such as `get` and `filter`) or perform further processing.\n\n*Example* Return all documents in the table 'marvel' of the default database.\n\n r.table('marvel').run(conn)\n\n*Example* Return all documents in the table 'marvel' of the database 'heroes'.\n\n r.db('heroes').table('marvel').run(conn)\n\nThere are two optional arguments.\n\n* `use_outdated`: if `True`, this allows potentially out-of-date data to be returned, with potentially faster reads. It also allows you to perform reads from a secondary replica if a primary has failed. Default `False`.\n* `identifier_format`: possible values are `name` and `uuid`, with a default of `name`. If set to `uuid`, then [system tables](http://rethinkdb.com/docs/system-tables/) will refer to servers, databases and tables by UUID rather than name. (This only has an effect when used with system tables.)\n\n*Example* Allow potentially out-of-date data in exchange for faster reads.\n\n r.db('heroes').table('marvel', use_outdated=True).run(conn)\n"), - (rethinkdb.ast.RqlQuery.downcase, b'string.downcase() -> string\n\nLowercases a string.\n\n*Example*\n\n > r.expr("Sentence about LaTeX.").downcase().run(conn)\n "sentence about latex."\n\n__Note:__ `upcase` and `downcase` only affect ASCII characters.\n'), - (rethinkdb.ast.RqlQuery.match, b'string.match(regexp) -> None/object\n\nMatches against a regular expression. If there is a match, returns an object with the fields:\n\n- `str`: The matched string\n- `start`: The matched string\'s start\n- `end`: The matched string\'s end\n- `groups`: The capture groups defined with parentheses\n\nIf no match is found, returns `None`.\n\nAccepts RE2 syntax\n([https://code.google.com/p/re2/wiki/Syntax](https://code.google.com/p/re2/wiki/Syntax)).\nYou can enable case-insensitive matching by prefixing the regular expression with\n`(?i)`. See the linked RE2 documentation for more flags.\n\nThe `match` command does not support backreferences.\n\n*Example* Get all users whose name starts with "A". Because `None` evaluates to `false` in\n`filter`, you can just use the result of `match` for the predicate.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("^A")\n ).run(conn)\n\n*Example* Get all users whose name ends with "n".\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("n$")\n ).run(conn)\n\n*Example* Get all users whose name has "li" in it\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("li")\n ).run(conn)\n\n*Example* Get all users whose name is "John" with a case-insensitive search.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("(?i)^john$")\n ).run(conn)\n\n*Example* Get all users whose name is composed of only characters between "a" and "z".\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("(?i)^[a-z]+$")\n ).run(conn)\n\n*Example* Get all users where the zipcode is a string of 5 digits.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'zipcode\'].match("\\d{5}")\n ).run(conn)\n\n*Example* Retrieve the domain of a basic email\n\n r.expr("name@domain.com").match(".*@(.*)").run(conn)\n\nResult:\n\n {\n "start": 0,\n "end": 20,\n "str": "name@domain.com",\n "groups":[\n {\n "end": 17,\n "start": 7,\n "str": "domain.com"\n }\n ]\n }\n\nYou can then retrieve only the domain with the [\\[\\]](http://rethinkdb.com/api/python/get_field) selector.\n\n r.expr("name@domain.com").match(".*@(.*)")["groups"][0]["str"].run(conn)\n\nReturns `\'domain.com\'`\n\n*Example* Fail to parse out the domain and returns `None`.\n\n r.expr("name[at]domain.com").match(".*@(.*)").run(conn)\n'), - (rethinkdb.ast.RqlQuery.split, b'string.split([separator, [max_splits]]) -> array\n\nSplits a string into substrings. Splits on whitespace when called\nwith no arguments. When called with a separator, splits on that\nseparator. When called with a separator and a maximum number of\nsplits, splits on that separator at most `max_splits` times. (Can be\ncalled with `None` as the separator if you want to split on whitespace\nwhile still specifying `max_splits`.)\n\nMimics the behavior of Python\'s `string.split` in edge cases, except\nfor splitting on the empty string, which instead produces an array of\nsingle-character strings.\n\n*Example* Split on whitespace.\n\n > r.expr("foo bar bax").split().run(conn)\n ["foo", "bar", "bax"]\n\n*Example* Split the entries in a CSV file.\n\n > r.expr("12,37,,22,").split(",").run(conn)\n ["12", "37", "", "22", ""]\n\n*Example* Split a string into characters.\n\n > r.expr("mlucy").split("").run(conn)\n ["m", "l", "u", "c", "y"]\n\n*Example* Split the entries in a CSV file, but only at most 3\ntimes.\n\n > r.expr("12,37,,22,").split(",", 3).run(conn)\n ["12", "37", "", "22,"]\n\n*Example* Split on whitespace at most once (i.e. get the first word).\n\n > r.expr("foo bar bax").split(None, 1).run(conn)\n ["foo", "bar bax"]\n'), - (rethinkdb.ast.RqlQuery.upcase, b'string.upcase() -> string\n\nUppercases a string.\n\n*Example*\n\n > r.expr("Sentence about LaTeX.").upcase().run(conn)\n "SENTENCE ABOUT LATEX."\n\n__Note:__ `upcase` and `downcase` only affect ASCII characters.\n'), - (rethinkdb.ast.RqlQuery.concat_map, b'stream.concat_map(mapping_function) -> stream\narray.concat_map(mapping_function) -> array\n\nConcatenate one or more elements into a single sequence using a mapping function.\n\n`concat_map` works in a similar fashion to `map`, applying the given function to each element in a sequence, but it will always return a single sequence. If the mapping function returns a sequence, `map` would produce a sequence of sequences:\n\n r.expr([1, 2, 3]).map(lambda x: [x, x.mul(2)]).run(conn)\n\nResult:\n\n [[1, 2], [2, 4], [3, 6]]\n\nWhereas `concat_map` with the same mapping function would merge those sequences into one:\n\n r.expr([1, 2, 3]).concat_map(lambda x: [x, x.mul(2)]).run(conn)\n\nResult:\n\n [1, 2, 2, 4, 3, 6]\n\nThe return value, array or stream, will be the same type as the input.\n\n*Example* Construct a sequence of all monsters defeated by Marvel heroes. The field "defeatedMonsters" is an array of one or more monster names.\n\n r.table(\'marvel\').concat_map(lambda hero: hero[\'defeatedMonsters\']).run(conn)\n\n*Example* Simulate an [eq_join](http://rethinkdb.com/api/python/eq_join/) using `concat_map`. (This is how ReQL joins are implemented internally.)\n\n r.table(\'posts\').concat_map(\n lambda post: r.table(\'comments\').get_all(\n post[\'id\'], index=\'post_id\'\n ).map(\n lambda comment: { \'left\': post, \'right\': comment}\n )\n ).run(conn)\n'), - (rethinkdb.ast.RqlQuery.is_empty, b"sequence.is_empty() -> bool\n\nTest if a sequence is empty.\n\n*Example* Are there any documents in the marvel table?\n\n r.table('marvel').is_empty().run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.limit, b"sequence.limit(n) -> stream\narray.limit(n) -> array\n\nEnd the sequence after the given number of elements.\n\n*Example* Only so many can fit in our Pantheon of heroes.\n\n r.table('marvel').order_by('belovedness').limit(10).run(conn)\n"), - (rethinkdb.ast.RqlQuery.map, b"sequence1.map([sequence2, ...], mapping_function) -> stream\narray1.map([sequence2, ...], mapping_function) -> array\nr.map(sequence1[, sequence2, ...], mapping_function) -> stream\nr.map(array1[, array2, ...], mapping_function) -> array\n\nTransform each element of one or more sequences by applying a mapping function to them. If `map` is run with two or more sequences, it will iterate for as many items as there are in the shortest sequence.\n\nNote that `map` can only be applied to sequences, not single values. If you wish to apply a function to a single value/selection (including an array), use the [do](http://rethinkdb.com/api/python/do) command.\n\n*Example* Return the first five squares.\n\n > r.expr([1, 2, 3, 4, 5]).map(lambda val: (val * val)).run(conn)\n \n [1, 4, 9, 16, 25]\n\n*Example* Sum the elements of three sequences.\n\n > sequence1 = [100, 200, 300, 400]\n > sequence2 = [10, 20, 30, 40]\n > sequence3 = [1, 2, 3, 4]\n > r.map(sequence1, sequence2, sequence3,\n lambda val1, val2, val3: (val1 + val2 + val3)).run(conn)\n \n [111, 222, 333, 444]\n\n*Example* Rename a field when retrieving documents using `map` and `merge`.\n\nThis example renames the field `id` to `user_id` when retrieving documents from the table `users`.\n\n r.table('users').map(\n lambda doc: doc.merge({'user_id': doc['id']}).without('id')).run(conn)\n\nNote that in this case, [row](http://rethinkdb.com/api/python/row) may be used as an alternative to writing an anonymous function, as it returns the same value as the function parameter receives:\n\n r.table('users').map(\n r.row.merge({'user_id': r.row['id']}).without('id')).run(conn)\n\n*Example* Assign every superhero an archenemy.\n\n r.table('heroes').map(r.table('villains'),\n lambda hero, villain: hero.merge({'villain': villain})).run(conn)\n"), - (rethinkdb.ast.RqlQuery.nth, b"sequence.nth(index) -> object\nselection.nth(index) -> selection<object>\n\nGet the *nth* element of a sequence, counting from zero. If the argument is negative, count from the last element.\n\nIn Python, you can use `[]` with an integer as a shorthand for `nth`.\n\n*Example* Select the second element in the array.\n\n r.expr([1,2,3]).nth(1).run(conn)\n r.expr([1,2,3])[1].run(conn)\n\n*Example* Select the bronze medalist from the competitors.\n\n r.table('players').order_by(index=r.desc('score')).nth(3).run(conn)\n\n*Example* Select the last place competitor.\n\n r.table('players').order_by(index=r.desc('score')).nth(-1).run(conn)\n"), - (rethinkdb.ast.RqlQuery.offsets_of, b"sequence.offsets_of(datum | predicate) -> array\n\nGet the indexes of an element in a sequence. If the argument is a predicate, get the indexes of all elements matching it.\n\n*Example* Find the position of the letter 'c'.\n\n r.expr(['a','b','c']).offsets_of('c').run(conn)\n\n*Example* Find the popularity ranking of invisible heroes.\n\n r.table('marvel').union(r.table('dc')).order_by('popularity').offsets_of(\n r.row['superpowers'].contains('invisibility')\n ).run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.order_by, b'table.order_by([key1...], index=index_name) -> selection<stream>\nselection.order_by(key1, [key2...]) -> selection<array>\nsequence.order_by(key1, [key2...]) -> array\n\nSort the sequence by document values of the given key(s). To specify\nthe ordering, wrap the attribute with either `r.asc` or `r.desc`\n(defaults to ascending).\n\n__Note:__ RethinkDB uses byte-wise ordering for `orderBy` and does not support Unicode collations; non-ASCII characters will be sorted by UTF-8 codepoint. For more information on RethinkDB\'s sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nSorting without an index requires the server to hold the sequence in\nmemory, and is limited to 100,000 documents (or the setting of the `arrayLimit` option for [run](http://rethinkdb.com/api/python/run)). Sorting with an index can\nbe done on arbitrarily large tables, or after a `between` command\nusing the same index.\n\n*Example* Order all the posts using the index `date`. \n\n r.table(\'posts\').order_by(index=\'date\').run(conn)\n\nThe index must either be the primary key or have been previously created with [index_create](http://rethinkdb.com/api/python/index_create/).\n\n r.table(\'posts\').index_create(\'date\').run(conn)\n\nYou can also select a descending ordering:\n\n r.table(\'posts\').order_by(index=r.desc(\'date\')).run(conn, callback)\n\n*Example* Order a sequence without an index.\n\n r.table(\'posts\').get(1)[\'comments\'].order_by(\'date\')\n\nYou can also select a descending ordering:\n\n r.table(\'posts\').get(1)[\'comments\'].order_by(r.desc(\'date\'))\n\nIf you\'re doing ad-hoc analysis and know your table won\'t have more then 100,000\nelements (or you\'ve changed the setting of the `arrayLimit` option for [run](http://rethinkdb.com/api/python/run)) you can run `order_by` without an index:\n\n r.table(\'small_table\').order_by(\'date\')\n\n*Example* You can efficiently order using multiple fields by using a\n[compound index](http://www.rethinkdb.com/docs/secondary-indexes/python/).\n\nOrder by date and title.\n\n r.table(\'posts\').order_by(index=\'date_and_title\').run(conn)\n\nThe index must have been previously created with [index_create](http://rethinkdb.com/api/python/index_create/).\n\n r.table(\'posts\').index_create(\'date_and_title\', lambda post:\n [post["date"], post["title"]]).run(conn)\n\n_Note_: You cannot specify multiple orders in a compound index. See [issue #2306](https://github.com/rethinkdb/rethinkdb/issues/2306)\nto track progress.\n\n*Example* If you have a sequence with fewer documents than the `array_limit`, you can order it\nby multiple fields without an index.\n\n r.table(\'small_table\').order_by(\'date\', r.desc(\'title\'))\n\n*Example* Notice that an index ordering always has highest\nprecedence. The following query orders posts by date, and if multiple\nposts were published on the same date, they will be ordered by title.\n\n r.table(\'post\').order_by(\'title\', index=\'date\').run(conn)\n*Example* You can use [nested field](http://rethinkdb.com/docs/cookbook/python/#filtering-based-on-nested-fields) syntax to sort on fields from subdocuments. (You can also create indexes on nested fields using this syntax with `index_create`.)\n\n r.table(\'user\').order_by(lambda user: user[\'group\'][\'id\']).run(conn)\n\n*Example* You can efficiently order data on arbitrary expressions using indexes.\n\n r.table(\'posts\').order_by(index=\'votes\').run(conn)\n\nThe index must have been previously created with [index_create](http://rethinkdb.com/api/ruby/index_create/).\n\n r.table(\'posts\').index_create(\'votes\', lambda post:\n post["upvotes"]-post["downvotes"]\n ).run(conn)\n\n*Example* If you have a sequence with fewer documents than the `array_limit`, you can order it with an arbitrary function directly.\n\n r.table(\'small_table\').order_by(lambda doc:\n doc[\'upvotes\']-doc[\'downvotes\']\n );\n\nYou can also select a descending ordering:\n\n r.table(\'small_table\').order_by(r.desc(lambda doc:\n doc[\'upvotes\']-doc[\'downvotes\']\n ));\n\n*Example* Ordering after a `between` command can be done as long as the same index is being used.\n\n r.table("posts").between(r.time(2013, 1, 1, \'+00:00\'), r.time(2013, 1, 1, \'+00:00\'), index=\'date\')\n .order_by(index=\'date\').run(conn);\n\n'), - (rethinkdb.ast.RqlQuery.sample, b"sequence.sample(number) -> selection\nstream.sample(number) -> array\narray.sample(number) -> array\n\nSelect a given number of elements from a sequence with uniform random distribution. Selection is done without replacement.\n\nIf the sequence has less than the requested number of elements (i.e., calling `sample(10)` on a sequence with only five elements), `sample` will return the entire sequence in a random order.\n\n*Example* Select 3 random heroes.\n\n r.table('marvel').sample(3).run(conn)\n"), - (rethinkdb.ast.RqlQuery.skip, b"sequence.skip(n) -> stream\narray.skip(n) -> array\n\nSkip a number of elements from the head of the sequence.\n\n*Example* Here in conjunction with `order_by` we choose to ignore the most successful heroes.\n\n r.table('marvel').order_by('successMetric').skip(10).run(conn)\n\n"), - (rethinkdb.ast.RqlQuery.slice, b"selection.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> selection\nstream.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> stream\narray.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> array\nbinary.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> binary\n\nReturn the elements of a sequence within the specified range.\n\n`slice` returns the range between `start_index` and `end_index`. If only `start_index` is specified, `slice` returns the range from that index to the end of the sequence. Specify `left_bound` or `right_bound` as `open` or `closed` to indicate whether to include that endpoint of the range by default: `closed` returns that endpoint, while `open` does not. By default, `left_bound` is closed and `right_bound` is open, so the range `(10,13)` will return the tenth, eleventh and twelfth elements in the sequence.\n\nIf `end_index` is past the end of the sequence, all elements from `start_index` to the end of the sequence will be returned. If `start_index` is past the end of the sequence or `end_index` is less than `start_index`, a zero-element sequence will be returned (although see below for negative `end_index` values). An error will be raised on a negative `start_index`.\n\nA negative `end_index` is allowed with arrays; in that case, the returned range counts backward from the array's end. That is, the range of `(2,-1)` returns the second element through the next-to-last element of the range. A negative `end_index` is not allowed with a stream. (An `end_index` of −1 *is* allowed with a stream if `right_bound` is closed; this behaves as if no `end_index` was specified.)\n\nIf `slice` is used with a [binary](http://rethinkdb.com/api/python/binary) object, the indexes refer to byte positions within the object. That is, the range `(10,20)` will refer to the 10th byte through the 19th byte.\n\nIf you are only specifying the indexes and not the bounding options, you may use Python's slice operator as a shorthand: `[start_index:end_index]`.\n\n**Example:** Return the fourth, fifth and sixth youngest players. (The youngest player is at index 0, so those are elements 3–5.)\n\n r.table('players').order_by(index='age').slice(3,6).run(conn)\n\nOr, using Python's slice operator:\n\n r.table('players').filter({'class': 'amateur'})[10:20].run(conn)\n\n**Example:** Return all but the top three players who have a red flag.\n\n r.table('players').filter({'flag': 'red'}).order_by(index=r.desc('score')).slice(3).run(conn)\n\n**Example:** Return holders of tickets `X` through `Y`, assuming tickets are numbered sequentially. We want to include ticket `Y`.\n\n r.table('users').order_by(index='ticket').slice(x, y, right_bound='closed').run(conn)\n\n**Example:** Return the elements of an array from the second through two from the end (that is, not including the last two).\n\n r.expr([0,1,2,3,4,5]).slice(2,-2).run(conn)\n\nResult:\n\n [2,3]\n"), - (rethinkdb.ast.RqlQuery.union, b"stream.union(sequence[, sequence, ...]) -> stream\narray.union(sequence[, sequence, ...]) -> array\n\nConcatenate two or more sequences.\n\n*Example* Construct a stream of all heroes.\n\n r.table('marvel').union(r.table('dc')).run(conn)\n\n*Example* Combine four arrays into one.\n\n r.expr([1, 2]).union([3, 4], [5, 6], [7, 8, 9]).run(conn)\n \n [1, 2, 3, 4, 5, 6, 7, 8, 9]\n"), - (rethinkdb.ast.RqlQuery.with_fields, b"sequence.with_fields([selector1, selector2...]) -> stream\narray.with_fields([selector1, selector2...]) -> array\n\nPlucks one or more attributes from a sequence of objects, filtering out any objects in the sequence that do not have the specified fields. Functionally, this is identical to `has_fields` followed by `pluck` on a sequence.\n\n*Example* Get a list of users and their posts, excluding any users who have not made any posts.\n\nExisting table structure:\n\n [\n { 'id': 1, 'user': 'bob', 'email': 'bob@foo.com', 'posts': [ 1, 4, 5 ] },\n { 'id': 2, 'user': 'george', 'email': 'george@foo.com' },\n { 'id': 3, 'user': 'jane', 'email': 'jane@foo.com', 'posts': [ 2, 3, 6 ] }\n ]\n\nCommand and output:\n\n r.table('users').with_fields('id', 'user', 'posts').run(conn)\n \n [\n { 'id': 1, 'user': 'bob', 'posts': [ 1, 4, 5 ] },\n { 'id': 3, 'user': 'jane', 'posts': [ 2, 3, 6 ] }\n ]\n\n*Example* Use the [nested field syntax](http://rethinkdb.com/docs/nested-fields/) to get a list of users with cell phone numbers in their contacts.\n\n r.table('users').with_fields('id', 'user', {contact: {'phone': 'work'}).run(conn)\n"), - (rethinkdb.ast.Table.delete, b'table.delete([durability="hard", return_changes=False])\n -> object\nselection.delete([durability="hard", return_changes=False])\n -> object\nsingleSelection.delete([durability="hard", return_changes=False])\n -> object\n\nDelete one or more documents from a table.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the\ntable or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). \nIn soft durability mode RethinkDB will acknowledge the write immediately after\nreceiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n\nDelete returns an object that contains the following attributes:\n\n- `deleted`: the number of documents that were deleted.\n- `skipped`: the number of documents that were skipped. \nFor example, if you attempt to delete a batch of documents, and another concurrent query\ndeletes some of those documents first, they will be counted as skipped.\n- `errors`: the number of errors encountered while performing the delete.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `inserted`, `replaced`, and `unchanged`: all 0 for a delete operation.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `delete` operation. Each object will have two keys: `{"new_val": None, "old_val": <old value>}`.\n\n*Example* Delete a single document from the table `comments`.\n\n r.table("comments").get("7eab9e63-73f1-4f33-8ce4-95cbea626f59").delete().run(conn)\n\n*Example* Delete all documents from the table `comments`.\n\n r.table("comments").delete().run(conn)\n\n*Example* Delete all comments where the field `id_post` is `3`.\n\n r.table("comments").filter({"id_post": 3}).delete().run(conn)\n\n*Example* Delete a single document from the table `comments` and return its value.\n\n r.table("comments").get("7eab9e63-73f1-4f33-8ce4-95cbea626f59").delete(return_changes=True).run(conn)\n\nThe result will look like:\n\n {\n "deleted": 1,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": None,\n "old_val": {\n "id": "7eab9e63-73f1-4f33-8ce4-95cbea626f59",\n "author": "William",\n "comment": "Great post",\n "id_post": 3\n }\n }\n ],\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\n*Example* Delete all documents from the table `comments` without waiting for the\noperation to be flushed to disk.\n\n r.table("comments").delete(durability="soft"}).run(conn)\n'), - (rethinkdb.ast.Table.insert, b'table.insert(object | [object1, object2, ...][, durability="hard", return_changes=False, conflict="error"])\n -> object\n\nInsert documents into a table. Accepts a single document or an array of\ndocuments.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the table or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). In soft durability mode RethinkDB will acknowledge the write immediately after receiving and caching it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `conflict`: Determine handling of inserting documents with the same primary key as existing entries. Possible values are `"error"`, `"replace"` or `"update"`.\n - `"error"`: Do not insert the new document and record the conflict as an error. This is the default.\n - `"replace"`: [Replace](http://rethinkdb.com/api/python/replace/) the old document in its entirety with the new one.\n - `"update"`: [Update](http://rethinkdb.com/api/python/update/) fields of the old document with fields from the new one.\n\nInsert returns an object that contains the following attributes:\n\n- `inserted`: the number of documents successfully inserted.\n- `replaced`: the number of documents updated when `conflict` is set to `"replace"` or `"update"`.\n- `unchanged`: the number of documents whose fields are identical to existing documents with the same primary key when `conflict` is set to `"replace"` or `"update"`.\n- `errors`: the number of errors encountered while performing the insert.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `deleted` and `skipped`: 0 for an insert operation.\n- `generated_keys`: a list of generated primary keys for inserted documents whose primary keys were not specified (capped to 100,000).\n- `warnings`: if the field `generated_keys` is truncated, you will get the warning _"Too many generated keys (<X>), array truncated to 100000."_.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `insert` operation. Each object will have two keys: `{"new_val": <new value>, "old_val": None}`.\n\n*Example* Insert a document into the table `posts`.\n\n r.table("posts").insert({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }).run(conn)\n\nThe result will be:\n\n {\n "deleted": 0,\n "errors": 0,\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\n*Example* Insert a document without a defined primary key into the table `posts` where the\nprimary key is `id`.\n\n r.table("posts").insert({\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }).run(conn)\n\nRethinkDB will generate a primary key and return it in `generated_keys`.\n\n {\n "deleted": 0,\n "errors": 0,\n "generated_keys": [\n "dd782b64-70a7-43e4-b65e-dd14ae61d947"\n ],\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\nRetrieve the document you just inserted with:\n\n r.table("posts").get("dd782b64-70a7-43e4-b65e-dd14ae61d947").run(conn)\n\nAnd you will get back:\n\n {\n "id": "dd782b64-70a7-43e4-b65e-dd14ae61d947",\n "title": "Lorem ipsum",\n "content": "Dolor sit amet",\n }\n\n*Example* Insert multiple documents into the table `users`.\n\n r.table("users").insert([\n {"id": "william", "email": "william@rethinkdb.com"},\n {"id": "lara", "email": "lara@rethinkdb.com"}\n ]).run(conn)\n\n*Example* Insert a document into the table `users`, replacing the document if the document\nalready exists. \n\n r.table("users").insert(\n {"id": "william", "email": "william@rethinkdb.com"},\n conflict="error"\n ).run(conn)\n\n*Example* Copy the documents from `posts` to `posts_backup`.\n\n r.table("posts_backup").insert( r.table("posts") ).run(conn)\n\n*Example* Get back a copy of the inserted document (with its generated primary key).\n\n r.table("posts").insert(\n {"title": "Lorem ipsum", "content": "Dolor sit amet"},\n return_changes=True\n ).run(conn)\n\nThe result will be\n\n {\n "deleted": 0,\n "errors": 0,\n "generated_keys": [\n "dd782b64-70a7-43e4-b65e-dd14ae61d947"\n ],\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0,\n "changes": [\n {\n "old_val": None,\n "new_val": {\n "id": "dd782b64-70a7-43e4-b65e-dd14ae61d947",\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }\n }\n ]\n }\n'), - (rethinkdb.ast.Table.replace, b'table.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nselection.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nsingleSelection.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\n\nReplace documents in a table. Accepts a JSON document or a ReQL expression, and replaces\nthe original document with the new one. The new document must have the same primary key\nas the original document.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the\ntable or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). \nIn soft durability mode RethinkDB will acknowledge the write immediately after\nreceiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `non_atomic`: if set to `True`, executes the replacement and distributes the result to replicas in a non-atomic fashion. This flag is required to perform non-deterministic updates, such as those that require reading data from another table.\n\nReplace returns an object that contains the following attributes:\n\n- `replaced`: the number of documents that were replaced\n- `unchanged`: the number of documents that would have been modified, except that the\nnew value was the same as the old value\n- `inserted`: the number of new documents added. You can have new documents inserted if\nyou do a point-replace on a key that isn\'t in the table or you do a replace on a\nselection and one of the documents you are replacing has been deleted\n- `deleted`: the number of deleted documents when doing a replace with `None`\n- `errors`: the number of errors encountered while performing the replace.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `skipped`: 0 for a replace operation\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `replace` operation. Each object will have two keys: `{"new_val": <new value>, "old_val": <old value>}`.\n\n*Example* Replace the document with the primary key `1`.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "draft"\n }).run(conn)\n\n*Example* Remove the field `status` from all posts.\n\n r.table("posts").replace(lambda post:\n post.without("status")\n ).run(conn)\n\n*Example* Remove all the fields that are not `id`, `title` or `content`.\n\n r.table("posts").replace(lambda post:\n post.pluck("id", "title", "content")\n ).run(conn)\n\n*Example* Replace the document with the primary key `1` using soft durability.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "draft"\n }, durability="soft").run(conn)\n\n*Example* Replace the document with the primary key `1` and return the values of the document before\nand after the replace operation.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "published"\n }, return_changes=True).run(conn)\n\nThe result will have a `changes` field:\n\n {\n "deleted": 0,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": {\n "id":1,\n "title": "Lorem ipsum"\n "content": "Aleas jacta est",\n "status": "published",\n },\n "old_val": {\n "id":1,\n "title": "Lorem ipsum"\n "content": "TODO",\n "status": "draft",\n "author": "William",\n }\n }\n ], \n "replaced": 1,\n "skipped": 0,\n "unchanged": 0\n }\n'), - (rethinkdb.ast.Table.sync, b'table.sync() -> object\n\n`sync` ensures that writes on a given table are written to permanent storage. Queries\nthat specify soft durability (`durability=\'soft\'`) do not give such guarantees, so\n`sync` can be used to ensure the state of these queries. A call to `sync` does not return\nuntil all previous writes to the table are persisted.\n\nIf successful, the operation returns an object: `{"synced": 1}`.\n\n*Example* After having updated multiple heroes with soft durability, we now want to wait\nuntil these changes are persisted.\n\n r.table(\'marvel\').sync().run(conn)\n\n'), - (rethinkdb.ast.Table.update, b'table.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nselection.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nsingleSelection.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\n\nUpdate JSON documents in a table. Accepts a JSON document, a ReQL expression, or a combination of the two.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the table or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). In soft durability mode RethinkDB will acknowledge the write immediately after receiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `non_atomic`: if set to `True`, executes the update and distributes the result to replicas in a non-atomic fashion. This flag is required to perform non-deterministic updates, such as those that require reading data from another table.\n\nUpdate returns an object that contains the following attributes:\n\n- `replaced`: the number of documents that were updated.\n- `unchanged`: the number of documents that would have been modified except the new value was the same as the old value.\n- `skipped`: the number of documents that were skipped because the document didn\'t exist.\n- `errors`: the number of errors encountered while performing the update.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `deleted` and `inserted`: 0 for an update operation.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `update` operation. Each object will have two keys: `{"new_val": <new value>, "old_val": <old value>}`.\n\n*Example* Update the status of the post with `id` of `1` to `published`.\n\n r.table("posts").get(1).update({"status": "published"}).run(conn)\n\n*Example* Update the status of all posts to `published`.\n\n r.table("posts").update({"status": "published"}).run(conn)\n\n*Example* Update the status of all the posts written by William.\n\n r.table("posts").filter({"author": "William"}).update({"status": "published"}).run(conn)\n\n*Example* Increment the field `view` with `id` of `1`.\nThis query will throw an error if the field `views` doesn\'t exist.\n\n r.table("posts").get(1).update({\n "views": r.row["views"]+1\n }).run(conn)\n\n*Example* Increment the field `view` of the post with `id` of `1`.\nIf the field `views` does not exist, it will be set to `0`.\n\n r.table("posts").update({\n "views": (r.row["views"]+1).default(0)\n }).run(conn)\n\n*Example* Perform a conditional update. \nIf the post has more than 100 views, set the `type` of a post to `hot`, else set it to `normal`.\n\n r.table("posts").get(1).update(lambda post:\n r.branch(\n post["views"] > 100,\n {"type": "hot"},\n {"type": "normal"}\n )\n ).run(conn)\n\n*Example* Update the field `num_comments` with the result of a sub-query. Because this update is not atomic, you must pass the `non_atomic` flag.\n\n r.table("posts").get(1).update({\n "num_comments": r.table("comments").filter({"id_post": 1}).count()\n }, non_atomic=True).run(conn)\n\nIf you forget to specify the `non_atomic` flag, you will get a `RqlRuntimeError`:\n\nRqlRuntimeError: Could not prove function deterministic. Maybe you want to use the non_atomic flag? \n\n*Example* Update the field `num_comments` with a random value between 0 and 100. This update cannot be proven deterministic because of `r.js` (and in fact is not), so you must pass the `non_atomic` flag.\n\n r.table("posts").get(1).update({\n "num_comments": r.js("Math.floor(Math.random()*100)")\n }, non_atomic=True).run(conn)\n\n*Example* Update the status of the post with `id` of `1` using soft durability.\n\n r.table("posts").get(1).update({status: "published"}, durability="soft").run(conn)\n\n*Example* Increment the field `views` and return the values of the document before and after the update operation.\n\n r.table("posts").get(1).update({\n "views": r.row["views"]+1\n }, return_changes=True).run(conn)\n\nThe result will now include a `changes` field:\n\n {\n "deleted": 1,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": {\n "id": 1,\n "author": "Julius_Caesar",\n "title": "Commentarii de Bello Gallico",\n "content": "Aleas jacta est",\n "views": 207\n },\n "old_val": {\n "id": 1,\n "author": "Julius_Caesar",\n "title": "Commentarii de Bello Gallico",\n "content": "Aleas jacta est",\n "views": 206\n }\n }\n ],\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\nThe `update` command supports RethinkDB\'s [nested field][nf] syntax to update subdocuments. Consider a user table with contact information in this format:\n\n[nf]: /docs/nested-fields/python\n\n {\n "id": 10001,\n "name": "Bob Smith",\n "contact": {\n "phone": {\n "work": "408-555-1212",\n "home": "408-555-1213",\n "cell": "408-555-1214"\n },\n "email": {\n "work": "bob@smith.com",\n "home": "bobsmith@example.com",\n "other": "bobbys@moosecall.net"\n },\n "im": {\n "skype": "Bob Smith",\n "aim": "bobmoose",\n "icq": "nobodyremembersicqnumbers"\n }\n },\n "notes": [\n {\n "date": r.time(2014,1,1,\'Z\'),\n "from": "John Doe",\n "subject": "My name is even more boring than Bob\'s"\n },\n {\n "date": r.time(2014,2,2,\'Z\'),\n "from": "Bob Smith Sr",\n "subject": "Happy Second of February"\n }\n ]\n }\n\n*Example* Update Bob Smith\'s cell phone number.\n\n r.table("users").get(10001).update(\n {"contact": {"phone": {"cell": "408-555-4242"}}}\n ).run(conn)\n\n*Example* Add another note to Bob Smith\'s record.\n\n new_note = {\n "date": r.now(),\n "from": "Inigo Montoya",\n "subject": "You killed my father"\n }\n r.table("users").get(10001).update(\n {"notes": r.row["notes"].append(new_note)}\n ).run(conn)\n\n*Example* Send a note to every user with an ICQ number.\n\n icq_note = {\n "date": r.now(),\n "from": "Admin",\n "subject": "Welcome to the future"\n }\n r.table("users").filter(\n r.row.has_fields({"contact": {"im": "icq"}})\n ).update(\n {"notes": r.row["notes"].append(icq_note)}\n ).run(conn)\n\n*Example* Replace all of Bob\'s IM records. Normally, `update` will merge nested documents together; to replace the entire `"im"` document, use the [literal][] command.\n\n[literal]: /api/python/literal/\n\n r.table(\'users\').get(10001).update(\n {"contact": {"im": r.literal({"aim": "themoosemeister"})}}\n ).run(conn)\n'), -] - -for function, text in docsSource: - try: - text = str(text.decode('utf-8')) - except UnicodeEncodeError: - pass - if hasattr(function, "__func__"): - function.__func__.__doc__ = text - else: - function.__doc__ = text diff --git a/ext/librethinkdbxx/reql/ql2.proto b/ext/librethinkdbxx/reql/ql2.proto deleted file mode 100644 index e40c5be5..00000000 --- a/ext/librethinkdbxx/reql/ql2.proto +++ /dev/null @@ -1,843 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// THE HIGH-LEVEL VIEW // -//////////////////////////////////////////////////////////////////////////////// - -// Process: When you first open a connection, send the magic number -// for the version of the protobuf you're targeting (in the [Version] -// enum). This should **NOT** be sent as a protobuf; just send the -// little-endian 32-bit integer over the wire raw. This number should -// only be sent once per connection. - -// The magic number shall be followed by an authorization key. The -// first 4 bytes are the length of the key to be sent as a little-endian -// 32-bit integer, followed by the key string. Even if there is no key, -// an empty string should be sent (length 0 and no data). - -// Following the authorization key, the client shall send a magic number -// for the communication protocol they want to use (in the [Protocol] -// enum). This shall be a little-endian 32-bit integer. - -// The server will then respond with a NULL-terminated string response. -// "SUCCESS" indicates that the connection has been accepted. Any other -// response indicates an error, and the response string should describe -// the error. - -// Next, for each query you want to send, construct a [Query] protobuf -// and serialize it to a binary blob. Send the blob's size to the -// server encoded as a little-endian 32-bit integer, followed by the -// blob itself. You will recieve a [Response] protobuf back preceded -// by its own size, once again encoded as a little-endian 32-bit -// integer. You can see an example exchange below in **EXAMPLE**. - -// A query consists of a [Term] to evaluate and a unique-per-connection -// [token]. - -// Tokens are used for two things: -// * Keeping track of which responses correspond to which queries. -// * Batched queries. Some queries return lots of results, so we send back -// batches of <1000, and you need to send a [CONTINUE] query with the same -// token to get more results from the original query. -//////////////////////////////////////////////////////////////////////////////// - -message VersionDummy { // We need to wrap it like this for some - // non-conforming protobuf libraries - // This enum contains the magic numbers for your version. See **THE HIGH-LEVEL - // VIEW** for what to do with it. - enum Version { - V0_1 = 0x3f61ba36; - V0_2 = 0x723081e1; // Authorization key during handshake - V0_3 = 0x5f75e83e; // Authorization key and protocol during handshake - V0_4 = 0x400c2d20; // Queries execute in parallel - V1_0 = 0x34c2bdc3; // Users and permissions - } - - // The protocol to use after the handshake, specified in V0_3 - enum Protocol { - PROTOBUF = 0x271ffc41; - JSON = 0x7e6970c7; - } -} - -// You send one of: -// * A [START] query with a [Term] to evaluate and a unique-per-connection token. -// * A [CONTINUE] query with the same token as a [START] query that returned -// [SUCCESS_PARTIAL] in its [Response]. -// * A [STOP] query with the same token as a [START] query that you want to stop. -// * A [NOREPLY_WAIT] query with a unique per-connection token. The server answers -// with a [WAIT_COMPLETE] [Response]. -// * A [SERVER_INFO] query. The server answers with a [SERVER_INFO] [Response]. -message Query { - enum QueryType { - START = 1; // Start a new query. - CONTINUE = 2; // Continue a query that returned [SUCCESS_PARTIAL] - // (see [Response]). - STOP = 3; // Stop a query partway through executing. - NOREPLY_WAIT = 4; // Wait for noreply operations to finish. - SERVER_INFO = 5; // Get server information. - } - optional QueryType type = 1; - // A [Term] is how we represent the operations we want a query to perform. - optional Term query = 2; // only present when [type] = [START] - optional int64 token = 3; - // This flag is ignored on the server. `noreply` should be added - // to `global_optargs` instead (the key "noreply" should map to - // either true or false). - optional bool OBSOLETE_noreply = 4 [default = false]; - - // If this is set to [true], then [Datum] values will sometimes be - // of [DatumType] [R_JSON] (see below). This can provide enormous - // speedups in languages with poor protobuf libraries. - optional bool accepts_r_json = 5 [default = false]; - - message AssocPair { - optional string key = 1; - optional Term val = 2; - } - repeated AssocPair global_optargs = 6; -} - -// A backtrace frame (see `backtrace` in Response below) -message Frame { - enum FrameType { - POS = 1; // Error occurred in a positional argument. - OPT = 2; // Error occurred in an optional argument. - } - optional FrameType type = 1; - optional int64 pos = 2; // The index of the positional argument. - optional string opt = 3; // The name of the optional argument. -} -message Backtrace { - repeated Frame frames = 1; -} - -// You get back a response with the same [token] as your query. -message Response { - enum ResponseType { - // These response types indicate success. - SUCCESS_ATOM = 1; // Query returned a single RQL datatype. - SUCCESS_SEQUENCE = 2; // Query returned a sequence of RQL datatypes. - SUCCESS_PARTIAL = 3; // Query returned a partial sequence of RQL - // datatypes. If you send a [CONTINUE] query with - // the same token as this response, you will get - // more of the sequence. Keep sending [CONTINUE] - // queries until you get back [SUCCESS_SEQUENCE]. - WAIT_COMPLETE = 4; // A [NOREPLY_WAIT] query completed. - SERVER_INFO = 5; // The data for a [SERVER_INFO] request. This is - // the same as `SUCCESS_ATOM` except that there will - // never be profiling data. - - // These response types indicate failure. - CLIENT_ERROR = 16; // Means the client is buggy. An example is if the - // client sends a malformed protobuf, or tries to - // send [CONTINUE] for an unknown token. - COMPILE_ERROR = 17; // Means the query failed during parsing or type - // checking. For example, if you pass too many - // arguments to a function. - RUNTIME_ERROR = 18; // Means the query failed at runtime. An example is - // if you add together two values from a table, but - // they turn out at runtime to be booleans rather - // than numbers. - } - optional ResponseType type = 1; - - // If `ResponseType` is `RUNTIME_ERROR`, this may be filled in with more - // information about the error. - enum ErrorType { - INTERNAL = 1000000; - RESOURCE_LIMIT = 2000000; - QUERY_LOGIC = 3000000; - NON_EXISTENCE = 3100000; - OP_FAILED = 4100000; - OP_INDETERMINATE = 4200000; - USER = 5000000; - PERMISSION_ERROR = 6000000; - } - optional ErrorType error_type = 7; - - // ResponseNotes are used to provide information about the query - // response that may be useful for people writing drivers or ORMs. - // Currently all the notes we send indicate that a stream has certain - // special properties. - enum ResponseNote { - // The stream is a changefeed stream (e.g. `r.table('test').changes()`). - SEQUENCE_FEED = 1; - // The stream is a point changefeed stream - // (e.g. `r.table('test').get(0).changes()`). - ATOM_FEED = 2; - // The stream is an order_by_limit changefeed stream - // (e.g. `r.table('test').order_by(index: 'id').limit(5).changes()`). - ORDER_BY_LIMIT_FEED = 3; - // The stream is a union of multiple changefeed types that can't be - // collapsed to a single type - // (e.g. `r.table('test').changes().union(r.table('test').get(0).changes())`). - UNIONED_FEED = 4; - // The stream is a changefeed stream and includes notes on what state - // the changefeed stream is in (e.g. objects of the form `{state: - // 'initializing'}`). - INCLUDES_STATES = 5; - } - repeated ResponseNote notes = 6; - - optional int64 token = 2; // Indicates what [Query] this response corresponds to. - - // [response] contains 1 RQL datum if [type] is [SUCCESS_ATOM] or - // [SERVER_INFO]. [response] contains many RQL data if [type] is - // [SUCCESS_SEQUENCE] or [SUCCESS_PARTIAL]. [response] contains 1 - // error message (of type [R_STR]) in all other cases. - repeated Datum response = 3; - - // If [type] is [CLIENT_ERROR], [TYPE_ERROR], or [RUNTIME_ERROR], then a - // backtrace will be provided. The backtrace says where in the query the - // error occurred. Ideally this information will be presented to the user as - // a pretty-printed version of their query with the erroneous section - // underlined. A backtrace is a series of 0 or more [Frame]s, each of which - // specifies either the index of a positional argument or the name of an - // optional argument. (Those words will make more sense if you look at the - // [Term] message below.) - optional Backtrace backtrace = 4; // Contains n [Frame]s when you get back an error. - - // If the [global_optargs] in the [Query] that this [Response] is a - // response to contains a key "profile" which maps to a static value of - // true then [profile] will contain a [Datum] which provides profiling - // information about the execution of the query. This field should be - // returned to the user along with the result that would normally be - // returned (a datum or a cursor). In official drivers this is accomplished - // by putting them inside of an object with "value" mapping to the return - // value and "profile" mapping to the profile object. - optional Datum profile = 5; -} - -// A [Datum] is a chunk of data that can be serialized to disk or returned to -// the user in a Response. Currently we only support JSON types, but we may -// support other types in the future (e.g., a date type or an integer type). -message Datum { - enum DatumType { - R_NULL = 1; - R_BOOL = 2; - R_NUM = 3; // a double - R_STR = 4; - R_ARRAY = 5; - R_OBJECT = 6; - // This [DatumType] will only be used if [accepts_r_json] is - // set to [true] in [Query]. [r_str] will be filled with a - // JSON encoding of the [Datum]. - R_JSON = 7; // uses r_str - } - optional DatumType type = 1; - optional bool r_bool = 2; - optional double r_num = 3; - optional string r_str = 4; - - repeated Datum r_array = 5; - message AssocPair { - optional string key = 1; - optional Datum val = 2; - } - repeated AssocPair r_object = 6; -} - -// A [Term] is either a piece of data (see **Datum** above), or an operator and -// its operands. If you have a [Datum], it's stored in the member [datum]. If -// you have an operator, its positional arguments are stored in [args] and its -// optional arguments are stored in [optargs]. -// -// A note about type signatures: -// We use the following notation to denote types: -// arg1_type, arg2_type, argrest_type... -> result_type -// So, for example, if we have a function `avg` that takes any number of -// arguments and averages them, we might write: -// NUMBER... -> NUMBER -// Or if we had a function that took one number modulo another: -// NUMBER, NUMBER -> NUMBER -// Or a function that takes a table and a primary key of any Datum type, then -// retrieves the entry with that primary key: -// Table, DATUM -> OBJECT -// Some arguments must be provided as literal values (and not the results of sub -// terms). These are marked with a `!`. -// Optional arguments are specified within curly braces as argname `:` value -// type (e.x `{noreply:BOOL}`) -// Many RQL operations are polymorphic. For these, alterantive type signatures -// are separated by `|`. -// -// The RQL type hierarchy is as follows: -// Top -// DATUM -// NULL -// BOOL -// NUMBER -// STRING -// OBJECT -// SingleSelection -// ARRAY -// Sequence -// ARRAY -// Stream -// StreamSelection -// Table -// Database -// Function -// Ordering - used only by ORDER_BY -// Pathspec -- an object, string, or array that specifies a path -// Error -message Term { - enum TermType { - // A RQL datum, stored in `datum` below. - DATUM = 1; - - MAKE_ARRAY = 2; // DATUM... -> ARRAY - // Evaluate the terms in [optargs] and make an object - MAKE_OBJ = 3; // {...} -> OBJECT - - // * Compound types - - // Takes an integer representing a variable and returns the value stored - // in that variable. It's the responsibility of the client to translate - // from their local representation of a variable to a unique _non-negative_ - // integer for that variable. (We do it this way instead of letting - // clients provide variable names as strings to discourage - // variable-capturing client libraries, and because it's more efficient - // on the wire.) - VAR = 10; // !NUMBER -> DATUM - // Takes some javascript code and executes it. - JAVASCRIPT = 11; // STRING {timeout: !NUMBER} -> DATUM | - // STRING {timeout: !NUMBER} -> Function(*) - UUID = 169; // () -> DATUM - - // Takes an HTTP URL and gets it. If the get succeeds and - // returns valid JSON, it is converted into a DATUM - HTTP = 153; // STRING {data: OBJECT | STRING, - // timeout: !NUMBER, - // method: STRING, - // params: OBJECT, - // header: OBJECT | ARRAY, - // attempts: NUMBER, - // redirects: NUMBER, - // verify: BOOL, - // page: FUNC | STRING, - // page_limit: NUMBER, - // auth: OBJECT, - // result_format: STRING, - // } -> STRING | STREAM - - // Takes a string and throws an error with that message. - // Inside of a `default` block, you can omit the first - // argument to rethrow whatever error you catch (this is most - // useful as an argument to the `default` filter optarg). - ERROR = 12; // STRING -> Error | -> Error - // Takes nothing and returns a reference to the implicit variable. - IMPLICIT_VAR = 13; // -> DATUM - - // * Data Operators - // Returns a reference to a database. - DB = 14; // STRING -> Database - // Returns a reference to a table. - TABLE = 15; // Database, STRING, {read_mode:STRING, identifier_format:STRING} -> Table - // STRING, {read_mode:STRING, identifier_format:STRING} -> Table - // Gets a single element from a table by its primary or a secondary key. - GET = 16; // Table, STRING -> SingleSelection | Table, NUMBER -> SingleSelection | - // Table, STRING -> NULL | Table, NUMBER -> NULL | - GET_ALL = 78; // Table, DATUM..., {index:!STRING} => ARRAY - - // Simple DATUM Ops - EQ = 17; // DATUM... -> BOOL - NE = 18; // DATUM... -> BOOL - LT = 19; // DATUM... -> BOOL - LE = 20; // DATUM... -> BOOL - GT = 21; // DATUM... -> BOOL - GE = 22; // DATUM... -> BOOL - NOT = 23; // BOOL -> BOOL - // ADD can either add two numbers or concatenate two arrays. - ADD = 24; // NUMBER... -> NUMBER | STRING... -> STRING - SUB = 25; // NUMBER... -> NUMBER - MUL = 26; // NUMBER... -> NUMBER - DIV = 27; // NUMBER... -> NUMBER - MOD = 28; // NUMBER, NUMBER -> NUMBER - - FLOOR = 183; // NUMBER -> NUMBER - CEIL = 184; // NUMBER -> NUMBER - ROUND = 185; // NUMBER -> NUMBER - - // DATUM Array Ops - // Append a single element to the end of an array (like `snoc`). - APPEND = 29; // ARRAY, DATUM -> ARRAY - // Prepend a single element to the end of an array (like `cons`). - PREPEND = 80; // ARRAY, DATUM -> ARRAY - //Remove the elements of one array from another array. - DIFFERENCE = 95; // ARRAY, ARRAY -> ARRAY - - // DATUM Set Ops - // Set ops work on arrays. They don't use actual sets and thus have - // performance characteristics you would expect from arrays rather than - // from sets. All set operations have the post condition that they - // array they return contains no duplicate values. - SET_INSERT = 88; // ARRAY, DATUM -> ARRAY - SET_INTERSECTION = 89; // ARRAY, ARRAY -> ARRAY - SET_UNION = 90; // ARRAY, ARRAY -> ARRAY - SET_DIFFERENCE = 91; // ARRAY, ARRAY -> ARRAY - - SLICE = 30; // Sequence, NUMBER, NUMBER -> Sequence - SKIP = 70; // Sequence, NUMBER -> Sequence - LIMIT = 71; // Sequence, NUMBER -> Sequence - OFFSETS_OF = 87; // Sequence, DATUM -> Sequence | Sequence, Function(1) -> Sequence - CONTAINS = 93; // Sequence, (DATUM | Function(1))... -> BOOL - - // Stream/Object Ops - // Get a particular field from an object, or map that over a - // sequence. - GET_FIELD = 31; // OBJECT, STRING -> DATUM - // | Sequence, STRING -> Sequence - // Return an array containing the keys of the object. - KEYS = 94; // OBJECT -> ARRAY - // Return an array containing the values of the object. - VALUES = 186; // OBJECT -> ARRAY - // Creates an object - OBJECT = 143; // STRING, DATUM, ... -> OBJECT - // Check whether an object contains all the specified fields, - // or filters a sequence so that all objects inside of it - // contain all the specified fields. - HAS_FIELDS = 32; // OBJECT, Pathspec... -> BOOL - // x.with_fields(...) <=> x.has_fields(...).pluck(...) - WITH_FIELDS = 96; // Sequence, Pathspec... -> Sequence - // Get a subset of an object by selecting some attributes to preserve, - // or map that over a sequence. (Both pick and pluck, polymorphic.) - PLUCK = 33; // Sequence, Pathspec... -> Sequence | OBJECT, Pathspec... -> OBJECT - // Get a subset of an object by selecting some attributes to discard, or - // map that over a sequence. (Both unpick and without, polymorphic.) - WITHOUT = 34; // Sequence, Pathspec... -> Sequence | OBJECT, Pathspec... -> OBJECT - // Merge objects (right-preferential) - MERGE = 35; // OBJECT... -> OBJECT | Sequence -> Sequence - - // Sequence Ops - // Get all elements of a sequence between two values. - // Half-open by default, but the openness of either side can be - // changed by passing 'closed' or 'open for `right_bound` or - // `left_bound`. - BETWEEN_DEPRECATED = 36; // Deprecated version of between, which allows `null` to specify unboundedness - // With the newer version, clients should use `r.minval` and `r.maxval` for unboundedness - BETWEEN = 182; // StreamSelection, DATUM, DATUM, {index:!STRING, right_bound:STRING, left_bound:STRING} -> StreamSelection - REDUCE = 37; // Sequence, Function(2) -> DATUM - MAP = 38; // Sequence, Function(1) -> Sequence - // The arity of the function should be - // Sequence..., Function(sizeof...(Sequence)) -> Sequence - - FOLD = 187; // Sequence, Datum, Function(2), {Function(3), Function(1) - - // Filter a sequence with either a function or a shortcut - // object (see API docs for details). The body of FILTER is - // wrapped in an implicit `.default(false)`, and you can - // change the default value by specifying the `default` - // optarg. If you make the default `r.error`, all errors - // caught by `default` will be rethrown as if the `default` - // did not exist. - FILTER = 39; // Sequence, Function(1), {default:DATUM} -> Sequence | - // Sequence, OBJECT, {default:DATUM} -> Sequence - // Map a function over a sequence and then concatenate the results together. - CONCAT_MAP = 40; // Sequence, Function(1) -> Sequence - // Order a sequence based on one or more attributes. - ORDER_BY = 41; // Sequence, (!STRING | Ordering)..., {index: (!STRING | Ordering)} -> Sequence - // Get all distinct elements of a sequence (like `uniq`). - DISTINCT = 42; // Sequence -> Sequence - // Count the number of elements in a sequence, or only the elements that match - // a given filter. - COUNT = 43; // Sequence -> NUMBER | Sequence, DATUM -> NUMBER | Sequence, Function(1) -> NUMBER - IS_EMPTY = 86; // Sequence -> BOOL - // Take the union of multiple sequences (preserves duplicate elements! (use distinct)). - UNION = 44; // Sequence... -> Sequence - // Get the Nth element of a sequence. - NTH = 45; // Sequence, NUMBER -> DATUM - // do NTH or GET_FIELD depending on target object - BRACKET = 170; // Sequence | OBJECT, NUMBER | STRING -> DATUM - // OBSOLETE_GROUPED_MAPREDUCE = 46; - // OBSOLETE_GROUPBY = 47; - - INNER_JOIN = 48; // Sequence, Sequence, Function(2) -> Sequence - OUTER_JOIN = 49; // Sequence, Sequence, Function(2) -> Sequence - // An inner-join that does an equality comparison on two attributes. - EQ_JOIN = 50; // Sequence, !STRING, Sequence, {index:!STRING} -> Sequence - ZIP = 72; // Sequence -> Sequence - RANGE = 173; // -> Sequence [0, +inf) - // NUMBER -> Sequence [0, a) - // NUMBER, NUMBER -> Sequence [a, b) - - // Array Ops - // Insert an element in to an array at a given index. - INSERT_AT = 82; // ARRAY, NUMBER, DATUM -> ARRAY - // Remove an element at a given index from an array. - DELETE_AT = 83; // ARRAY, NUMBER -> ARRAY | - // ARRAY, NUMBER, NUMBER -> ARRAY - // Change the element at a given index of an array. - CHANGE_AT = 84; // ARRAY, NUMBER, DATUM -> ARRAY - // Splice one array in to another array. - SPLICE_AT = 85; // ARRAY, NUMBER, ARRAY -> ARRAY - - // * Type Ops - // Coerces a datum to a named type (e.g. "bool"). - // If you previously used `stream_to_array`, you should use this instead - // with the type "array". - COERCE_TO = 51; // Top, STRING -> Top - // Returns the named type of a datum (e.g. TYPE_OF(true) = "BOOL") - TYPE_OF = 52; // Top -> STRING - - // * Write Ops (the OBJECTs contain data about number of errors etc.) - // Updates all the rows in a selection. Calls its Function with the row - // to be updated, and then merges the result of that call. - UPDATE = 53; // StreamSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | - // SingleSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | - // StreamSelection, OBJECT, {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | - // SingleSelection, OBJECT, {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT - // Deletes all the rows in a selection. - DELETE = 54; // StreamSelection, {durability:STRING, return_changes:BOOL} -> OBJECT | SingleSelection -> OBJECT - // Replaces all the rows in a selection. Calls its Function with the row - // to be replaced, and then discards it and stores the result of that - // call. - REPLACE = 55; // StreamSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | SingleSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT - // Inserts into a table. If `conflict` is replace, overwrites - // entries with the same primary key. If `conflict` is - // update, does an update on the entry. If `conflict` is - // error, or is omitted, conflicts will trigger an error. - INSERT = 56; // Table, OBJECT, {conflict:STRING, durability:STRING, return_changes:BOOL} -> OBJECT | Table, Sequence, {conflict:STRING, durability:STRING, return_changes:BOOL} -> OBJECT - - // * Administrative OPs - // Creates a database with a particular name. - DB_CREATE = 57; // STRING -> OBJECT - // Drops a database with a particular name. - DB_DROP = 58; // STRING -> OBJECT - // Lists all the databases by name. (Takes no arguments) - DB_LIST = 59; // -> ARRAY - // Creates a table with a particular name in a particular - // database. (You may omit the first argument to use the - // default database.) - TABLE_CREATE = 60; // Database, STRING, {primary_key:STRING, shards:NUMBER, replicas:NUMBER, primary_replica_tag:STRING} -> OBJECT - // Database, STRING, {primary_key:STRING, shards:NUMBER, replicas:OBJECT, primary_replica_tag:STRING} -> OBJECT - // STRING, {primary_key:STRING, shards:NUMBER, replicas:NUMBER, primary_replica_tag:STRING} -> OBJECT - // STRING, {primary_key:STRING, shards:NUMBER, replicas:OBJECT, primary_replica_tag:STRING} -> OBJECT - // Drops a table with a particular name from a particular - // database. (You may omit the first argument to use the - // default database.) - TABLE_DROP = 61; // Database, STRING -> OBJECT - // STRING -> OBJECT - // Lists all the tables in a particular database. (You may - // omit the first argument to use the default database.) - TABLE_LIST = 62; // Database -> ARRAY - // -> ARRAY - // Returns the row in the `rethinkdb.table_config` or `rethinkdb.db_config` table - // that corresponds to the given database or table. - CONFIG = 174; // Database -> SingleSelection - // Table -> SingleSelection - // Returns the row in the `rethinkdb.table_status` table that corresponds to the - // given table. - STATUS = 175; // Table -> SingleSelection - // Called on a table, waits for that table to be ready for read/write operations. - // Called on a database, waits for all of the tables in the database to be ready. - // Returns the corresponding row or rows from the `rethinkdb.table_status` table. - WAIT = 177; // Table -> OBJECT - // Database -> OBJECT - // Generates a new config for the given table, or all tables in the given database - // The `shards` and `replicas` arguments are required. If `emergency_repair` is - // specified, it will enter a completely different mode of repairing a table - // which has lost half or more of its replicas. - RECONFIGURE = 176; // Database|Table, {shards:NUMBER, replicas:NUMBER [, - // dry_run:BOOLEAN] - // } -> OBJECT - // Database|Table, {shards:NUMBER, replicas:OBJECT [, - // primary_replica_tag:STRING, - // nonvoting_replica_tags:ARRAY, - // dry_run:BOOLEAN] - // } -> OBJECT - // Table, {emergency_repair:STRING, dry_run:BOOLEAN} -> OBJECT - // Balances the table's shards but leaves everything else the same. Can also be - // applied to an entire database at once. - REBALANCE = 179; // Table -> OBJECT - // Database -> OBJECT - - // Ensures that previously issued soft-durability writes are complete and - // written to disk. - SYNC = 138; // Table -> OBJECT - - // Set global, database, or table-specific permissions - GRANT = 188; // -> OBJECT - // Database -> OBJECT - // Table -> OBJECT - - // * Secondary indexes OPs - // Creates a new secondary index with a particular name and definition. - INDEX_CREATE = 75; // Table, STRING, Function(1), {multi:BOOL} -> OBJECT - // Drops a secondary index with a particular name from the specified table. - INDEX_DROP = 76; // Table, STRING -> OBJECT - // Lists all secondary indexes on a particular table. - INDEX_LIST = 77; // Table -> ARRAY - // Gets information about whether or not a set of indexes are ready to - // be accessed. Returns a list of objects that look like this: - // {index:STRING, ready:BOOL[, progress:NUMBER]} - INDEX_STATUS = 139; // Table, STRING... -> ARRAY - // Blocks until a set of indexes are ready to be accessed. Returns the - // same values INDEX_STATUS. - INDEX_WAIT = 140; // Table, STRING... -> ARRAY - // Renames the given index to a new name - INDEX_RENAME = 156; // Table, STRING, STRING, {overwrite:BOOL} -> OBJECT - - // * Control Operators - // Calls a function on data - FUNCALL = 64; // Function(*), DATUM... -> DATUM - // Executes its first argument, and returns its second argument if it - // got [true] or its third argument if it got [false] (like an `if` - // statement). - BRANCH = 65; // BOOL, Top, Top -> Top - // Returns true if any of its arguments returns true (short-circuits). - OR = 66; // BOOL... -> BOOL - // Returns true if all of its arguments return true (short-circuits). - AND = 67; // BOOL... -> BOOL - // Calls its Function with each entry in the sequence - // and executes the array of terms that Function returns. - FOR_EACH = 68; // Sequence, Function(1) -> OBJECT - -//////////////////////////////////////////////////////////////////////////////// -////////// Special Terms -//////////////////////////////////////////////////////////////////////////////// - - // An anonymous function. Takes an array of numbers representing - // variables (see [VAR] above), and a [Term] to execute with those in - // scope. Returns a function that may be passed an array of arguments, - // then executes the Term with those bound to the variable names. The - // user will never construct this directly. We use it internally for - // things like `map` which take a function. The "arity" of a [Function] is - // the number of arguments it takes. - // For example, here's what `_X_.map{|x| x+2}` turns into: - // Term { - // type = MAP; - // args = [_X_, - // Term { - // type = Function; - // args = [Term { - // type = DATUM; - // datum = Datum { - // type = R_ARRAY; - // r_array = [Datum { type = R_NUM; r_num = 1; }]; - // }; - // }, - // Term { - // type = ADD; - // args = [Term { - // type = VAR; - // args = [Term { - // type = DATUM; - // datum = Datum { type = R_NUM; - // r_num = 1}; - // }]; - // }, - // Term { - // type = DATUM; - // datum = Datum { type = R_NUM; r_num = 2; }; - // }]; - // }]; - // }]; - FUNC = 69; // ARRAY, Top -> ARRAY -> Top - - // Indicates to ORDER_BY that this attribute is to be sorted in ascending order. - ASC = 73; // !STRING -> Ordering - // Indicates to ORDER_BY that this attribute is to be sorted in descending order. - DESC = 74; // !STRING -> Ordering - - // Gets info about anything. INFO is most commonly called on tables. - INFO = 79; // Top -> OBJECT - - // `a.match(b)` returns a match object if the string `a` - // matches the regular expression `b`. - MATCH = 97; // STRING, STRING -> DATUM - - // Change the case of a string. - UPCASE = 141; // STRING -> STRING - DOWNCASE = 142; // STRING -> STRING - - // Select a number of elements from sequence with uniform distribution. - SAMPLE = 81; // Sequence, NUMBER -> Sequence - - // Evaluates its first argument. If that argument returns - // NULL or throws an error related to the absence of an - // expected value (for instance, accessing a non-existent - // field or adding NULL to an integer), DEFAULT will either - // return its second argument or execute it if it's a - // function. If the second argument is a function, it will be - // passed either the text of the error or NULL as its - // argument. - DEFAULT = 92; // Top, Top -> Top - - // Parses its first argument as a json string and returns it as a - // datum. - JSON = 98; // STRING -> DATUM - // Returns the datum as a JSON string. - // N.B.: we would really prefer this be named TO_JSON and that exists as - // an alias in Python and JavaScript drivers; however it conflicts with the - // standard `to_json` method defined by Ruby's standard json library. - TO_JSON_STRING = 172; // DATUM -> STRING - - // Parses its first arguments as an ISO 8601 time and returns it as a - // datum. - ISO8601 = 99; // STRING -> PSEUDOTYPE(TIME) - // Prints a time as an ISO 8601 time. - TO_ISO8601 = 100; // PSEUDOTYPE(TIME) -> STRING - - // Returns a time given seconds since epoch in UTC. - EPOCH_TIME = 101; // NUMBER -> PSEUDOTYPE(TIME) - // Returns seconds since epoch in UTC given a time. - TO_EPOCH_TIME = 102; // PSEUDOTYPE(TIME) -> NUMBER - - // The time the query was received by the server. - NOW = 103; // -> PSEUDOTYPE(TIME) - // Puts a time into an ISO 8601 timezone. - IN_TIMEZONE = 104; // PSEUDOTYPE(TIME), STRING -> PSEUDOTYPE(TIME) - // a.during(b, c) returns whether a is in the range [b, c) - DURING = 105; // PSEUDOTYPE(TIME), PSEUDOTYPE(TIME), PSEUDOTYPE(TIME) -> BOOL - // Retrieves the date portion of a time. - DATE = 106; // PSEUDOTYPE(TIME) -> PSEUDOTYPE(TIME) - // x.time_of_day == x.date - x - TIME_OF_DAY = 126; // PSEUDOTYPE(TIME) -> NUMBER - // Returns the timezone of a time. - TIMEZONE = 127; // PSEUDOTYPE(TIME) -> STRING - - // These access the various components of a time. - YEAR = 128; // PSEUDOTYPE(TIME) -> NUMBER - MONTH = 129; // PSEUDOTYPE(TIME) -> NUMBER - DAY = 130; // PSEUDOTYPE(TIME) -> NUMBER - DAY_OF_WEEK = 131; // PSEUDOTYPE(TIME) -> NUMBER - DAY_OF_YEAR = 132; // PSEUDOTYPE(TIME) -> NUMBER - HOURS = 133; // PSEUDOTYPE(TIME) -> NUMBER - MINUTES = 134; // PSEUDOTYPE(TIME) -> NUMBER - SECONDS = 135; // PSEUDOTYPE(TIME) -> NUMBER - - // Construct a time from a date and optional timezone or a - // date+time and optional timezone. - TIME = 136; // NUMBER, NUMBER, NUMBER, STRING -> PSEUDOTYPE(TIME) | - // NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, STRING -> PSEUDOTYPE(TIME) | - - // Constants for ISO 8601 days of the week. - MONDAY = 107; // -> 1 - TUESDAY = 108; // -> 2 - WEDNESDAY = 109; // -> 3 - THURSDAY = 110; // -> 4 - FRIDAY = 111; // -> 5 - SATURDAY = 112; // -> 6 - SUNDAY = 113; // -> 7 - - // Constants for ISO 8601 months. - JANUARY = 114; // -> 1 - FEBRUARY = 115; // -> 2 - MARCH = 116; // -> 3 - APRIL = 117; // -> 4 - MAY = 118; // -> 5 - JUNE = 119; // -> 6 - JULY = 120; // -> 7 - AUGUST = 121; // -> 8 - SEPTEMBER = 122; // -> 9 - OCTOBER = 123; // -> 10 - NOVEMBER = 124; // -> 11 - DECEMBER = 125; // -> 12 - - // Indicates to MERGE to replace, or remove in case of an empty literal, the - // other object rather than merge it. - LITERAL = 137; // -> Merging - // JSON -> Merging - - // SEQUENCE, STRING -> GROUPED_SEQUENCE | SEQUENCE, FUNCTION -> GROUPED_SEQUENCE - GROUP = 144; - SUM = 145; - AVG = 146; - MIN = 147; - MAX = 148; - - // `str.split()` splits on whitespace - // `str.split(" ")` splits on spaces only - // `str.split(" ", 5)` splits on spaces with at most 5 results - // `str.split(nil, 5)` splits on whitespace with at most 5 results - SPLIT = 149; // STRING -> ARRAY | STRING, STRING -> ARRAY | STRING, STRING, NUMBER -> ARRAY | STRING, NULL, NUMBER -> ARRAY - - UNGROUP = 150; // GROUPED_DATA -> ARRAY - - // Takes a range of numbers and returns a random number within the range - RANDOM = 151; // NUMBER, NUMBER {float:BOOL} -> DATUM - - CHANGES = 152; // TABLE -> STREAM - ARGS = 154; // ARRAY -> SPECIAL (used to splice arguments) - - // BINARY is client-only at the moment, it is not supported on the server - BINARY = 155; // STRING -> PSEUDOTYPE(BINARY) - - GEOJSON = 157; // OBJECT -> PSEUDOTYPE(GEOMETRY) - TO_GEOJSON = 158; // PSEUDOTYPE(GEOMETRY) -> OBJECT - POINT = 159; // NUMBER, NUMBER -> PSEUDOTYPE(GEOMETRY) - LINE = 160; // (ARRAY | PSEUDOTYPE(GEOMETRY))... -> PSEUDOTYPE(GEOMETRY) - POLYGON = 161; // (ARRAY | PSEUDOTYPE(GEOMETRY))... -> PSEUDOTYPE(GEOMETRY) - DISTANCE = 162; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) {geo_system:STRING, unit:STRING} -> NUMBER - INTERSECTS = 163; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> BOOL - INCLUDES = 164; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> BOOL - CIRCLE = 165; // PSEUDOTYPE(GEOMETRY), NUMBER {num_vertices:NUMBER, geo_system:STRING, unit:STRING, fill:BOOL} -> PSEUDOTYPE(GEOMETRY) - GET_INTERSECTING = 166; // TABLE, PSEUDOTYPE(GEOMETRY) {index:!STRING} -> StreamSelection - FILL = 167; // PSEUDOTYPE(GEOMETRY) -> PSEUDOTYPE(GEOMETRY) - GET_NEAREST = 168; // TABLE, PSEUDOTYPE(GEOMETRY) {index:!STRING, max_results:NUM, max_dist:NUM, geo_system:STRING, unit:STRING} -> ARRAY - POLYGON_SUB = 171; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> PSEUDOTYPE(GEOMETRY) - - // Constants for specifying key ranges - MINVAL = 180; - MAXVAL = 181; - } - optional TermType type = 1; - - // This is only used when type is DATUM. - optional Datum datum = 2; - - repeated Term args = 3; // Holds the positional arguments of the query. - message AssocPair { - optional string key = 1; - optional Term val = 2; - } - repeated AssocPair optargs = 4; // Holds the optional arguments of the query. - // (Note that the order of the optional arguments doesn't matter; think of a - // Hash.) -} - -//////////////////////////////////////////////////////////////////////////////// -// EXAMPLE // -//////////////////////////////////////////////////////////////////////////////// -// ```ruby -// r.table('tbl', {:read_mode => 'outdated'}).insert([{:id => 0}, {:id => 1}]) -// ``` -// Would turn into: -// Term { -// type = INSERT; -// args = [Term { -// type = TABLE; -// args = [Term { -// type = DATUM; -// datum = Datum { type = R_STR; r_str = "tbl"; }; -// }]; -// optargs = [["read_mode", -// Term { -// type = DATUM; -// datum = Datum { type = R_STR; r_bool = "outdated"; }; -// }]]; -// }, -// Term { -// type = MAKE_ARRAY; -// args = [Term { -// type = DATUM; -// datum = Datum { type = R_OBJECT; r_object = [["id", 0]]; }; -// }, -// Term { -// type = DATUM; -// datum = Datum { type = R_OBJECT; r_object = [["id", 1]]; }; -// }]; -// }] -// } -// And the server would reply: -// Response { -// type = SUCCESS_ATOM; -// token = 1; -// response = [Datum { type = R_OBJECT; r_object = [["inserted", 2]]; }]; -// } -// Or, if there were an error: -// Response { -// type = RUNTIME_ERROR; -// token = 1; -// response = [Datum { type = R_STR; r_str = "The table `tbl` doesn't exist!"; }]; -// backtrace = [Frame { type = POS; pos = 0; }, Frame { type = POS; pos = 0; }]; -// } diff --git a/ext/librethinkdbxx/src/connection.cc b/ext/librethinkdbxx/src/connection.cc deleted file mode 100644 index 53d106ec..00000000 --- a/ext/librethinkdbxx/src/connection.cc +++ /dev/null @@ -1,434 +0,0 @@ -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/select.h> - -#include <netdb.h> -#include <unistd.h> - -#include <algorithm> -#include <cstring> -#include <cinttypes> -#include <memory> - -#include "connection.h" -#include "connection_p.h" -#include "json_p.h" -#include "exceptions.h" -#include "term.h" -#include "cursor_p.h" - -#include "rapidjson-config.h" -#include "rapidjson/rapidjson.h" -#include "rapidjson/encodedstream.h" -#include "rapidjson/document.h" - -namespace RethinkDB { - -using QueryType = Protocol::Query::QueryType; - -// constants -const int debug_net = 0; -const uint32_t version_magic = - static_cast<uint32_t>(Protocol::VersionDummy::Version::V0_4); -const uint32_t json_magic = - static_cast<uint32_t>(Protocol::VersionDummy::Protocol::JSON); - -std::unique_ptr<Connection> connect(std::string host, int port, std::string auth_key) { - struct addrinfo hints; - memset(&hints, 0, sizeof hints); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - - char port_str[16]; - snprintf(port_str, 16, "%d", port); - struct addrinfo *servinfo; - int ret = getaddrinfo(host.c_str(), port_str, &hints, &servinfo); - if (ret) throw Error("getaddrinfo: %s\n", gai_strerror(ret)); - - struct addrinfo *p; - Error error; - int sockfd; - for (p = servinfo; p != NULL; p = p->ai_next) { - sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); - if (sockfd == -1) { - error = Error::from_errno("socket"); - continue; - } - - if (connect(sockfd, p->ai_addr, p->ai_addrlen) == -1) { - ::close(sockfd); - error = Error::from_errno("connect"); - continue; - } - - break; - } - - if (p == NULL) { - throw error; - } - - freeaddrinfo(servinfo); - - std::unique_ptr<ConnectionPrivate> conn_private(new ConnectionPrivate(sockfd)); - WriteLock writer(conn_private.get()); - { - size_t size = auth_key.size(); - char buf[12 + size]; - memcpy(buf, &version_magic, 4); - uint32_t n = size; - memcpy(buf + 4, &n, 4); - memcpy(buf + 8, auth_key.data(), size); - memcpy(buf + 8 + size, &json_magic, 4); - writer.send(buf, sizeof buf); - } - - ReadLock reader(conn_private.get()); - { - const size_t max_response_length = 1024; - char buf[max_response_length + 1]; - size_t len = reader.recv_cstring(buf, max_response_length); - if (len == max_response_length || strcmp(buf, "SUCCESS")) { - buf[len] = 0; - ::close(sockfd); - throw Error("Server rejected connection with message: %s", buf); - } - } - - return std::unique_ptr<Connection>(new Connection(conn_private.release())); -} - -Connection::Connection(ConnectionPrivate *dd) : d(dd) { } -Connection::~Connection() { - // close(); - if (d->guarded_sockfd >= 0) - ::close(d->guarded_sockfd); -} - -size_t ReadLock::recv_some(char* buf, size_t size, double wait) { - if (wait != FOREVER) { - while (true) { - fd_set readfds; - struct timeval tv; - - FD_ZERO(&readfds); - FD_SET(conn->guarded_sockfd, &readfds); - - tv.tv_sec = (int)wait; - tv.tv_usec = (int)((wait - (int)wait) / MICROSECOND); - int rv = select(conn->guarded_sockfd + 1, &readfds, NULL, NULL, &tv); - if (rv == -1) { - throw Error::from_errno("select"); - } else if (rv == 0) { - throw TimeoutException(); - } - - if (FD_ISSET(conn->guarded_sockfd, &readfds)) { - break; - } - } - } - - ssize_t numbytes = ::recv(conn->guarded_sockfd, buf, size, 0); - if (numbytes <= 0) throw Error::from_errno("recv"); - if (debug_net > 1) { - fprintf(stderr, "<< %s\n", write_datum(std::string(buf, numbytes)).c_str()); - } - - return numbytes; -} - -void ReadLock::recv(char* buf, size_t size, double wait) { - while (size) { - size_t numbytes = recv_some(buf, size, wait); - - buf += numbytes; - size -= numbytes; - } -} - -size_t ReadLock::recv_cstring(char* buf, size_t max_size){ - size_t size = 0; - for (; size < max_size; size++) { - recv(buf, 1, FOREVER); - if (*buf == 0) { - break; - } - buf++; - } - return size; -} - -void WriteLock::send(const char* buf, size_t size) { - while (size) { - ssize_t numbytes = ::write(conn->guarded_sockfd, buf, size); - if (numbytes == -1) throw Error::from_errno("write"); - if (debug_net > 1) { - fprintf(stderr, ">> %s\n", write_datum(std::string(buf, numbytes)).c_str()); - } - - buf += numbytes; - size -= numbytes; - } -} - -void WriteLock::send(const std::string data) { - send(data.data(), data.size()); -} - -std::string ReadLock::recv(size_t size) { - char buf[size]; - recv(buf, size, FOREVER); - return buf; -} - -void Connection::close() { - CacheLock guard(d.get()); - for (auto& it : d->guarded_cache) { - stop_query(it.first); - } - - int ret = ::close(d->guarded_sockfd); - if (ret == -1) { - throw Error::from_errno("close"); - } - d->guarded_sockfd = -1; -} - -Response ConnectionPrivate::wait_for_response(uint64_t token_want, double wait) { - CacheLock guard(this); - ConnectionPrivate::TokenCache& cache = guarded_cache[token_want]; - - while (true) { - if (!cache.responses.empty()) { - Response response(std::move(cache.responses.front())); - cache.responses.pop(); - if (cache.closed && cache.responses.empty()) { - guarded_cache.erase(token_want); - } - - return response; - } - - if (cache.closed) { - throw Error("Trying to read from a closed token"); - } - - if (guarded_loop_active) { - cache.cond.wait(guard.inner_lock); - } else { - break; - } - } - - ReadLock reader(this); - return reader.read_loop(token_want, std::move(guard), wait); -} - -Response ReadLock::read_loop(uint64_t token_want, CacheLock&& guard, double wait) { - if (!guard.inner_lock) { - guard.lock(); - } - if (conn->guarded_loop_active) { - throw Error("Cannot run more than one read loop on the same connection"); - } - conn->guarded_loop_active = true; - guard.unlock(); - - try { - while (true) { - char buf[12]; - bzero(buf, sizeof(buf)); - recv(buf, 12, wait); - uint64_t token_got; - memcpy(&token_got, buf, 8); - uint32_t length; - memcpy(&length, buf + 8, 4); - - std::unique_ptr<char[]> bufmem(new char[length + 1]); - char *buffer = bufmem.get(); - bzero(buffer, length + 1); - recv(buffer, length, wait); - buffer[length] = '\0'; - - rapidjson::Document json; - json.ParseInsitu(buffer); - if (json.HasParseError()) { - fprintf(stderr, "json parse error, code: %d, position: %d\n", - (int)json.GetParseError(), (int)json.GetErrorOffset()); - } else if (json.IsNull()) { - fprintf(stderr, "null value, read: %s\n", buffer); - } - - Datum datum = read_datum(json); - if (debug_net > 0) { - fprintf(stderr, "[%" PRIu64 "] << %s\n", token_got, write_datum(datum).c_str()); - } - - Response response(std::move(datum)); - - if (token_got == token_want) { - guard.lock(); - if (response.type != Protocol::Response::ResponseType::SUCCESS_PARTIAL) { - auto it = conn->guarded_cache.find(token_got); - if (it != conn->guarded_cache.end()) { - it->second.closed = true; - it->second.cond.notify_all(); - } - conn->guarded_cache.erase(it); - } - conn->guarded_loop_active = false; - for (auto& it : conn->guarded_cache) { - it.second.cond.notify_all(); - } - return response; - } else { - guard.lock(); - auto it = conn->guarded_cache.find(token_got); - if (it == conn->guarded_cache.end()) { - // drop the response - } else if (!it->second.closed) { - it->second.responses.emplace(std::move(response)); - if (response.type != Protocol::Response::ResponseType::SUCCESS_PARTIAL) { - it->second.closed = true; - } - } - it->second.cond.notify_all(); - guard.unlock(); - } - } - } catch (const TimeoutException &e) { - if (!guard.inner_lock){ - guard.lock(); - } - conn->guarded_loop_active = false; - throw e; - } -} - -void ConnectionPrivate::run_query(Query query, bool no_reply) { - WriteLock writer(this); - writer.send(query.serialize()); -} - -Cursor Connection::start_query(Term *term, OptArgs&& opts) { - bool no_reply = false; - auto it = opts.find("noreply"); - if (it != opts.end()) { - no_reply = *(it->second.datum.get_boolean()); - } - - uint64_t token = d->new_token(); - { - CacheLock guard(d.get()); - d->guarded_cache[token]; - } - - d->run_query(Query{QueryType::START, token, term->datum, std::move(opts)}); - if (no_reply) { - return Cursor(new CursorPrivate(token, this, Nil())); - } - - Cursor cursor(new CursorPrivate(token, this)); - Response response = d->wait_for_response(token, FOREVER); - cursor.d->add_response(std::move(response)); - return cursor; -} - -void Connection::stop_query(uint64_t token) { - const auto& it = d->guarded_cache.find(token); - if (it != d->guarded_cache.end() && !it->second.closed) { - d->run_query(Query{QueryType::STOP, token}, true); - } -} - -void Connection::continue_query(uint64_t token) { - d->run_query(Query{QueryType::CONTINUE, token}, true); -} - -Error Response::as_error() { - std::string repr; - if (result.size() == 1) { - std::string* string = result[0].get_string(); - if (string) { - repr = *string; - } else { - repr = write_datum(result[0]); - } - } else { - repr = write_datum(Datum(result)); - } - std::string err; - using RT = Protocol::Response::ResponseType; - using ET = Protocol::Response::ErrorType; - switch (type) { - case RT::SUCCESS_SEQUENCE: err = "unexpected response: SUCCESS_SEQUENCE"; break; - case RT::SUCCESS_PARTIAL: err = "unexpected response: SUCCESS_PARTIAL"; break; - case RT::SUCCESS_ATOM: err = "unexpected response: SUCCESS_ATOM"; break; - case RT::WAIT_COMPLETE: err = "unexpected response: WAIT_COMPLETE"; break; - case RT::SERVER_INFO: err = "unexpected response: SERVER_INFO"; break; - case RT::CLIENT_ERROR: err = "ReqlDriverError"; break; - case RT::COMPILE_ERROR: err = "ReqlCompileError"; break; - case RT::RUNTIME_ERROR: - switch (error_type) { - case ET::INTERNAL: err = "ReqlInternalError"; break; - case ET::RESOURCE_LIMIT: err = "ReqlResourceLimitError"; break; - case ET::QUERY_LOGIC: err = "ReqlQueryLogicError"; break; - case ET::NON_EXISTENCE: err = "ReqlNonExistenceError"; break; - case ET::OP_FAILED: err = "ReqlOpFailedError"; break; - case ET::OP_INDETERMINATE: err = "ReqlOpIndeterminateError"; break; - case ET::USER: err = "ReqlUserError"; break; - case ET::PERMISSION_ERROR: err = "ReqlPermissionError"; break; - default: err = "ReqlRuntimeError"; break; - } - } - throw Error("%s: %s", err.c_str(), repr.c_str()); -} - -Protocol::Response::ResponseType response_type(double t) { - int n = static_cast<int>(t); - using RT = Protocol::Response::ResponseType; - switch (n) { - case static_cast<int>(RT::SUCCESS_ATOM): - return RT::SUCCESS_ATOM; - case static_cast<int>(RT::SUCCESS_SEQUENCE): - return RT::SUCCESS_SEQUENCE; - case static_cast<int>(RT::SUCCESS_PARTIAL): - return RT::SUCCESS_PARTIAL; - case static_cast<int>(RT::WAIT_COMPLETE): - return RT::WAIT_COMPLETE; - case static_cast<int>(RT::CLIENT_ERROR): - return RT::CLIENT_ERROR; - case static_cast<int>(RT::COMPILE_ERROR): - return RT::COMPILE_ERROR; - case static_cast<int>(RT::RUNTIME_ERROR): - return RT::RUNTIME_ERROR; - default: - throw Error("Unknown response type"); - } -} - -Protocol::Response::ErrorType runtime_error_type(double t) { - int n = static_cast<int>(t); - using ET = Protocol::Response::ErrorType; - switch (n) { - case static_cast<int>(ET::INTERNAL): - return ET::INTERNAL; - case static_cast<int>(ET::RESOURCE_LIMIT): - return ET::RESOURCE_LIMIT; - case static_cast<int>(ET::QUERY_LOGIC): - return ET::QUERY_LOGIC; - case static_cast<int>(ET::NON_EXISTENCE): - return ET::NON_EXISTENCE; - case static_cast<int>(ET::OP_FAILED): - return ET::OP_FAILED; - case static_cast<int>(ET::OP_INDETERMINATE): - return ET::OP_INDETERMINATE; - case static_cast<int>(ET::USER): - return ET::USER; - default: - throw Error("Unknown error type"); - } -} - -} diff --git a/ext/librethinkdbxx/src/connection.h b/ext/librethinkdbxx/src/connection.h deleted file mode 100644 index d3882857..00000000 --- a/ext/librethinkdbxx/src/connection.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include <string> -#include <queue> -#include <mutex> -#include <memory> -#include <condition_variable> - -#include "protocol_defs.h" -#include "datum.h" -#include "error.h" - -#define FOREVER (-1) -#define SECOND 1 -#define MICROSECOND 0.000001 - -namespace RethinkDB { - -class Term; -using OptArgs = std::map<std::string, Term>; - -// A connection to a RethinkDB server -// It contains: -// * A socket -// * Read and write locks -// * A cache of responses that have not been read by the corresponding Cursor -class ConnectionPrivate; -class Connection { -public: - Connection() = delete; - Connection(const Connection&) noexcept = delete; - Connection(Connection&&) noexcept = delete; - Connection& operator=(Connection&&) noexcept = delete; - Connection& operator=(const Connection&) noexcept = delete; - ~Connection(); - - void close(); - -private: - explicit Connection(ConnectionPrivate *dd); - std::unique_ptr<ConnectionPrivate> d; - - Cursor start_query(Term *term, OptArgs&& args); - void stop_query(uint64_t); - void continue_query(uint64_t); - - friend class Cursor; - friend class CursorPrivate; - friend class Token; - friend class Term; - friend std::unique_ptr<Connection> - connect(std::string host, int port, std::string auth_key); - -}; - -// $doc(connect) -std::unique_ptr<Connection> connect(std::string host = "localhost", int port = 28015, std::string auth_key = ""); - -} diff --git a/ext/librethinkdbxx/src/connection_p.h b/ext/librethinkdbxx/src/connection_p.h deleted file mode 100644 index d8a95e3c..00000000 --- a/ext/librethinkdbxx/src/connection_p.h +++ /dev/null @@ -1,133 +0,0 @@ -#ifndef CONNECTION_P_H -#define CONNECTION_P_H - -#include <inttypes.h> - -#include "connection.h" -#include "term.h" -#include "json_p.h" - -namespace RethinkDB { - -extern const int debug_net; - -struct Query { - Protocol::Query::QueryType type; - uint64_t token; - Datum term; - OptArgs optArgs; - - std::string serialize() { - Array query_arr{static_cast<double>(type)}; - if (term.is_valid()) query_arr.emplace_back(term); - if (!optArgs.empty()) - query_arr.emplace_back(Term(std::move(optArgs)).datum); - - std::string query_str = write_datum(query_arr); - if (debug_net > 0) { - fprintf(stderr, "[%" PRIu64 "] >> %s\n", token, query_str.c_str()); - } - - char header[12]; - memcpy(header, &token, 8); - uint32_t size = query_str.size(); - memcpy(header + 8, &size, 4); - query_str.insert(0, header, 12); - return query_str; - } -}; - -// Used internally to convert a raw response type into an enum -Protocol::Response::ResponseType response_type(double t); -Protocol::Response::ErrorType runtime_error_type(double t); - -// Contains a response from the server. Use the Cursor class to interact with these responses -class Response { -public: - Response() = delete; - explicit Response(Datum&& datum) : - type(response_type(std::move(datum).extract_field("t").extract_number())), - error_type(datum.get_field("e") ? - runtime_error_type(std::move(datum).extract_field("e").extract_number()) : - Protocol::Response::ErrorType(0)), - result(std::move(datum).extract_field("r").extract_array()) { } - Error as_error(); - Protocol::Response::ResponseType type; - Protocol::Response::ErrorType error_type; - Array result; -}; - -class Token; -class ConnectionPrivate { -public: - ConnectionPrivate(int sockfd) - : guarded_next_token(1), guarded_sockfd(sockfd), guarded_loop_active(false) - { } - - void run_query(Query query, bool no_reply = false); - - Response wait_for_response(uint64_t, double); - uint64_t new_token() { - return guarded_next_token++; - } - - std::mutex read_lock; - std::mutex write_lock; - std::mutex cache_lock; - - struct TokenCache { - bool closed = false; - std::condition_variable cond; - std::queue<Response> responses; - }; - - std::map<uint64_t, TokenCache> guarded_cache; - uint64_t guarded_next_token; - int guarded_sockfd; - bool guarded_loop_active; -}; - -class CacheLock { -public: - CacheLock(ConnectionPrivate* conn) : inner_lock(conn->cache_lock) { } - - void lock() { - inner_lock.lock(); - } - - void unlock() { - inner_lock.unlock(); - } - - std::unique_lock<std::mutex> inner_lock; -}; - -class ReadLock { -public: - ReadLock(ConnectionPrivate* conn_) : lock(conn_->read_lock), conn(conn_) { } - - size_t recv_some(char*, size_t, double wait); - void recv(char*, size_t, double wait); - std::string recv(size_t); - size_t recv_cstring(char*, size_t); - - Response read_loop(uint64_t, CacheLock&&, double); - - std::lock_guard<std::mutex> lock; - ConnectionPrivate* conn; -}; - -class WriteLock { -public: - WriteLock(ConnectionPrivate* conn_) : lock(conn_->write_lock), conn(conn_) { } - - void send(const char*, size_t); - void send(std::string); - - std::lock_guard<std::mutex> lock; - ConnectionPrivate* conn; -}; - -} // namespace RethinkDB - -#endif // CONNECTION_P_H diff --git a/ext/librethinkdbxx/src/cursor.cc b/ext/librethinkdbxx/src/cursor.cc deleted file mode 100644 index df0621eb..00000000 --- a/ext/librethinkdbxx/src/cursor.cc +++ /dev/null @@ -1,223 +0,0 @@ -#include "cursor.h" -#include "cursor_p.h" -#include "exceptions.h" - -namespace RethinkDB { - -// for type completion, in order to forward declare with unique_ptr -Cursor::Cursor(Cursor&&) = default; -Cursor& Cursor::operator=(Cursor&&) = default; - -CursorPrivate::CursorPrivate(uint64_t token_, Connection *conn_) - : single(false), no_more(false), index(0), - token(token_), conn(conn_) -{ } - -CursorPrivate::CursorPrivate(uint64_t token_, Connection *conn_, Datum&& datum) - : single(true), no_more(true), index(0), buffer(Array{std::move(datum)}), - token(token_), conn(conn_) -{ } - -Cursor::Cursor(CursorPrivate *dd) : d(dd) {} - -Cursor::~Cursor() { - try { - if (d && d->conn) { - close(); - } - } catch ( ... ) {} -} - -Datum& Cursor::next(double wait) const { - if (!has_next(wait)) { - throw Error("next: No more data"); - } - - return d->buffer[d->index++]; -} - -Datum& Cursor::peek(double wait) const { - if (!has_next(wait)) { - throw Error("next: No more data"); - } - - return d->buffer[d->index]; -} - -void Cursor::each(std::function<void(Datum&&)> f, double wait) const { - while (has_next(wait)) { - f(std::move(d->buffer[d->index++])); - } -} - -void CursorPrivate::convert_single() const { - if (index != 0) { - throw Error("Cursor: already consumed"); - } - - if (buffer.size() != 1) { - throw Error("Cursor: invalid response from server"); - } - - if (!buffer[0].is_array()) { - throw Error("Cursor: not an array"); - } - - buffer.swap(buffer[0].extract_array()); - single = false; -} - -void CursorPrivate::clear_and_read_all() const { - if (single) { - convert_single(); - } - if (index != 0) { - buffer.erase(buffer.begin(), buffer.begin() + index); - index = 0; - } - while (!no_more) { - add_response(conn->d->wait_for_response(token, FOREVER)); - } -} - -Array&& Cursor::to_array() && { - d->clear_and_read_all(); - return std::move(d->buffer); -} - -Array Cursor::to_array() const & { - d->clear_and_read_all(); - return d->buffer; -} - -Datum Cursor::to_datum() const & { - if (d->single) { - if (d->index != 0) { - throw Error("to_datum: already consumed"); - } - return d->buffer[0]; - } - - d->clear_and_read_all(); - return d->buffer; -} - -Datum Cursor::to_datum() && { - Datum ret((Nil())); - if (d->single) { - if (d->index != 0) { - throw Error("to_datum: already consumed"); - } - ret = std::move(d->buffer[0]); - } else { - d->clear_and_read_all(); - ret = std::move(d->buffer); - } - - return ret; -} - -void Cursor::close() const { - d->conn->stop_query(d->token); - d->no_more = true; -} - -bool Cursor::has_next(double wait) const { - if (d->single) { - d->convert_single(); - } - - while (true) { - if (d->index >= d->buffer.size()) { - if (d->no_more) { - return false; - } - d->add_response(d->conn->d->wait_for_response(d->token, wait)); - } else { - return true; - } - } -} - -bool Cursor::is_single() const { - return d->single; -} - -void CursorPrivate::add_results(Array&& results) const { - if (index >= buffer.size()) { - buffer = std::move(results); - index = 0; - } else { - for (auto& it : results) { - buffer.emplace_back(std::move(it)); - } - } -} - -void CursorPrivate::add_response(Response&& response) const { - using RT = Protocol::Response::ResponseType; - switch (response.type) { - case RT::SUCCESS_SEQUENCE: - add_results(std::move(response.result)); - no_more = true; - break; - case RT::SUCCESS_PARTIAL: - conn->continue_query(token); - add_results(std::move(response.result)); - break; - case RT::SUCCESS_ATOM: - add_results(std::move(response.result)); - single = true; - no_more = true; - break; - case RT::SERVER_INFO: - add_results(std::move(response.result)); - single = true; - no_more = true; - break; - case RT::WAIT_COMPLETE: - case RT::CLIENT_ERROR: - case RT::COMPILE_ERROR: - case RT::RUNTIME_ERROR: - no_more = true; - throw response.as_error(); - } -} - -Cursor::iterator Cursor::begin() { - return iterator(this); -} - -Cursor::iterator Cursor::end() { - return iterator(nullptr); -} - -Cursor::iterator::iterator(Cursor* cursor_) : cursor(cursor_) {} - -Cursor::iterator& Cursor::iterator::operator++ () { - if (cursor == nullptr) { - throw Error("incrementing an exhausted Cursor iterator"); - } - - cursor->next(); - return *this; -} - -Datum& Cursor::iterator::operator* () { - if (cursor == nullptr) { - throw Error("reading from empty Cursor iterator"); - } - - return cursor->peek(); -} - -bool Cursor::iterator::operator!= (const Cursor::iterator& other) const { - if (cursor == other.cursor) { - return false; - } - - return !((cursor == nullptr && !other.cursor->has_next()) || - (other.cursor == nullptr && !cursor->has_next())); -} - -} diff --git a/ext/librethinkdbxx/src/cursor.h b/ext/librethinkdbxx/src/cursor.h deleted file mode 100644 index 60ae1817..00000000 --- a/ext/librethinkdbxx/src/cursor.h +++ /dev/null @@ -1,76 +0,0 @@ -#pragma once - -#include "connection.h" - -namespace RethinkDB { - -// The response from the server, as returned by run. -// The response is either a single datum or a stream: -// * If it is a stream, the cursor represents each element of the stream. -// - Batches are fetched from the server as needed. -// * If it is a single datum, is_single() returns true. -// - If it is an array, the cursor represents each element of that array -// - Otherwise, to_datum() returns the datum and iteration throws an exception. -// The cursor can only be iterated over once, it discards data that has already been read. -class CursorPrivate; -class Cursor { -public: - Cursor() = delete; - ~Cursor(); - - Cursor(Cursor&&); // movable - Cursor& operator=(Cursor&&); - Cursor(const Cursor&) = delete; // not copyable - Cursor& operator=(const Cursor&) = delete; - - // Returned by begin() and end() - class iterator { - public: - iterator(Cursor*); - iterator& operator++ (); - Datum& operator* (); - bool operator!= (const iterator&) const; - - private: - Cursor *cursor; - }; - - // Consume the next element - Datum& next(double wait = FOREVER) const; - - // Peek at the next element - Datum& peek(double wait = FOREVER) const; - - // Call f on every element of the Cursor - void each(std::function<void(Datum&&)> f, double wait = FOREVER) const; - - // Consume and return all elements - Array&& to_array() &&; - - // If is_single(), returns the single datum. Otherwise returns to_array(). - Datum to_datum() &&; - Datum to_datum() const &; - - // Efficiently consume and return all elements - Array to_array() const &; - - // Close the cursor - void close() const; - - // Returns false if there are no more elements - bool has_next(double wait = FOREVER) const; - - // Returns false if the cursor is a stream - bool is_single() const; - - iterator begin(); - iterator end(); - -private: - explicit Cursor(CursorPrivate *dd); - std::unique_ptr<CursorPrivate> d; - - friend class Connection; -}; - -} diff --git a/ext/librethinkdbxx/src/cursor_p.h b/ext/librethinkdbxx/src/cursor_p.h deleted file mode 100644 index ce584cd7..00000000 --- a/ext/librethinkdbxx/src/cursor_p.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef CURSOR_P_H -#define CURSOR_P_H - -#include "connection_p.h" - -namespace RethinkDB { - -class CursorPrivate { -public: - CursorPrivate(uint64_t token, Connection *conn); - CursorPrivate(uint64_t token, Connection *conn, Datum&&); - - void add_response(Response&&) const; - void add_results(Array&&) const; - void clear_and_read_all() const; - void convert_single() const; - - mutable bool single = false; - mutable bool no_more = false; - mutable size_t index = 0; - mutable Array buffer; - - uint64_t token; - Connection *conn; -}; - -} // namespace RethinkDB - -#endif // CURSOR_P_H
\ No newline at end of file diff --git a/ext/librethinkdbxx/src/datum.cc b/ext/librethinkdbxx/src/datum.cc deleted file mode 100644 index e4dbc8dc..00000000 --- a/ext/librethinkdbxx/src/datum.cc +++ /dev/null @@ -1,449 +0,0 @@ -#include <float.h> -#include <cmath> - -#include "datum.h" -#include "json_p.h" -#include "utils.h" -#include "cursor.h" - -#include "rapidjson-config.h" -#include "rapidjson/prettywriter.h" -#include "rapidjson/stringbuffer.h" - -namespace RethinkDB { - -using TT = Protocol::Term::TermType; - -bool Datum::is_nil() const { - return type == Type::NIL; -} - -bool Datum::is_boolean() const { - return type == Type::BOOLEAN; -} - -bool Datum::is_number() const { - return type == Type::NUMBER; -} - -bool Datum::is_string() const { - return type == Type::STRING; -} - -bool Datum::is_object() const { - return type == Type::OBJECT; -} - -bool Datum::is_array() const { - return type == Type::ARRAY; -} - -bool Datum::is_binary() const { - return type == Type::BINARY; -} - -bool Datum::is_time() const { - return type == Type::TIME; -} - -bool* Datum::get_boolean() { - if (type == Type::BOOLEAN) { - return &value.boolean; - } else { - return NULL; - } -} - -const bool* Datum::get_boolean() const { - if (type == Type::BOOLEAN) { - return &value.boolean; - } else { - return NULL; - } -} - -double* Datum::get_number() { - if (type == Type::NUMBER) { - return &value.number; - } else { - return NULL; - } -} - -const double* Datum::get_number() const { - if (type == Type::NUMBER) { - return &value.number; - } else { - return NULL; - } -} - -std::string* Datum::get_string() { - if (type == Type::STRING) { - return &value.string; - } else { - return NULL; - } -} - -const std::string* Datum::get_string() const { - if (type == Type::STRING) { - return &value.string; - } else { - return NULL; - } -} - -Datum* Datum::get_field(std::string key) { - if (type != Type::OBJECT) { - return NULL; - } - auto it = value.object.find(key); - if (it == value.object.end()) { - return NULL; - } - return &it->second; -} - -const Datum* Datum::get_field(std::string key) const { - if (type != Type::OBJECT) { - return NULL; - } - auto it = value.object.find(key); - if (it == value.object.end()) { - return NULL; - } - return &it->second; -} - -Datum* Datum::get_nth(size_t i) { - if (type != Type::ARRAY) { - return NULL; - } - if (i >= value.array.size()) { - return NULL; - } - return &value.array[i]; -} - -const Datum* Datum::get_nth(size_t i) const { - if (type != Type::ARRAY) { - return NULL; - } - if (i >= value.array.size()) { - return NULL; - } - return &value.array[i]; -} - -Object* Datum::get_object() { - if (type == Type::OBJECT) { - return &value.object; - } else { - return NULL; - } -} - -const Object* Datum::get_object() const { - if (type == Type::OBJECT) { - return &value.object; - } else { - return NULL; - } -} - -Array* Datum::get_array() { - if (type == Type::ARRAY) { - return &value.array; - } else { - return NULL; - } -} - -const Array* Datum::get_array() const { - if (type == Type::ARRAY) { - return &value.array; - } else { - return NULL; - } -} - -Binary* Datum::get_binary() { - if (type == Type::BINARY) { - return &value.binary; - } else { - return NULL; - } -} - -const Binary* Datum::get_binary() const { - if (type == Type::BINARY) { - return &value.binary; - } else { - return NULL; - } -} - -Time* Datum::get_time() { - if (type == Type::TIME) { - return &value.time; - } else { - return NULL; - } -} - -const Time* Datum::get_time() const { - if (type == Type::TIME) { - return &value.time; - } else { - return NULL; - } -} - -bool& Datum::extract_boolean() { - if (type != Type::BOOLEAN) { - throw Error("extract_bool: Not a boolean"); - } - return value.boolean; -} - -double& Datum::extract_number() { - if (type != Type::NUMBER) { - throw Error("extract_number: Not a number: %s", write_datum(*this).c_str()); - } - return value.number; -} - -std::string& Datum::extract_string() { - if (type != Type::STRING) { - throw Error("extract_string: Not a string"); - } - return value.string; -} - -Object& Datum::extract_object() { - if (type != Type::OBJECT) { - throw Error("extract_object: Not an object"); - } - return value.object; -} - -Datum& Datum::extract_field(std::string key) { - if (type != Type::OBJECT) { - throw Error("extract_field: Not an object"); - } - auto it = value.object.find(key); - if (it == value.object.end()) { - throw Error("extract_field: No such key in object"); - } - return it->second; -} - -Datum& Datum::extract_nth(size_t i) { - if (type != Type::ARRAY) { - throw Error("extract_nth: Not an array"); - } - if (i >= value.array.size()) { - throw Error("extract_nth: index too large"); - } - return value.array[i]; -} - -Array& Datum::extract_array() { - if (type != Type::ARRAY) { - throw Error("get_array: Not an array"); - } - return value.array; -} - -Binary& Datum::extract_binary() { - if (type != Type::BINARY) { - throw Error("get_binary: Not a binary"); - } - return value.binary; -} - -Time& Datum::extract_time() { - if (type != Type::TIME) { - throw Error("get_time: Not a time"); - } - return value.time; -} - -int Datum::compare(const Datum& other) const { -#define COMPARE(a, b) do { \ - if (a < b) { return -1; } \ - if (a > b) { return 1; } } while(0) -#define COMPARE_OTHER(x) COMPARE(x, other.x) - - COMPARE_OTHER(type); - int c; - switch (type) { - case Type::NIL: case Type::INVALID: break; - case Type::BOOLEAN: COMPARE_OTHER(value.boolean); break; - case Type::NUMBER: COMPARE_OTHER(value.number); break; - case Type::STRING: - c = value.string.compare(other.value.string); - COMPARE(c, 0); - break; - case Type::BINARY: - c = value.binary.data.compare(other.value.binary.data); - COMPARE(c, 0); - break; - case Type::TIME: - COMPARE(value.time.epoch_time, other.value.time.epoch_time); - COMPARE(value.time.utc_offset, other.value.time.utc_offset); - break; - case Type::ARRAY: - COMPARE_OTHER(value.array.size()); - for (size_t i = 0; i < value.array.size(); i++) { - c = value.array[i].compare(other.value.array[i]); - COMPARE(c, 0); - } - break; - case Type::OBJECT: - COMPARE_OTHER(value.object.size()); - for (Object::const_iterator l = value.object.begin(), - r = other.value.object.begin(); - l != value.object.end(); - ++l, ++r) { - COMPARE(l->first, r->first); - c = l->second.compare(r->second); - COMPARE(c, 0); - } - break; - default: - throw Error("cannot compare invalid datum"); - } - return 0; -#undef COMPARE_OTHER -#undef COMPARE -} - -bool Datum::operator== (const Datum& other) const { - return compare(other) == 0; -} - -Datum Datum::from_raw() const { - do { - const Datum* type_field = get_field("$reql_type$"); - if (!type_field) break; - const std::string* type = type_field->get_string(); - if (!type) break;; - if (!strcmp(type->c_str(), "BINARY")) { - const Datum* data_field = get_field("data"); - if (!data_field) break; - const std::string* encoded_data = data_field->get_string(); - if (!encoded_data) break; - Binary binary(""); - if (base64_decode(*encoded_data, binary.data)) { - return binary; - } - } else if (!strcmp(type->c_str(), "TIME")) { - const Datum* epoch_field = get_field("epoch_time"); - if (!epoch_field) break; - const Datum* tz_field = get_field("timezone"); - if (!tz_field) break; - const double* epoch_time = epoch_field->get_number(); - if (!epoch_time) break; - const std::string* tz = tz_field->get_string(); - if (!tz) break; - double offset; - if (!Time::parse_utc_offset(*tz, &offset)) break; - return Time(*epoch_time, offset); - } - } while (0); - return *this; -} - -Datum Datum::to_raw() const { - if (type == Type::BINARY) { - return Object{ - {"$reql_type$", "BINARY"}, - {"data", base64_encode(value.binary.data)}}; - } else if (type == Type::TIME) { - return Object{ - {"$reql_type$", "TIME"}, - {"epoch_time", value.time.epoch_time}, - {"timezone", Time::utc_offset_string(value.time.utc_offset)}}; - } - return *this; -} - -Datum::Datum(Cursor&& cursor) : Datum(cursor.to_datum()) { } -Datum::Datum(const Cursor& cursor) : Datum(cursor.to_datum()) { } - -static const double max_dbl_int = 0x1LL << DBL_MANT_DIG; -static const double min_dbl_int = max_dbl_int * -1; -bool number_as_integer(double d, int64_t *i_out) { - static_assert(DBL_MANT_DIG == 53, "Doubles are wrong size."); - - if (min_dbl_int <= d && d <= max_dbl_int) { - int64_t i = d; - if (static_cast<double>(i) == d) { - *i_out = i; - return true; - } - } - return false; -} - -template void Datum::write_json( - rapidjson::Writer<rapidjson::StringBuffer> *writer) const; -template void Datum::write_json( - rapidjson::PrettyWriter<rapidjson::StringBuffer> *writer) const; - -template <class json_writer_t> -void Datum::write_json(json_writer_t *writer) const { - switch (type) { - case Type::NIL: writer->Null(); break; - case Type::BOOLEAN: writer->Bool(value.boolean); break; - case Type::NUMBER: { - const double d = value.number; - // Always print -0.0 as a double since integers cannot represent -0. - // Otherwise check if the number is an integer and print it as such. - int64_t i; - if (!(d == 0.0 && std::signbit(d)) && number_as_integer(d, &i)) { - writer->Int64(i); - } else { - writer->Double(d); - } - } break; - case Type::STRING: writer->String(value.string.data(), value.string.size()); break; - case Type::ARRAY: { - writer->StartArray(); - for (auto it : value.array) { - it.write_json(writer); - } - writer->EndArray(); - } break; - case Type::OBJECT: { - writer->StartObject(); - for (auto it : value.object) { - writer->Key(it.first.data(), it.first.size()); - it.second.write_json(writer); - } - writer->EndObject(); - } break; - - case Type::BINARY: - case Type::TIME: - to_raw().write_json(writer); - break; - default: - throw Error("cannot write invalid datum"); - } -} - -std::string Datum::as_json() const { - rapidjson::StringBuffer buffer; - rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); - write_json(&writer); - return std::string(buffer.GetString(), buffer.GetSize()); -} - -Datum Datum::from_json(const std::string& json) { - return read_datum(json); -} - -} // namespace RethinkDB diff --git a/ext/librethinkdbxx/src/datum.h b/ext/librethinkdbxx/src/datum.h deleted file mode 100644 index 051e2ca2..00000000 --- a/ext/librethinkdbxx/src/datum.h +++ /dev/null @@ -1,287 +0,0 @@ -#pragma once - -#include <string> -#include <vector> -#include <map> -#include <functional> - -#include "protocol_defs.h" -#include "error.h" -#include "types.h" - -namespace RethinkDB { - -class Cursor; - -// The type of data stored in a RethinkDB database. -// The following JSON types are represented in a Datum as -// * null -> Nil -// * boolean -> bool -// * number -> double -// * unicode strings -> std::string -// * array -> Array (aka std::vector<Datum> -// * object -> Object (aka std::map<std::string, Datum>> -// Datums can also contain one of the following extra types -// * binary strings -> Binary -// * timestamps -> Time -// * points. lines and polygons -> not implemented -class Datum { -public: - Datum() : type(Type::INVALID), value() {} - Datum(Nil) : type(Type::NIL), value() { } - Datum(bool boolean_) : type(Type::BOOLEAN), value(boolean_) { } - Datum(double number_) : type(Type::NUMBER), value(number_) { } - Datum(const std::string& string_) : type(Type::STRING), value(string_) { } - Datum(std::string&& string_) : type(Type::STRING), value(std::move(string_)) { } - Datum(const Array& array_) : type(Type::ARRAY), value(array_) { } - Datum(Array&& array_) : type(Type::ARRAY), value(std::move(array_)) { } - Datum(const Binary& binary) : type(Type::BINARY), value(binary) { } - Datum(Binary&& binary) : type(Type::BINARY), value(std::move(binary)) { } - Datum(const Time time) : type(Type::TIME), value(time) { } - Datum(const Object& object_) : type(Type::OBJECT), value(object_) { } - Datum(Object&& object_) : type(Type::OBJECT), value(std::move(object_)) { } - Datum(const Datum& other) : type(other.type), value(other.type, other.value) { } - Datum(Datum&& other) : type(other.type), value(other.type, std::move(other.value)) { } - - Datum& operator=(const Datum& other) { - value.destroy(type); - type = other.type; - value.set(type, other.value); - return *this; - } - - Datum& operator=(Datum&& other) { - value.destroy(type); - type = other.type; - value.set(type, std::move(other.value)); - return *this; - } - - Datum(unsigned short number_) : Datum(static_cast<double>(number_)) { } - Datum(signed short number_) : Datum(static_cast<double>(number_)) { } - Datum(unsigned int number_) : Datum(static_cast<double>(number_)) { } - Datum(signed int number_) : Datum(static_cast<double>(number_)) { } - Datum(unsigned long number_) : Datum(static_cast<double>(number_)) { } - Datum(signed long number_) : Datum(static_cast<double>(number_)) { } - Datum(unsigned long long number_) : Datum(static_cast<double>(number_)) { } - Datum(signed long long number_) : Datum(static_cast<double>(number_)) { } - - Datum(Protocol::Term::TermType type) : Datum(static_cast<double>(type)) { } - Datum(const char* string) : Datum(static_cast<std::string>(string)) { } - - // Cursors are implicitly converted into datums - Datum(Cursor&&); - Datum(const Cursor&); - - template <class T> - Datum(const std::map<std::string, T>& map) : type(Type::OBJECT), value(Object()) { - for (const auto& it : map) { - value.object.emplace(it.left, Datum(it.right)); - } - } - - template <class T> - Datum(std::map<std::string, T>&& map) : type(Type::OBJECT), value(Object()) { - for (auto& it : map) { - value.object.emplace(it.first, Datum(std::move(it.second))); - } - } - - template <class T> - Datum(const std::vector<T>& vec) : type(Type::ARRAY), value(Array()) { - for (const auto& it : vec) { - value.array.emplace_back(it); - } - } - - template <class T> - Datum(std::vector<T>&& vec) : type(Type::ARRAY), value(Array()) { - for (auto& it : vec) { - value.array.emplace_back(std::move(it)); - } - } - - ~Datum() { - value.destroy(type); - } - - // Apply a visitor - template <class R, class F, class ...A> - R apply(F f, A&& ...args) const & { - switch (type) { - case Type::NIL: return f(Nil(), std::forward<A>(args)...); break; - case Type::BOOLEAN: return f(value.boolean, std::forward<A>(args)...); break; - case Type::NUMBER: return f(value.number, std::forward<A>(args)...); break; - case Type::STRING: return f(value.string, std::forward<A>(args)...); break; - case Type::OBJECT: return f(value.object, std::forward<A>(args)...); break; - case Type::ARRAY: return f(value.array, std::forward<A>(args)...); break; - case Type::BINARY: return f(value.binary, std::forward<A>(args)...); break; - case Type::TIME: return f(value.time, std::forward<A>(args)...); break; - default: - throw Error("internal error: no such datum type %d", static_cast<int>(type)); - } - } - - template <class R, class F, class ...A> - R apply(F f, A&& ...args) && { - switch (type) { - case Type::NIL: return f(Nil(), std::forward<A>(args)...); break; - case Type::BOOLEAN: return f(std::move(value.boolean), std::forward<A>(args)...); break; - case Type::NUMBER: return f(std::move(value.number), std::forward<A>(args)...); break; - case Type::STRING: return f(std::move(value.string), std::forward<A>(args)...); break; - case Type::OBJECT: return f(std::move(value.object), std::forward<A>(args)...); break; - case Type::ARRAY: return f(std::move(value.array), std::forward<A>(args)...); break; - case Type::BINARY: return f(std::move(value.binary), std::forward<A>(args)...); break; - case Type::TIME: return f(std::move(value.time), std::forward<A>(args)...); break; - default: - throw Error("internal error: no such datum type %d", static_cast<int>(type)); - } - } - - bool is_nil() const; - bool is_boolean() const; - bool is_number() const; - bool is_string() const; - bool is_object() const; - bool is_array() const; - bool is_binary() const; - bool is_time() const; - - // get_* returns nullptr if the datum has a different type - - bool* get_boolean(); - const bool* get_boolean() const; - double* get_number(); - const double* get_number() const; - std::string* get_string(); - const std::string* get_string() const; - Object* get_object(); - const Object* get_object() const; - Datum* get_field(std::string); - const Datum* get_field(std::string) const; - Array* get_array(); - const Array* get_array() const; - Datum* get_nth(size_t); - const Datum* get_nth(size_t) const; - Binary* get_binary(); - const Binary* get_binary() const; - Time* get_time(); - const Time* get_time() const; - - // extract_* throws an exception if the types don't match - - bool& extract_boolean(); - double& extract_number(); - std::string& extract_string(); - Object& extract_object(); - Datum& extract_field(std::string); - Array& extract_array(); - Datum& extract_nth(size_t); - Binary& extract_binary(); - Time& extract_time(); - - // negative, zero or positive if this datum is smaller, identical or larger than the other one, respectively - // This is meant to match the results of RethinkDB's comparison operators - int compare(const Datum&) const; - - // Deep equality - bool operator== (const Datum&) const; - - // Recusively replace non-JSON types into objects that represent them - Datum to_raw() const; - - // Recursively replace objects with a $reql_type$ field into the datum they represent - Datum from_raw() const; - - template <class json_writer_t> void write_json(json_writer_t *writer) const; - - std::string as_json() const; - static Datum from_json(const std::string&); - - bool is_valid() const { return type != Type::INVALID; } - -private: - enum class Type { - INVALID, // default constructed - ARRAY, BOOLEAN, NIL, NUMBER, OBJECT, BINARY, STRING, TIME - // POINT, LINE, POLYGON - }; - Type type; - - union datum_value { - bool boolean; - double number; - std::string string; - Object object; - Array array; - Binary binary; - Time time; - - datum_value() { } - datum_value(bool boolean_) : boolean(boolean_) { } - datum_value(double number_) : number(number_) { } - datum_value(const std::string& string_) : string(string_) { } - datum_value(std::string&& string_) : string(std::move(string_)) { } - datum_value(const Object& object_) : object(object_) { } - datum_value(Object&& object_) : object(std::move(object_)) { } - datum_value(const Array& array_) : array(array_) { } - datum_value(Array&& array_) : array(std::move(array_)) { } - datum_value(const Binary& binary_) : binary(binary_) { } - datum_value(Binary&& binary_) : binary(std::move(binary_)) { } - datum_value(Time time) : time(std::move(time)) { } - - datum_value(Type type, const datum_value& other){ - set(type, other); - } - - datum_value(Type type, datum_value&& other){ - set(type, std::move(other)); - } - - void set(Type type, datum_value&& other) { - switch(type){ - case Type::NIL: case Type::INVALID: break; - case Type::BOOLEAN: new (this) bool(other.boolean); break; - case Type::NUMBER: new (this) double(other.number); break; - case Type::STRING: new (this) std::string(std::move(other.string)); break; - case Type::OBJECT: new (this) Object(std::move(other.object)); break; - case Type::ARRAY: new (this) Array(std::move(other.array)); break; - case Type::BINARY: new (this) Binary(std::move(other.binary)); break; - case Type::TIME: new (this) Time(std::move(other.time)); break; - } - } - - void set(Type type, const datum_value& other) { - switch(type){ - case Type::NIL: case Type::INVALID: break; - case Type::BOOLEAN: new (this) bool(other.boolean); break; - case Type::NUMBER: new (this) double(other.number); break; - case Type::STRING: new (this) std::string(other.string); break; - case Type::OBJECT: new (this) Object(other.object); break; - case Type::ARRAY: new (this) Array(other.array); break; - case Type::BINARY: new (this) Binary(other.binary); break; - case Type::TIME: new (this) Time(other.time); break; - } - } - - void destroy(Type type) { - switch(type){ - case Type::INVALID: break; - case Type::NIL: break; - case Type::BOOLEAN: break; - case Type::NUMBER: break; - case Type::STRING: { typedef std::string str; string.~str(); } break; - case Type::OBJECT: object.~Object(); break; - case Type::ARRAY: array.~Array(); break; - case Type::BINARY: binary.~Binary(); break; - case Type::TIME: time.~Time(); break; - } - } - - ~datum_value() { } - }; - - datum_value value; -}; - -} diff --git a/ext/librethinkdbxx/src/error.h b/ext/librethinkdbxx/src/error.h deleted file mode 100644 index ab75e248..00000000 --- a/ext/librethinkdbxx/src/error.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include <cstdarg> -#include <cstring> -#include <string> -#include <cerrno> - -namespace RethinkDB { - -// All errors thrown by the server have this type -struct Error { - template <class ...T> - explicit Error(const char* format_, T... A) { - format(format_, A...); - } - - Error() = default; - Error(Error&&) = default; - Error(const Error&) = default; - - Error& operator= (Error&& other) { - message = std::move(other.message); - return *this; - } - - static Error from_errno(const char* str){ - return Error("%s: %s", str, strerror(errno)); - } - - // The error message - std::string message; - -private: - const size_t max_message_size = 2048; - - void format(const char* format_, ...) { - va_list args; - va_start(args, format_); - char message_[max_message_size]; - vsnprintf(message_, max_message_size, format_, args); - va_end(args); - message = message_; - } -}; - -} diff --git a/ext/librethinkdbxx/src/exceptions.h b/ext/librethinkdbxx/src/exceptions.h deleted file mode 100644 index 08c0b0a0..00000000 --- a/ext/librethinkdbxx/src/exceptions.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef EXCEPTIONS_H -#define EXCEPTIONS_H - -namespace RethinkDB { - -class TimeoutException : public std::exception { -public: - const char *what() const throw () { return "operation timed out"; } -}; - -} - -#endif // EXCEPTIONS_H diff --git a/ext/librethinkdbxx/src/json.cc b/ext/librethinkdbxx/src/json.cc deleted file mode 100644 index c908eefb..00000000 --- a/ext/librethinkdbxx/src/json.cc +++ /dev/null @@ -1,62 +0,0 @@ -#include "json_p.h" -#include "error.h" -#include "utils.h" - -#include "rapidjson-config.h" -#include "rapidjson/document.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/writer.h" -#include "rapidjson/prettywriter.h" - -namespace RethinkDB { - -Datum read_datum(const std::string& json) { - rapidjson::Document document; - document.Parse(json); - return read_datum(document); -} - -Datum read_datum(const rapidjson::Value &json) { - switch(json.GetType()) { - case rapidjson::kNullType: return Nil(); - case rapidjson::kFalseType: return false; - case rapidjson::kTrueType: return true; - case rapidjson::kNumberType: return json.GetDouble(); - case rapidjson::kStringType: - return std::string(json.GetString(), json.GetStringLength()); - - case rapidjson::kObjectType: { - Object result; - for (rapidjson::Value::ConstMemberIterator it = json.MemberBegin(); - it != json.MemberEnd(); ++it) { - result.insert(std::make_pair(std::string(it->name.GetString(), - it->name.GetStringLength()), - read_datum(it->value))); - } - - if (result.count("$reql_type$")) - return Datum(std::move(result)).from_raw(); - return std::move(result); - } break; - case rapidjson::kArrayType: { - Array result; - result.reserve(json.Size()); - for (rapidjson::Value::ConstValueIterator it = json.Begin(); - it != json.End(); ++it) { - result.push_back(read_datum(*it)); - } - return std::move(result); - } break; - default: - throw Error("invalid rapidjson value"); - } -} - -std::string write_datum(const Datum& datum) { - rapidjson::StringBuffer buffer; - rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); - datum.write_json(&writer); - return std::string(buffer.GetString(), buffer.GetSize()); -} - -} diff --git a/ext/librethinkdbxx/src/json_p.h b/ext/librethinkdbxx/src/json_p.h deleted file mode 100644 index ebf537a9..00000000 --- a/ext/librethinkdbxx/src/json_p.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include "datum.h" - -namespace rapidjson { - class CrtAllocator; - template<typename> struct UTF8; - template <typename, typename> class GenericValue; - template <typename> class MemoryPoolAllocator; - typedef GenericValue<UTF8<char>, MemoryPoolAllocator<CrtAllocator> > Value; -} - -namespace RethinkDB { - -Datum read_datum(const std::string&); -Datum read_datum(const rapidjson::Value &json); -std::string write_datum(const Datum&); - -} diff --git a/ext/librethinkdbxx/src/rapidjson-config.h b/ext/librethinkdbxx/src/rapidjson-config.h deleted file mode 100644 index 320c4048..00000000 --- a/ext/librethinkdbxx/src/rapidjson-config.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once - -#define RAPIDJSON_HAS_STDSTRING 1 -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 -#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 -#define RAPIDJSON_HAS_CXX11_TYPETRAITS 1 -#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 -#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseFullPrecisionFlag diff --git a/ext/librethinkdbxx/src/rapidjson/allocators.h b/ext/librethinkdbxx/src/rapidjson/allocators.h deleted file mode 100644 index c7059697..00000000 --- a/ext/librethinkdbxx/src/rapidjson/allocators.h +++ /dev/null @@ -1,263 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ALLOCATORS_H_ -#define RAPIDJSON_ALLOCATORS_H_ - -#include "rapidjson.h" - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// Allocator - -/*! \class rapidjson::Allocator - \brief Concept for allocating, resizing and freeing memory block. - - Note that Malloc() and Realloc() are non-static but Free() is static. - - So if an allocator need to support Free(), it needs to put its pointer in - the header of memory block. - -\code -concept Allocator { - static const bool kNeedFree; //!< Whether this allocator needs to call Free(). - - // Allocate a memory block. - // \param size of the memory block in bytes. - // \returns pointer to the memory block. - void* Malloc(size_t size); - - // Resize a memory block. - // \param originalPtr The pointer to current memory block. Null pointer is permitted. - // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) - // \param newSize the new size in bytes. - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); - - // Free a memory block. - // \param pointer to the memory block. Null pointer is permitted. - static void Free(void *ptr); -}; -\endcode -*/ - -/////////////////////////////////////////////////////////////////////////////// -// CrtAllocator - -//! C-runtime library allocator. -/*! This class is just wrapper for standard C library memory routines. - \note implements Allocator concept -*/ -class CrtAllocator { -public: - static const bool kNeedFree = true; - void* Malloc(size_t size) { - if (size) // behavior of malloc(0) is implementation defined. - return std::malloc(size); - else - return NULL; // standardize to returning NULL. - } - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { - (void)originalSize; - if (newSize == 0) { - std::free(originalPtr); - return NULL; - } - return std::realloc(originalPtr, newSize); - } - static void Free(void *ptr) { std::free(ptr); } -}; - -/////////////////////////////////////////////////////////////////////////////// -// MemoryPoolAllocator - -//! Default memory allocator used by the parser and DOM. -/*! This allocator allocate memory blocks from pre-allocated memory chunks. - - It does not free memory blocks. And Realloc() only allocate new memory. - - The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. - - User may also supply a buffer as the first chunk. - - If the user-buffer is full then additional chunks are allocated by BaseAllocator. - - The user-buffer is not deallocated by this allocator. - - \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. - \note implements Allocator concept -*/ -template <typename BaseAllocator = CrtAllocator> -class MemoryPoolAllocator { -public: - static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) - - //! Constructor with chunkSize. - /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - } - - //! Constructor with user-supplied buffer. - /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. - - The user buffer will not be deallocated when this allocator is destructed. - - \param buffer User supplied buffer. - \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). - \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - RAPIDJSON_ASSERT(buffer != 0); - RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); - chunkHead_ = reinterpret_cast<ChunkHeader*>(buffer); - chunkHead_->capacity = size - sizeof(ChunkHeader); - chunkHead_->size = 0; - chunkHead_->next = 0; - } - - //! Destructor. - /*! This deallocates all memory chunks, excluding the user-supplied buffer. - */ - ~MemoryPoolAllocator() { - Clear(); - RAPIDJSON_DELETE(ownBaseAllocator_); - } - - //! Deallocates all memory chunks, excluding the user-supplied buffer. - void Clear() { - while (chunkHead_ && chunkHead_ != userBuffer_) { - ChunkHeader* next = chunkHead_->next; - baseAllocator_->Free(chunkHead_); - chunkHead_ = next; - } - if (chunkHead_ && chunkHead_ == userBuffer_) - chunkHead_->size = 0; // Clear user buffer - } - - //! Computes the total capacity of allocated memory chunks. - /*! \return total capacity in bytes. - */ - size_t Capacity() const { - size_t capacity = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - capacity += c->capacity; - return capacity; - } - - //! Computes the memory blocks allocated. - /*! \return total used bytes. - */ - size_t Size() const { - size_t size = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - size += c->size; - return size; - } - - //! Allocates a memory block. (concept Allocator) - void* Malloc(size_t size) { - if (!size) - return NULL; - - size = RAPIDJSON_ALIGN(size); - if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) - AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); - - void *buffer = reinterpret_cast<char *>(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; - chunkHead_->size += size; - return buffer; - } - - //! Resizes a memory block (concept Allocator) - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { - if (originalPtr == 0) - return Malloc(newSize); - - if (newSize == 0) - return NULL; - - originalSize = RAPIDJSON_ALIGN(originalSize); - newSize = RAPIDJSON_ALIGN(newSize); - - // Do not shrink if new size is smaller than original - if (originalSize >= newSize) - return originalPtr; - - // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == reinterpret_cast<char *>(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { - size_t increment = static_cast<size_t>(newSize - originalSize); - if (chunkHead_->size + increment <= chunkHead_->capacity) { - chunkHead_->size += increment; - return originalPtr; - } - } - - // Realloc process: allocate and copy memory, do not free original buffer. - void* newBuffer = Malloc(newSize); - RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. - if (originalSize) - std::memcpy(newBuffer, originalPtr, originalSize); - return newBuffer; - } - - //! Frees a memory block (concept Allocator) - static void Free(void *ptr) { (void)ptr; } // Do nothing - -private: - //! Copy constructor is not permitted. - MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; - //! Copy assignment operator is not permitted. - MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; - - //! Creates a new chunk. - /*! \param capacity Capacity of the chunk in bytes. - */ - void AddChunk(size_t capacity) { - if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); - ChunkHeader* chunk = reinterpret_cast<ChunkHeader*>(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity)); - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; - } - - static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. - - //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. - */ - struct ChunkHeader { - size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). - size_t size; //!< Current size of allocated memory in bytes. - ChunkHeader *next; //!< Next chunk in the linked list. - }; - - ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. - size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. - void *userBuffer_; //!< User supplied buffer. - BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. - BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. -}; - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/document.h b/ext/librethinkdbxx/src/rapidjson/document.h deleted file mode 100644 index f7f846f2..00000000 --- a/ext/librethinkdbxx/src/rapidjson/document.h +++ /dev/null @@ -1,2575 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_DOCUMENT_H_ -#define RAPIDJSON_DOCUMENT_H_ - -/*! \file document.h */ - -#include "reader.h" -#include "internal/meta.h" -#include "internal/strfunc.h" -#include "memorystream.h" -#include "encodedstream.h" -#include <new> // placement new -#include <limits> - -RAPIDJSON_DIAG_PUSH -#ifdef _MSC_VER -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_OFF(effc++) -#if __GNUC__ >= 6 -RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions -#endif -#endif // __GNUC__ - -#ifndef RAPIDJSON_NOMEMBERITERATORCLASS -#include <iterator> // std::iterator, std::random_access_iterator_tag -#endif - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS -#include <utility> // std::move -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -// Forward declaration. -template <typename Encoding, typename Allocator> -class GenericValue; - -template <typename Encoding, typename Allocator, typename StackAllocator> -class GenericDocument; - -//! Name-value pair in a JSON object value. -/*! - This class was internal to GenericValue. It used to be a inner struct. - But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. - https://code.google.com/p/rapidjson/issues/detail?id=64 -*/ -template <typename Encoding, typename Allocator> -struct GenericMember { - GenericValue<Encoding, Allocator> name; //!< name of member (must be a string) - GenericValue<Encoding, Allocator> value; //!< value of member. -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericMemberIterator - -#ifndef RAPIDJSON_NOMEMBERITERATORCLASS - -//! (Constant) member iterator for a JSON object value -/*! - \tparam Const Is this a constant iterator? - \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) - \tparam Allocator Allocator type for allocating memory of object, array and string. - - This class implements a Random Access Iterator for GenericMember elements - of a GenericValue, see ISO/IEC 14882:2003(E) C++ standard, 24.1 [lib.iterator.requirements]. - - \note This iterator implementation is mainly intended to avoid implicit - conversions from iterator values to \c NULL, - e.g. from GenericValue::FindMember. - - \note Define \c RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a - pointer-based implementation, if your platform doesn't provide - the C++ <iterator> header. - - \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator - */ -template <bool Const, typename Encoding, typename Allocator> -class GenericMemberIterator - : public std::iterator<std::random_access_iterator_tag - , typename internal::MaybeAddConst<Const,GenericMember<Encoding,Allocator> >::Type> { - - friend class GenericValue<Encoding,Allocator>; - template <bool, typename, typename> friend class GenericMemberIterator; - - typedef GenericMember<Encoding,Allocator> PlainType; - typedef typename internal::MaybeAddConst<Const,PlainType>::Type ValueType; - typedef std::iterator<std::random_access_iterator_tag,ValueType> BaseType; - -public: - //! Iterator type itself - typedef GenericMemberIterator Iterator; - //! Constant iterator type - typedef GenericMemberIterator<true,Encoding,Allocator> ConstIterator; - //! Non-constant iterator type - typedef GenericMemberIterator<false,Encoding,Allocator> NonConstIterator; - - //! Pointer to (const) GenericMember - typedef typename BaseType::pointer Pointer; - //! Reference to (const) GenericMember - typedef typename BaseType::reference Reference; - //! Signed integer type (e.g. \c ptrdiff_t) - typedef typename BaseType::difference_type DifferenceType; - - //! Default constructor (singular value) - /*! Creates an iterator pointing to no element. - \note All operations, except for comparisons, are undefined on such values. - */ - GenericMemberIterator() : ptr_() {} - - //! Iterator conversions to more const - /*! - \param it (Non-const) iterator to copy from - - Allows the creation of an iterator from another GenericMemberIterator - that is "less const". Especially, creating a non-constant iterator - from a constant iterator are disabled: - \li const -> non-const (not ok) - \li const -> const (ok) - \li non-const -> const (ok) - \li non-const -> non-const (ok) - - \note If the \c Const template parameter is already \c false, this - constructor effectively defines a regular copy-constructor. - Otherwise, the copy constructor is implicitly defined. - */ - GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} - Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } - - //! @name stepping - //@{ - Iterator& operator++(){ ++ptr_; return *this; } - Iterator& operator--(){ --ptr_; return *this; } - Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } - Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } - //@} - - //! @name increment/decrement - //@{ - Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } - Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } - - Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } - Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } - //@} - - //! @name relations - //@{ - bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } - bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } - bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } - bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } - bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } - bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } - //@} - - //! @name dereference - //@{ - Reference operator*() const { return *ptr_; } - Pointer operator->() const { return ptr_; } - Reference operator[](DifferenceType n) const { return ptr_[n]; } - //@} - - //! Distance - DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } - -private: - //! Internal constructor from plain pointer - explicit GenericMemberIterator(Pointer p) : ptr_(p) {} - - Pointer ptr_; //!< raw pointer -}; - -#else // RAPIDJSON_NOMEMBERITERATORCLASS - -// class-based member iterator implementation disabled, use plain pointers - -template <bool Const, typename Encoding, typename Allocator> -struct GenericMemberIterator; - -//! non-const GenericMemberIterator -template <typename Encoding, typename Allocator> -struct GenericMemberIterator<false,Encoding,Allocator> { - //! use plain pointer as iterator type - typedef GenericMember<Encoding,Allocator>* Iterator; -}; -//! const GenericMemberIterator -template <typename Encoding, typename Allocator> -struct GenericMemberIterator<true,Encoding,Allocator> { - //! use plain const pointer as iterator type - typedef const GenericMember<Encoding,Allocator>* Iterator; -}; - -#endif // RAPIDJSON_NOMEMBERITERATORCLASS - -/////////////////////////////////////////////////////////////////////////////// -// GenericStringRef - -//! Reference to a constant string (not taking a copy) -/*! - \tparam CharType character type of the string - - This helper class is used to automatically infer constant string - references for string literals, especially from \c const \b (!) - character arrays. - - The main use is for creating JSON string values without copying the - source string via an \ref Allocator. This requires that the referenced - string pointers have a sufficient lifetime, which exceeds the lifetime - of the associated GenericValue. - - \b Example - \code - Value v("foo"); // ok, no need to copy & calculate length - const char foo[] = "foo"; - v.SetString(foo); // ok - - const char* bar = foo; - // Value x(bar); // not ok, can't rely on bar's lifetime - Value x(StringRef(bar)); // lifetime explicitly guaranteed by user - Value y(StringRef(bar, 3)); // ok, explicitly pass length - \endcode - - \see StringRef, GenericValue::SetString -*/ -template<typename CharType> -struct GenericStringRef { - typedef CharType Ch; //!< character type of the string - - //! Create string reference from \c const character array -#ifndef __clang__ // -Wdocumentation - /*! - This constructor implicitly creates a constant string reference from - a \c const character array. It has better performance than - \ref StringRef(const CharType*) by inferring the string \ref length - from the array length, and also supports strings containing null - characters. - - \tparam N length of the string, automatically inferred - - \param str Constant character array, lifetime assumed to be longer - than the use of the string in e.g. a GenericValue - - \post \ref s == str - - \note Constant complexity. - \note There is a hidden, private overload to disallow references to - non-const character arrays to be created via this constructor. - By this, e.g. function-scope arrays used to be filled via - \c snprintf are excluded from consideration. - In such cases, the referenced string should be \b copied to the - GenericValue instead. - */ -#endif - template<SizeType N> - GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT - : s(str), length(N-1) {} - - //! Explicitly create string reference from \c const character pointer -#ifndef __clang__ // -Wdocumentation - /*! - This constructor can be used to \b explicitly create a reference to - a constant string pointer. - - \see StringRef(const CharType*) - - \param str Constant character pointer, lifetime assumed to be longer - than the use of the string in e.g. a GenericValue - - \post \ref s == str - - \note There is a hidden, private overload to disallow references to - non-const character arrays to be created via this constructor. - By this, e.g. function-scope arrays used to be filled via - \c snprintf are excluded from consideration. - In such cases, the referenced string should be \b copied to the - GenericValue instead. - */ -#endif - explicit GenericStringRef(const CharType* str) - : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != 0); } - - //! Create constant string reference from pointer and length -#ifndef __clang__ // -Wdocumentation - /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \param len length of the string, excluding the trailing NULL terminator - - \post \ref s == str && \ref length == len - \note Constant complexity. - */ -#endif - GenericStringRef(const CharType* str, SizeType len) - : s(str), length(len) { RAPIDJSON_ASSERT(s != 0); } - - GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} - - GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } - - //! implicit conversion to plain CharType pointer - operator const Ch *() const { return s; } - - const Ch* const s; //!< plain CharType pointer - const SizeType length; //!< length of the string (excluding the trailing NULL terminator) - -private: - //! Disallow construction from non-const array - template<SizeType N> - GenericStringRef(CharType (&str)[N]) /* = delete */; -}; - -//! Mark a character pointer as constant string -/*! Mark a plain character pointer as a "string literal". This function - can be used to avoid copying a character string to be referenced as a - value in a JSON GenericValue object, if the string's lifetime is known - to be valid long enough. - \tparam CharType Character type of the string - \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \return GenericStringRef string reference object - \relatesalso GenericStringRef - - \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember -*/ -template<typename CharType> -inline GenericStringRef<CharType> StringRef(const CharType* str) { - return GenericStringRef<CharType>(str, internal::StrLen(str)); -} - -//! Mark a character pointer as constant string -/*! Mark a plain character pointer as a "string literal". This function - can be used to avoid copying a character string to be referenced as a - value in a JSON GenericValue object, if the string's lifetime is known - to be valid long enough. - - This version has better performance with supplied length, and also - supports string containing null characters. - - \tparam CharType character type of the string - \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \param length The length of source string. - \return GenericStringRef string reference object - \relatesalso GenericStringRef -*/ -template<typename CharType> -inline GenericStringRef<CharType> StringRef(const CharType* str, size_t length) { - return GenericStringRef<CharType>(str, SizeType(length)); -} - -#if RAPIDJSON_HAS_STDSTRING -//! Mark a string object as constant string -/*! Mark a string object (e.g. \c std::string) as a "string literal". - This function can be used to avoid copying a string to be referenced as a - value in a JSON GenericValue object, if the string's lifetime is known - to be valid long enough. - - \tparam CharType character type of the string - \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \return GenericStringRef string reference object - \relatesalso GenericStringRef - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. -*/ -template<typename CharType> -inline GenericStringRef<CharType> StringRef(const std::basic_string<CharType>& str) { - return GenericStringRef<CharType>(str.data(), SizeType(str.size())); -} -#endif - -/////////////////////////////////////////////////////////////////////////////// -// GenericValue type traits -namespace internal { - -template <typename T, typename Encoding = void, typename Allocator = void> -struct IsGenericValueImpl : FalseType {}; - -// select candidates according to nested encoding and allocator types -template <typename T> struct IsGenericValueImpl<T, typename Void<typename T::EncodingType>::Type, typename Void<typename T::AllocatorType>::Type> - : IsBaseOf<GenericValue<typename T::EncodingType, typename T::AllocatorType>, T>::Type {}; - -// helper to match arbitrary GenericValue instantiations, including derived classes -template <typename T> struct IsGenericValue : IsGenericValueImpl<T>::Type {}; - -} // namespace internal - -/////////////////////////////////////////////////////////////////////////////// -// TypeHelper - -namespace internal { - -template <typename ValueType, typename T> -struct TypeHelper {}; - -template<typename ValueType> -struct TypeHelper<ValueType, bool> { - static bool Is(const ValueType& v) { return v.IsBool(); } - static bool Get(const ValueType& v) { return v.GetBool(); } - static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } - static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, int> { - static bool Is(const ValueType& v) { return v.IsInt(); } - static int Get(const ValueType& v) { return v.GetInt(); } - static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } - static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, unsigned> { - static bool Is(const ValueType& v) { return v.IsUint(); } - static unsigned Get(const ValueType& v) { return v.GetUint(); } - static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } - static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, int64_t> { - static bool Is(const ValueType& v) { return v.IsInt64(); } - static int64_t Get(const ValueType& v) { return v.GetInt64(); } - static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } - static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, uint64_t> { - static bool Is(const ValueType& v) { return v.IsUint64(); } - static uint64_t Get(const ValueType& v) { return v.GetUint64(); } - static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } - static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, double> { - static bool Is(const ValueType& v) { return v.IsDouble(); } - static double Get(const ValueType& v) { return v.GetDouble(); } - static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } - static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, float> { - static bool Is(const ValueType& v) { return v.IsFloat(); } - static float Get(const ValueType& v) { return v.GetFloat(); } - static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } - static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, const typename ValueType::Ch*> { - typedef const typename ValueType::Ch* StringType; - static bool Is(const ValueType& v) { return v.IsString(); } - static StringType Get(const ValueType& v) { return v.GetString(); } - static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } - static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } -}; - -#if RAPIDJSON_HAS_STDSTRING -template<typename ValueType> -struct TypeHelper<ValueType, std::basic_string<typename ValueType::Ch> > { - typedef std::basic_string<typename ValueType::Ch> StringType; - static bool Is(const ValueType& v) { return v.IsString(); } - static StringType Get(const ValueType& v) { return v.GetString(); } - static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } -}; -#endif - -template<typename ValueType> -struct TypeHelper<ValueType, typename ValueType::Array> { - typedef typename ValueType::Array ArrayType; - static bool Is(const ValueType& v) { return v.IsArray(); } - static ArrayType Get(ValueType& v) { return v.GetArray(); } - static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } - static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, typename ValueType::ConstArray> { - typedef typename ValueType::ConstArray ArrayType; - static bool Is(const ValueType& v) { return v.IsArray(); } - static ArrayType Get(const ValueType& v) { return v.GetArray(); } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, typename ValueType::Object> { - typedef typename ValueType::Object ObjectType; - static bool Is(const ValueType& v) { return v.IsObject(); } - static ObjectType Get(ValueType& v) { return v.GetObject(); } - static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } - static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; } -}; - -template<typename ValueType> -struct TypeHelper<ValueType, typename ValueType::ConstObject> { - typedef typename ValueType::ConstObject ObjectType; - static bool Is(const ValueType& v) { return v.IsObject(); } - static ObjectType Get(const ValueType& v) { return v.GetObject(); } -}; - -} // namespace internal - -// Forward declarations -template <bool, typename> class GenericArray; -template <bool, typename> class GenericObject; - -/////////////////////////////////////////////////////////////////////////////// -// GenericValue - -//! Represents a JSON value. Use Value for UTF8 encoding and default allocator. -/*! - A JSON value can be one of 7 types. This class is a variant type supporting - these types. - - Use the Value if UTF8 and default allocator - - \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) - \tparam Allocator Allocator type for allocating memory of object, array and string. -*/ -template <typename Encoding, typename Allocator = MemoryPoolAllocator<> > -class GenericValue { -public: - //! Name-value pair in an object. - typedef GenericMember<Encoding, Allocator> Member; - typedef Encoding EncodingType; //!< Encoding type from template parameter. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef GenericStringRef<Ch> StringRefType; //!< Reference to a constant string - typedef typename GenericMemberIterator<false,Encoding,Allocator>::Iterator MemberIterator; //!< Member iterator for iterating in object. - typedef typename GenericMemberIterator<true,Encoding,Allocator>::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. - typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. - typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. - typedef GenericValue<Encoding, Allocator> ValueType; //!< Value type of itself. - typedef GenericArray<false, ValueType> Array; - typedef GenericArray<true, ValueType> ConstArray; - typedef GenericObject<false, ValueType> Object; - typedef GenericObject<true, ValueType> ConstObject; - - //!@name Constructors and destructor. - //@{ - - //! Default constructor creates a null value. - GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { - rhs.data_.f.flags = kNullFlag; // give up contents - } -#endif - -private: - //! Copy constructor is not permitted. - GenericValue(const GenericValue& rhs); - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Moving from a GenericDocument is not permitted. - template <typename StackAllocator> - GenericValue(GenericDocument<Encoding,Allocator,StackAllocator>&& rhs); - - //! Move assignment from a GenericDocument is not permitted. - template <typename StackAllocator> - GenericValue& operator=(GenericDocument<Encoding,Allocator,StackAllocator>&& rhs); -#endif - -public: - - //! Constructor with JSON value type. - /*! This creates a Value of specified type with default content. - \param type Type of the value. - \note Default content for number is zero. - */ - explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { - static const uint16_t defaultFlags[7] = { - kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, - kNumberAnyFlag - }; - RAPIDJSON_ASSERT(type <= kNumberType); - data_.f.flags = defaultFlags[type]; - - // Use ShortString to store empty string. - if (type == kStringType) - data_.ss.SetLength(0); - } - - //! Explicit copy constructor (with allocator) - /*! Creates a copy of a Value by using the given Allocator - \tparam SourceAllocator allocator of \c rhs - \param rhs Value to copy from (read-only) - \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). - \see CopyFrom() - */ - template< typename SourceAllocator > - GenericValue(const GenericValue<Encoding, SourceAllocator>& rhs, Allocator & allocator); - - //! Constructor for boolean value. - /*! \param b Boolean value - \note This constructor is limited to \em real boolean values and rejects - implicitly converted types like arbitrary pointers. Use an explicit cast - to \c bool, if you want to construct a boolean JSON value in such cases. - */ -#ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen - template <typename T> - explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame<bool, T>))) RAPIDJSON_NOEXCEPT // See #472 -#else - explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT -#endif - : data_() { - // safe-guard against failing SFINAE - RAPIDJSON_STATIC_ASSERT((internal::IsSame<bool,T>::Value)); - data_.f.flags = b ? kTrueFlag : kFalseFlag; - } - - //! Constructor for int value. - explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { - data_.n.i64 = i; - data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; - } - - //! Constructor for unsigned value. - explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { - data_.n.u64 = u; - data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); - } - - //! Constructor for int64_t value. - explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { - data_.n.i64 = i64; - data_.f.flags = kNumberInt64Flag; - if (i64 >= 0) { - data_.f.flags |= kNumberUint64Flag; - if (!(static_cast<uint64_t>(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - data_.f.flags |= kUintFlag; - if (!(static_cast<uint64_t>(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; - } - else if (i64 >= static_cast<int64_t>(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; - } - - //! Constructor for uint64_t value. - explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { - data_.n.u64 = u64; - data_.f.flags = kNumberUint64Flag; - if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) - data_.f.flags |= kInt64Flag; - if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - data_.f.flags |= kUintFlag; - if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; - } - - //! Constructor for double value. - explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } - - //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } - - //! Constructor for constant string (i.e. do not make a copy of string) - explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } - -#if RAPIDJSON_HAS_STDSTRING - //! Constructor for copy-string from a string object (i.e. do make a copy of string) - /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - GenericValue(const std::basic_string<Ch>& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } -#endif - - //! Constructor for Array. - /*! - \param a An array obtained by \c GetArray(). - \note \c Array is always pass-by-value. - \note the source array is moved into this value and the sourec array becomes empty. - */ - GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { - a.value_.data_ = Data(); - a.value_.data_.f.flags = kArrayFlag; - } - - //! Constructor for Object. - /*! - \param o An object obtained by \c GetObject(). - \note \c Object is always pass-by-value. - \note the source object is moved into this value and the sourec object becomes empty. - */ - GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { - o.value_.data_ = Data(); - o.value_.data_.f.flags = kObjectFlag; - } - - //! Destructor. - /*! Need to destruct elements of array, members of object, or copy-string. - */ - ~GenericValue() { - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - switch(data_.f.flags) { - case kArrayFlag: - { - GenericValue* e = GetElementsPointer(); - for (GenericValue* v = e; v != e + data_.a.size; ++v) - v->~GenericValue(); - Allocator::Free(e); - } - break; - - case kObjectFlag: - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - Allocator::Free(GetMembersPointer()); - break; - - case kCopyStringFlag: - Allocator::Free(const_cast<Ch*>(GetStringPointer())); - break; - - default: - break; // Do nothing for other types. - } - } - } - - //@} - - //!@name Assignment operators - //@{ - - //! Assignment with move semantics. - /*! \param rhs Source of the assignment. It will become a null value after assignment. - */ - GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { - RAPIDJSON_ASSERT(this != &rhs); - this->~GenericValue(); - RawAssign(rhs); - return *this; - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move assignment in C++11 - GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT { - return *this = rhs.Move(); - } -#endif - - //! Assignment of constant string reference (no copy) - /*! \param str Constant string reference to be assigned - \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. - \see GenericStringRef, operator=(T) - */ - GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT { - GenericValue s(str); - return *this = s; - } - - //! Assignment with primitive types. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param value The value to be assigned. - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref SetString(const Ch*, Allocator&) (for copying) or - \ref StringRef() (to explicitly mark the pointer as constant) instead. - All other pointer types would implicitly convert to \c bool, - use \ref SetBool() instead. - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer<T>), (GenericValue&)) - operator=(T value) { - GenericValue v(value); - return *this = v; - } - - //! Deep-copy assignment from Value - /*! Assigns a \b copy of the Value to the current Value object - \tparam SourceAllocator Allocator type of \c rhs - \param rhs Value to copy from (read-only) - \param allocator Allocator to use for copying - */ - template <typename SourceAllocator> - GenericValue& CopyFrom(const GenericValue<Encoding, SourceAllocator>& rhs, Allocator& allocator) { - RAPIDJSON_ASSERT(static_cast<void*>(this) != static_cast<void const*>(&rhs)); - this->~GenericValue(); - new (this) GenericValue(rhs, allocator); - return *this; - } - - //! Exchange the contents of this value with those of other. - /*! - \param other Another value. - \note Constant complexity. - */ - GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT { - GenericValue temp; - temp.RawAssign(*this); - RawAssign(other); - other.RawAssign(temp); - return *this; - } - - //! free-standing swap function helper - /*! - Helper function to enable support for common swap implementation pattern based on \c std::swap: - \code - void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.value, b.value); - // ... - } - \endcode - \see Swap() - */ - friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } - - //! Prepare Value for move semantics - /*! \return *this */ - GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } - //@} - - //!@name Equal-to and not-equal-to operators - //@{ - //! Equal-to operator - /*! - \note If an object contains duplicated named member, comparing equality with any object is always \c false. - \note Linear time complexity (number of all values in the subtree and total lengths of all strings). - */ - template <typename SourceAllocator> - bool operator==(const GenericValue<Encoding, SourceAllocator>& rhs) const { - typedef GenericValue<Encoding, SourceAllocator> RhsType; - if (GetType() != rhs.GetType()) - return false; - - switch (GetType()) { - case kObjectType: // Warning: O(n^2) inner-loop - if (data_.o.size != rhs.data_.o.size) - return false; - for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { - typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); - if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) - return false; - } - return true; - - case kArrayType: - if (data_.a.size != rhs.data_.a.size) - return false; - for (SizeType i = 0; i < data_.a.size; i++) - if ((*this)[i] != rhs[i]) - return false; - return true; - - case kStringType: - return StringEqual(rhs); - - case kNumberType: - if (IsDouble() || rhs.IsDouble()) { - double a = GetDouble(); // May convert from integer to double. - double b = rhs.GetDouble(); // Ditto - return a >= b && a <= b; // Prevent -Wfloat-equal - } - else - return data_.n.u64 == rhs.data_.n.u64; - - default: - return true; - } - } - - //! Equal-to operator with const C-string pointer - bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } - -#if RAPIDJSON_HAS_STDSTRING - //! Equal-to operator with string object - /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - bool operator==(const std::basic_string<Ch>& rhs) const { return *this == GenericValue(StringRef(rhs)); } -#endif - - //! Equal-to operator with primitive types - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false - */ - template <typename T> RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>,internal::IsGenericValue<T> >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } - - //! Not-equal-to operator - /*! \return !(*this == rhs) - */ - template <typename SourceAllocator> - bool operator!=(const GenericValue<Encoding, SourceAllocator>& rhs) const { return !(*this == rhs); } - - //! Not-equal-to operator with const C-string pointer - bool operator!=(const Ch* rhs) const { return !(*this == rhs); } - - //! Not-equal-to operator with arbitrary types - /*! \return !(*this == rhs) - */ - template <typename T> RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue<T>), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } - - //! Equal-to operator with arbitrary types (symmetric version) - /*! \return (rhs == lhs) - */ - template <typename T> friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue<T>), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } - - //! Not-Equal-to operator with arbitrary types (symmetric version) - /*! \return !(rhs == lhs) - */ - template <typename T> friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue<T>), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } - //@} - - //!@name Type - //@{ - - Type GetType() const { return static_cast<Type>(data_.f.flags & kTypeMask); } - bool IsNull() const { return data_.f.flags == kNullFlag; } - bool IsFalse() const { return data_.f.flags == kFalseFlag; } - bool IsTrue() const { return data_.f.flags == kTrueFlag; } - bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } - bool IsObject() const { return data_.f.flags == kObjectFlag; } - bool IsArray() const { return data_.f.flags == kArrayFlag; } - bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } - bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } - bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } - bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } - bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } - bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } - bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } - - // Checks whether a number can be losslessly converted to a double. - bool IsLosslessDouble() const { - if (!IsNumber()) return false; - if (IsUint64()) { - uint64_t u = GetUint64(); - volatile double d = static_cast<double>(u); - return (d >= 0.0) - && (d < static_cast<double>(std::numeric_limits<uint64_t>::max())) - && (u == static_cast<uint64_t>(d)); - } - if (IsInt64()) { - int64_t i = GetInt64(); - volatile double d = static_cast<double>(i); - return (d >= static_cast<double>(std::numeric_limits<int64_t>::min())) - && (d < static_cast<double>(std::numeric_limits<int64_t>::max())) - && (i == static_cast<int64_t>(d)); - } - return true; // double, int, uint are always lossless - } - - // Checks whether a number is a float (possible lossy). - bool IsFloat() const { - if ((data_.f.flags & kDoubleFlag) == 0) - return false; - double d = GetDouble(); - return d >= -3.4028234e38 && d <= 3.4028234e38; - } - // Checks whether a number can be losslessly converted to a float. - bool IsLosslessFloat() const { - if (!IsNumber()) return false; - double a = GetDouble(); - if (a < static_cast<double>(-std::numeric_limits<float>::max()) - || a > static_cast<double>(std::numeric_limits<float>::max())) - return false; - double b = static_cast<double>(static_cast<float>(a)); - return a >= b && a <= b; // Prevent -Wfloat-equal - } - - //@} - - //!@name Null - //@{ - - GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } - - //@} - - //!@name Bool - //@{ - - bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } - //!< Set boolean value - /*! \post IsBool() == true */ - GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } - - //@} - - //!@name Object - //@{ - - //! Set this value as an empty object. - /*! \post IsObject() == true */ - GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } - - //! Get the number of members in the object. - SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } - - //! Check whether the object is empty. - bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } - - //! Get a value from an object associated with the name. - /*! \pre IsObject() == true - \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) - \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. - Since 0.2, if the name is not correct, it will assert. - If user is unsure whether a member exists, user should use HasMember() first. - A better approach is to use FindMember(). - \note Linear time complexity. - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr<internal::IsSame<typename internal::RemoveConst<T>::Type, Ch> >),(GenericValue&)) operator[](T* name) { - GenericValue n(StringRef(name)); - return (*this)[n]; - } - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr<internal::IsSame<typename internal::RemoveConst<T>::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast<GenericValue&>(*this)[name]; } - - //! Get a value from an object associated with the name. - /*! \pre IsObject() == true - \tparam SourceAllocator Allocator of the \c name value - - \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). - And it can also handle strings with embedded null characters. - - \note Linear time complexity. - */ - template <typename SourceAllocator> - GenericValue& operator[](const GenericValue<Encoding, SourceAllocator>& name) { - MemberIterator member = FindMember(name); - if (member != MemberEnd()) - return member->value; - else { - RAPIDJSON_ASSERT(false); // see above note - - // This will generate -Wexit-time-destructors in clang - // static GenericValue NullValue; - // return NullValue; - - // Use static buffer and placement-new to prevent destruction - static char buffer[sizeof(GenericValue)]; - return *new (buffer) GenericValue(); - } - } - template <typename SourceAllocator> - const GenericValue& operator[](const GenericValue<Encoding, SourceAllocator>& name) const { return const_cast<GenericValue&>(*this)[name]; } - -#if RAPIDJSON_HAS_STDSTRING - //! Get a value from an object associated with name (string object). - GenericValue& operator[](const std::basic_string<Ch>& name) { return (*this)[GenericValue(StringRef(name))]; } - const GenericValue& operator[](const std::basic_string<Ch>& name) const { return (*this)[GenericValue(StringRef(name))]; } -#endif - - //! Const member iterator - /*! \pre IsObject() == true */ - ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } - //! Const \em past-the-end member iterator - /*! \pre IsObject() == true */ - ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } - //! Member iterator - /*! \pre IsObject() == true */ - MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } - //! \em Past-the-end member iterator - /*! \pre IsObject() == true */ - MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } - - //! Check whether a member exists in the object. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } - -#if RAPIDJSON_HAS_STDSTRING - //! Check whether a member exists in the object with string object. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - bool HasMember(const std::basic_string<Ch>& name) const { return FindMember(name) != MemberEnd(); } -#endif - - //! Check whether a member exists in the object with GenericValue name. - /*! - This version is faster because it does not need a StrLen(). It can also handle string with null character. - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - template <typename SourceAllocator> - bool HasMember(const GenericValue<Encoding, SourceAllocator>& name) const { return FindMember(name) != MemberEnd(); } - - //! Find member by name. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - - \note Earlier versions of Rapidjson returned a \c NULL pointer, in case - the requested member doesn't exist. For consistency with e.g. - \c std::map, this has been changed to MemberEnd() now. - \note Linear time complexity. - */ - MemberIterator FindMember(const Ch* name) { - GenericValue n(StringRef(name)); - return FindMember(n); - } - - ConstMemberIterator FindMember(const Ch* name) const { return const_cast<GenericValue&>(*this).FindMember(name); } - - //! Find member by name. - /*! - This version is faster because it does not need a StrLen(). It can also handle string with null character. - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - - \note Earlier versions of Rapidjson returned a \c NULL pointer, in case - the requested member doesn't exist. For consistency with e.g. - \c std::map, this has been changed to MemberEnd() now. - \note Linear time complexity. - */ - template <typename SourceAllocator> - MemberIterator FindMember(const GenericValue<Encoding, SourceAllocator>& name) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(name.IsString()); - MemberIterator member = MemberBegin(); - for ( ; member != MemberEnd(); ++member) - if (name.StringEqual(member->name)) - break; - return member; - } - template <typename SourceAllocator> ConstMemberIterator FindMember(const GenericValue<Encoding, SourceAllocator>& name) const { return const_cast<GenericValue&>(*this).FindMember(name); } - -#if RAPIDJSON_HAS_STDSTRING - //! Find member by string object name. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - */ - MemberIterator FindMember(const std::basic_string<Ch>& name) { return FindMember(StringRef(name)); } - ConstMemberIterator FindMember(const std::basic_string<Ch>& name) const { return FindMember(StringRef(name)); } -#endif - - //! Add a member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value Value of any type. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note The ownership of \c name and \c value will be transferred to this object on success. - \pre IsObject() && name.IsString() - \post name.IsNull() && value.IsNull() - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(name.IsString()); - - ObjectData& o = data_.o; - if (o.size >= o.capacity) { - if (o.capacity == 0) { - o.capacity = kDefaultObjectCapacity; - SetMembersPointer(reinterpret_cast<Member*>(allocator.Malloc(o.capacity * sizeof(Member)))); - } - else { - SizeType oldCapacity = o.capacity; - o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 - SetMembersPointer(reinterpret_cast<Member*>(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); - } - } - Member* members = GetMembersPointer(); - members[o.size].name.RawAssign(name); - members[o.size].value.RawAssign(value); - o.size++; - return *this; - } - - //! Add a constant string value as member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value constant string reference as value of member. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(GenericValue& name, StringRefType value, Allocator& allocator) { - GenericValue v(value); - return AddMember(name, v, allocator); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Add a string object as member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value constant string reference as value of member. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(GenericValue& name, std::basic_string<Ch>& value, Allocator& allocator) { - GenericValue v(value, allocator); - return AddMember(name, v, allocator); - } -#endif - - //! Add any primitive value as member (name-value pair) to the object. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param name A string value as name of member. - \param value Value of primitive type \c T as value of member - \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref - AddMember(StringRefType, StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized Constant time complexity. - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (GenericValue&)) - AddMember(GenericValue& name, T value, Allocator& allocator) { - GenericValue v(value); - return AddMember(name, v, allocator); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - - - //! Add a member (name-value pair) to the object. - /*! \param name A constant string reference as name of member. - \param value Value of any type. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note The ownership of \c value will be transferred to this object on success. - \pre IsObject() - \post value.IsNull() - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } - - //! Add a constant string value as member (name-value pair) to the object. - /*! \param name A constant string reference as name of member. - \param value constant string reference as value of member. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { - GenericValue v(value); - return AddMember(name, v, allocator); - } - - //! Add any primitive value as member (name-value pair) to the object. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param name A constant string reference as name of member. - \param value Value of primitive type \c T as value of member - \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref - AddMember(StringRefType, StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized Constant time complexity. - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (GenericValue&)) - AddMember(StringRefType name, T value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } - - //! Remove all members in the object. - /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. - \note Linear time complexity. - */ - void RemoveAllMembers() { - RAPIDJSON_ASSERT(IsObject()); - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - data_.o.size = 0; - } - - //! Remove a member in object by its name. - /*! \param name Name of member to be removed. - \return Whether the member existed. - \note This function may reorder the object members. Use \ref - EraseMember(ConstMemberIterator) if you need to preserve the - relative order of the remaining members. - \note Linear time complexity. - */ - bool RemoveMember(const Ch* name) { - GenericValue n(StringRef(name)); - return RemoveMember(n); - } - -#if RAPIDJSON_HAS_STDSTRING - bool RemoveMember(const std::basic_string<Ch>& name) { return RemoveMember(GenericValue(StringRef(name))); } -#endif - - template <typename SourceAllocator> - bool RemoveMember(const GenericValue<Encoding, SourceAllocator>& name) { - MemberIterator m = FindMember(name); - if (m != MemberEnd()) { - RemoveMember(m); - return true; - } - else - return false; - } - - //! Remove a member in object by iterator. - /*! \param m member iterator (obtained by FindMember() or MemberBegin()). - \return the new iterator after removal. - \note This function may reorder the object members. Use \ref - EraseMember(ConstMemberIterator) if you need to preserve the - relative order of the remaining members. - \note Constant time complexity. - */ - MemberIterator RemoveMember(MemberIterator m) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(GetMembersPointer() != 0); - RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - - MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) - *m = *last; // Move the last one to this place - else - m->~Member(); // Only one left, just destroy - --data_.o.size; - return m; - } - - //! Remove a member from an object by iterator. - /*! \param pos iterator to the member to remove - \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() - \return Iterator following the removed element. - If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. - \note This function preserves the relative order of the remaining object - members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). - \note Linear time complexity. - */ - MemberIterator EraseMember(ConstMemberIterator pos) { - return EraseMember(pos, pos +1); - } - - //! Remove members in the range [first, last) from an object. - /*! \param first iterator to the first member to remove - \param last iterator following the last member to remove - \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() - \return Iterator following the last removed element. - \note This function preserves the relative order of the remaining object - members. - \note Linear time complexity. - */ - MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(GetMembersPointer() != 0); - RAPIDJSON_ASSERT(first >= MemberBegin()); - RAPIDJSON_ASSERT(first <= last); - RAPIDJSON_ASSERT(last <= MemberEnd()); - - MemberIterator pos = MemberBegin() + (first - MemberBegin()); - for (MemberIterator itr = pos; itr != last; ++itr) - itr->~Member(); - std::memmove(&*pos, &*last, static_cast<size_t>(MemberEnd() - last) * sizeof(Member)); - data_.o.size -= static_cast<SizeType>(last - first); - return pos; - } - - //! Erase a member in object by its name. - /*! \param name Name of member to be removed. - \return Whether the member existed. - \note Linear time complexity. - */ - bool EraseMember(const Ch* name) { - GenericValue n(StringRef(name)); - return EraseMember(n); - } - -#if RAPIDJSON_HAS_STDSTRING - bool EraseMember(const std::basic_string<Ch>& name) { return EraseMember(GenericValue(StringRef(name))); } -#endif - - template <typename SourceAllocator> - bool EraseMember(const GenericValue<Encoding, SourceAllocator>& name) { - MemberIterator m = FindMember(name); - if (m != MemberEnd()) { - EraseMember(m); - return true; - } - else - return false; - } - - Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } - ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } - - //@} - - //!@name Array - //@{ - - //! Set this value as an empty array. - /*! \post IsArray == true */ - GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } - - //! Get the number of elements in array. - SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } - - //! Get the capacity of array. - SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } - - //! Check whether the array is empty. - bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } - - //! Remove all elements in the array. - /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. - \note Linear time complexity. - */ - void Clear() { - RAPIDJSON_ASSERT(IsArray()); - GenericValue* e = GetElementsPointer(); - for (GenericValue* v = e; v != e + data_.a.size; ++v) - v->~GenericValue(); - data_.a.size = 0; - } - - //! Get an element from array by index. - /*! \pre IsArray() == true - \param index Zero-based index of element. - \see operator[](T*) - */ - GenericValue& operator[](SizeType index) { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(index < data_.a.size); - return GetElementsPointer()[index]; - } - const GenericValue& operator[](SizeType index) const { return const_cast<GenericValue&>(*this)[index]; } - - //! Element iterator - /*! \pre IsArray() == true */ - ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } - //! \em Past-the-end element iterator - /*! \pre IsArray() == true */ - ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } - //! Constant element iterator - /*! \pre IsArray() == true */ - ConstValueIterator Begin() const { return const_cast<GenericValue&>(*this).Begin(); } - //! Constant \em past-the-end element iterator - /*! \pre IsArray() == true */ - ConstValueIterator End() const { return const_cast<GenericValue&>(*this).End(); } - - //! Request the array to have enough capacity to store elements. - /*! \param newCapacity The capacity that the array at least need to have. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note Linear time complexity. - */ - GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (newCapacity > data_.a.capacity) { - SetElementsPointer(reinterpret_cast<GenericValue*>(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); - data_.a.capacity = newCapacity; - } - return *this; - } - - //! Append a GenericValue at the end of the array. - /*! \param value Value to be appended. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \post value.IsNull() == true - \return The value itself for fluent API. - \note The ownership of \c value will be transferred to this array on success. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - \note Amortized constant time complexity. - */ - GenericValue& PushBack(GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (data_.a.size >= data_.a.capacity) - Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); - GetElementsPointer()[data_.a.size++].RawAssign(value); - return *this; - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { - return PushBack(value, allocator); - } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - - //! Append a constant string reference at the end of the array. - /*! \param value Constant string reference to be appended. - \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \return The value itself for fluent API. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - \note Amortized constant time complexity. - \see GenericStringRef - */ - GenericValue& PushBack(StringRefType value, Allocator& allocator) { - return (*this).template PushBack<StringRefType>(value, allocator); - } - - //! Append a primitive value at the end of the array. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param value Value of primitive type T to be appended. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \return The value itself for fluent API. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref PushBack(GenericValue&, Allocator&) or \ref - PushBack(StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized constant time complexity. - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (GenericValue&)) - PushBack(T value, Allocator& allocator) { - GenericValue v(value); - return PushBack(v, allocator); - } - - //! Remove the last element in the array. - /*! - \note Constant time complexity. - */ - GenericValue& PopBack() { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(!Empty()); - GetElementsPointer()[--data_.a.size].~GenericValue(); - return *this; - } - - //! Remove an element of array by iterator. - /*! - \param pos iterator to the element to remove - \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() - \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. - \note Linear time complexity. - */ - ValueIterator Erase(ConstValueIterator pos) { - return Erase(pos, pos + 1); - } - - //! Remove elements in the range [first, last) of the array. - /*! - \param first iterator to the first element to remove - \param last iterator following the last element to remove - \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() - \return Iterator following the last removed element. - \note Linear time complexity. - */ - ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(data_.a.size > 0); - RAPIDJSON_ASSERT(GetElementsPointer() != 0); - RAPIDJSON_ASSERT(first >= Begin()); - RAPIDJSON_ASSERT(first <= last); - RAPIDJSON_ASSERT(last <= End()); - ValueIterator pos = Begin() + (first - Begin()); - for (ValueIterator itr = pos; itr != last; ++itr) - itr->~GenericValue(); - std::memmove(pos, last, static_cast<size_t>(End() - last) * sizeof(GenericValue)); - data_.a.size -= static_cast<SizeType>(last - first); - return pos; - } - - Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } - ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } - - //@} - - //!@name Number - //@{ - - int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } - unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } - int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } - uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } - - //! Get the value as double type. - /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. - */ - double GetDouble() const { - RAPIDJSON_ASSERT(IsNumber()); - if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. - if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double - if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double - if ((data_.f.flags & kInt64Flag) != 0) return static_cast<double>(data_.n.i64); // int64_t -> double (may lose precision) - RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast<double>(data_.n.u64); // uint64_t -> double (may lose precision) - } - - //! Get the value as float type. - /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. - */ - float GetFloat() const { - return static_cast<float>(GetDouble()); - } - - GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } - GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } - GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } - GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } - GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } - GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(f); return *this; } - - //@} - - //!@name String - //@{ - - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } - - //! Get the length of string. - /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). - */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } - - //! Set this value as a string without copying source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string pointer. - \param length The length of source string, excluding the trailing null terminator. - \return The value itself for fluent API. - \post IsString() == true && GetString() == s && GetStringLength() == length - \see SetString(StringRefType) - */ - GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } - - //! Set this value as a string without copying source string. - /*! \param s source string reference - \return The value itself for fluent API. - \post IsString() == true && GetString() == s && GetStringLength() == s.length - */ - GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } - - //! Set this value as a string by copying from source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string. - \param length The length of source string, excluding the trailing null terminator. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length - */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } - - //! Set this value as a string by copying from source string. - /*! \param s source string. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length - */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } - -#if RAPIDJSON_HAS_STDSTRING - //! Set this value as a string by copying from source string. - /*! \param s source string. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - GenericValue& SetString(const std::basic_string<Ch>& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } -#endif - - //@} - - //!@name Array - //@{ - - //! Templated version for checking whether this value is type T. - /*! - \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string<Ch> - */ - template <typename T> - bool Is() const { return internal::TypeHelper<ValueType, T>::Is(*this); } - - template <typename T> - T Get() const { return internal::TypeHelper<ValueType, T>::Get(*this); } - - template <typename T> - T Get() { return internal::TypeHelper<ValueType, T>::Get(*this); } - - template<typename T> - ValueType& Set(const T& data) { return internal::TypeHelper<ValueType, T>::Set(*this, data); } - - template<typename T> - ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper<ValueType, T>::Set(*this, data, allocator); } - - //@} - - //! Generate events of this value to a Handler. - /*! This function adopts the GoF visitor pattern. - Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. - It can also be used to deep clone this value via GenericDocument, which is also a Handler. - \tparam Handler type of handler. - \param handler An object implementing concept Handler. - */ - template <typename Handler> - bool Accept(Handler& handler) const { - switch(GetType()) { - case kNullType: return handler.Null(); - case kFalseType: return handler.Bool(false); - case kTrueType: return handler.Bool(true); - - case kObjectType: - if (RAPIDJSON_UNLIKELY(!handler.StartObject())) - return false; - for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { - RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. - if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) - return false; - if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) - return false; - } - return handler.EndObject(data_.o.size); - - case kArrayType: - if (RAPIDJSON_UNLIKELY(!handler.StartArray())) - return false; - for (const GenericValue* v = Begin(); v != End(); ++v) - if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) - return false; - return handler.EndArray(data_.a.size); - - case kStringType: - return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); - - default: - RAPIDJSON_ASSERT(GetType() == kNumberType); - if (IsDouble()) return handler.Double(data_.n.d); - else if (IsInt()) return handler.Int(data_.n.i.i); - else if (IsUint()) return handler.Uint(data_.n.u.u); - else if (IsInt64()) return handler.Int64(data_.n.i64); - else return handler.Uint64(data_.n.u64); - } - } - -private: - template <typename, typename> friend class GenericValue; - template <typename, typename, typename> friend class GenericDocument; - - enum { - kBoolFlag = 0x0008, - kNumberFlag = 0x0010, - kIntFlag = 0x0020, - kUintFlag = 0x0040, - kInt64Flag = 0x0080, - kUint64Flag = 0x0100, - kDoubleFlag = 0x0200, - kStringFlag = 0x0400, - kCopyFlag = 0x0800, - kInlineStrFlag = 0x1000, - - // Initial flags of different types. - kNullFlag = kNullType, - kTrueFlag = kTrueType | kBoolFlag, - kFalseFlag = kFalseType | kBoolFlag, - kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, - kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, - kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, - kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, - kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, - kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, - kConstStringFlag = kStringType | kStringFlag, - kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, - kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, - kObjectFlag = kObjectType, - kArrayFlag = kArrayType, - - kTypeMask = 0x07 - }; - - static const SizeType kDefaultArrayCapacity = 16; - static const SizeType kDefaultObjectCapacity = 16; - - struct Flag { -#if RAPIDJSON_48BITPOINTER_OPTIMIZATION - char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer -#elif RAPIDJSON_64BIT - char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes -#else - char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes -#endif - uint16_t flags; - }; - - struct String { - SizeType length; - SizeType hashcode; //!< reserved - const Ch* str; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars - // (excluding the terminating zero) and store a value to determine the length of the contained - // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string - // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as - // the string terminator as well. For getting the string length back from that value just use - // "MaxSize - str[LenPos]". - // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, - // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). - struct ShortString { - enum { MaxChars = sizeof(static_cast<Flag*>(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; - Ch str[MaxChars]; - - inline static bool Usable(SizeType len) { return (MaxSize >= len); } - inline void SetLength(SizeType len) { str[LenPos] = static_cast<Ch>(MaxSize - len); } - inline SizeType GetLength() const { return static_cast<SizeType>(MaxSize - str[LenPos]); } - }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // By using proper binary layout, retrieval of different integer types do not need conversions. - union Number { -#if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN - struct I { - int i; - char padding[4]; - }i; - struct U { - unsigned u; - char padding2[4]; - }u; -#else - struct I { - char padding[4]; - int i; - }i; - struct U { - char padding2[4]; - unsigned u; - }u; -#endif - int64_t i64; - uint64_t u64; - double d; - }; // 8 bytes - - struct ObjectData { - SizeType size; - SizeType capacity; - Member* members; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - struct ArrayData { - SizeType size; - SizeType capacity; - GenericValue* elements; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - union Data { - String s; - ShortString ss; - Number n; - ObjectData o; - ArrayData a; - Flag f; - }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION - - RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } - RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } - RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } - RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } - RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } - RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } - - // Initialize this value as array with initial data, without calling destructor. - void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { - data_.f.flags = kArrayFlag; - if (count) { - GenericValue* e = static_cast<GenericValue*>(allocator.Malloc(count * sizeof(GenericValue))); - SetElementsPointer(e); - std::memcpy(e, values, count * sizeof(GenericValue)); - } - else - SetElementsPointer(0); - data_.a.size = data_.a.capacity = count; - } - - //! Initialize this value as object with initial data, without calling destructor. - void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { - data_.f.flags = kObjectFlag; - if (count) { - Member* m = static_cast<Member*>(allocator.Malloc(count * sizeof(Member))); - SetMembersPointer(m); - std::memcpy(m, members, count * sizeof(Member)); - } - else - SetMembersPointer(0); - data_.o.size = data_.o.capacity = count; - } - - //! Initialize this value as constant string, without calling destructor. - void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { - data_.f.flags = kConstStringFlag; - SetStringPointer(s); - data_.s.length = s.length; - } - - //! Initialize this value as copy string with initial data, without calling destructor. - void SetStringRaw(StringRefType s, Allocator& allocator) { - Ch* str = 0; - if (ShortString::Usable(s.length)) { - data_.f.flags = kShortStringFlag; - data_.ss.SetLength(s.length); - str = data_.ss.str; - } else { - data_.f.flags = kCopyStringFlag; - data_.s.length = s.length; - str = static_cast<Ch *>(allocator.Malloc((s.length + 1) * sizeof(Ch))); - SetStringPointer(str); - } - std::memcpy(str, s, s.length * sizeof(Ch)); - str[s.length] = '\0'; - } - - //! Assignment without calling destructor - void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { - data_ = rhs.data_; - // data_.f.flags = rhs.data_.f.flags; - rhs.data_.f.flags = kNullFlag; - } - - template <typename SourceAllocator> - bool StringEqual(const GenericValue<Encoding, SourceAllocator>& rhs) const { - RAPIDJSON_ASSERT(IsString()); - RAPIDJSON_ASSERT(rhs.IsString()); - - const SizeType len1 = GetStringLength(); - const SizeType len2 = rhs.GetStringLength(); - if(len1 != len2) { return false; } - - const Ch* const str1 = GetString(); - const Ch* const str2 = rhs.GetString(); - if(str1 == str2) { return true; } // fast path for constant string - - return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); - } - - Data data_; -}; - -//! GenericValue with UTF8 encoding -typedef GenericValue<UTF8<> > Value; - -/////////////////////////////////////////////////////////////////////////////// -// GenericDocument - -//! A document for parsing JSON text as DOM. -/*! - \note implements Handler concept - \tparam Encoding Encoding for both parsing and string storage. - \tparam Allocator Allocator for allocating memory for the DOM - \tparam StackAllocator Allocator for allocating memory for stack during parsing. - \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. -*/ -template <typename Encoding, typename Allocator = MemoryPoolAllocator<>, typename StackAllocator = CrtAllocator> -class GenericDocument : public GenericValue<Encoding, Allocator> { -public: - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef GenericValue<Encoding, Allocator> ValueType; //!< Value type of the document. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - - //! Constructor - /*! Creates an empty document of specified type. - \param type Mandatory type of object to create. - \param allocator Optional allocator for allocating memory. - \param stackCapacity Optional initial capacity of stack in bytes. - \param stackAllocator Optional allocator for allocating memory for stack. - */ - explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : - GenericValue<Encoding, Allocator>(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - } - - //! Constructor - /*! Creates an empty document which type is Null. - \param allocator Optional allocator for allocating memory. - \param stackCapacity Optional initial capacity of stack in bytes. - \param stackAllocator Optional allocator for allocating memory for stack. - */ - GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : - allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - : ValueType(std::forward<ValueType>(rhs)), // explicit cast to avoid prohibited move from Document - allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - stack_(std::move(rhs.stack_)), - parseResult_(rhs.parseResult_) - { - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.parseResult_ = ParseResult(); - } -#endif - - ~GenericDocument() { - Destroy(); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move assignment in C++11 - GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - { - // The cast to ValueType is necessary here, because otherwise it would - // attempt to call GenericValue's templated assignment operator. - ValueType::operator=(std::forward<ValueType>(rhs)); - - // Calling the destructor here would prematurely call stack_'s destructor - Destroy(); - - allocator_ = rhs.allocator_; - ownAllocator_ = rhs.ownAllocator_; - stack_ = std::move(rhs.stack_); - parseResult_ = rhs.parseResult_; - - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.parseResult_ = ParseResult(); - - return *this; - } -#endif - - //! Exchange the contents of this document with those of another. - /*! - \param rhs Another document. - \note Constant complexity. - \see GenericValue::Swap - */ - GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { - ValueType::Swap(rhs); - stack_.Swap(rhs.stack_); - internal::Swap(allocator_, rhs.allocator_); - internal::Swap(ownAllocator_, rhs.ownAllocator_); - internal::Swap(parseResult_, rhs.parseResult_); - return *this; - } - - //! free-standing swap function helper - /*! - Helper function to enable support for common swap implementation pattern based on \c std::swap: - \code - void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.doc, b.doc); - // ... - } - \endcode - \see Swap() - */ - friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } - - //! Populate this document by a generator which produces SAX events. - /*! \tparam Generator A functor with <tt>bool f(Handler)</tt> prototype. - \param g Generator functor which sends SAX events to the parameter. - \return The document itself for fluent API. - */ - template <typename Generator> - GenericDocument& Populate(Generator& g) { - ClearStackOnExit scope(*this); - if (g(*this)) { - RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - ValueType::operator=(*stack_.template Pop<ValueType>(1));// Move value from stack to document - } - return *this; - } - - //!@name Parse from stream - //!@{ - - //! Parse JSON text from an input stream (with Encoding conversion) - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam SourceEncoding Encoding of input stream - \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template <unsigned parseFlags, typename SourceEncoding, typename InputStream> - GenericDocument& ParseStream(InputStream& is) { - GenericReader<SourceEncoding, Encoding, StackAllocator> reader( - stack_.HasAllocator() ? &stack_.GetAllocator() : 0); - ClearStackOnExit scope(*this); - parseResult_ = reader.template Parse<parseFlags>(is, *this); - if (parseResult_) { - RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - ValueType::operator=(*stack_.template Pop<ValueType>(1));// Move value from stack to document - } - return *this; - } - - //! Parse JSON text from an input stream - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template <unsigned parseFlags, typename InputStream> - GenericDocument& ParseStream(InputStream& is) { - return ParseStream<parseFlags, Encoding, InputStream>(is); - } - - //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) - /*! \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template <typename InputStream> - GenericDocument& ParseStream(InputStream& is) { - return ParseStream<kParseDefaultFlags, Encoding, InputStream>(is); - } - //!@} - - //!@name Parse in-place from mutable string - //!@{ - - //! Parse JSON text from a mutable string - /*! \tparam parseFlags Combination of \ref ParseFlag. - \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - template <unsigned parseFlags> - GenericDocument& ParseInsitu(Ch* str) { - GenericInsituStringStream<Encoding> s(str); - return ParseStream<parseFlags | kParseInsituFlag>(s); - } - - //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) - /*! \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - GenericDocument& ParseInsitu(Ch* str) { - return ParseInsitu<kParseDefaultFlags>(str); - } - //!@} - - //!@name Parse from read-only string - //!@{ - - //! Parse JSON text from a read-only string (with Encoding conversion) - /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). - \tparam SourceEncoding Transcoding from input Encoding - \param str Read-only zero-terminated string to be parsed. - */ - template <unsigned parseFlags, typename SourceEncoding> - GenericDocument& Parse(const typename SourceEncoding::Ch* str) { - RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - GenericStringStream<SourceEncoding> s(str); - return ParseStream<parseFlags, SourceEncoding>(s); - } - - //! Parse JSON text from a read-only string - /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). - \param str Read-only zero-terminated string to be parsed. - */ - template <unsigned parseFlags> - GenericDocument& Parse(const Ch* str) { - return Parse<parseFlags, Encoding>(str); - } - - //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) - /*! \param str Read-only zero-terminated string to be parsed. - */ - GenericDocument& Parse(const Ch* str) { - return Parse<kParseDefaultFlags>(str); - } - - template <unsigned parseFlags, typename SourceEncoding> - GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { - RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - MemoryStream ms(static_cast<const char*>(str), length * sizeof(typename SourceEncoding::Ch)); - EncodedInputStream<SourceEncoding, MemoryStream> is(ms); - ParseStream<parseFlags, SourceEncoding>(is); - return *this; - } - - template <unsigned parseFlags> - GenericDocument& Parse(const Ch* str, size_t length) { - return Parse<parseFlags, Encoding>(str, length); - } - - GenericDocument& Parse(const Ch* str, size_t length) { - return Parse<kParseDefaultFlags>(str, length); - } - -#if RAPIDJSON_HAS_STDSTRING - template <unsigned parseFlags, typename SourceEncoding> - GenericDocument& Parse(const std::basic_string<typename SourceEncoding::Ch>& str) { - // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) - return Parse<parseFlags, SourceEncoding>(str.c_str()); - } - - template <unsigned parseFlags> - GenericDocument& Parse(const std::basic_string<Ch>& str) { - return Parse<parseFlags, Encoding>(str.c_str()); - } - - GenericDocument& Parse(const std::basic_string<Ch>& str) { - return Parse<kParseDefaultFlags>(str); - } -#endif // RAPIDJSON_HAS_STDSTRING - - //!@} - - //!@name Handling parse errors - //!@{ - - //! Whether a parse error has occured in the last parsing. - bool HasParseError() const { return parseResult_.IsError(); } - - //! Get the \ref ParseErrorCode of last parsing. - ParseErrorCode GetParseError() const { return parseResult_.Code(); } - - //! Get the position of last parsing error in input, 0 otherwise. - size_t GetErrorOffset() const { return parseResult_.Offset(); } - - //! Implicit conversion to get the last parse result -#ifndef __clang // -Wdocumentation - /*! \return \ref ParseResult of the last parse operation - - \code - Document doc; - ParseResult ok = doc.Parse(json); - if (!ok) - printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); - \endcode - */ -#endif - operator ParseResult() const { return parseResult_; } - //!@} - - //! Get the allocator of this document. - Allocator& GetAllocator() { - RAPIDJSON_ASSERT(allocator_); - return *allocator_; - } - - //! Get the capacity of stack in bytes. - size_t GetStackCapacity() const { return stack_.GetCapacity(); } - -private: - // clear stack on any exit from ParseStream, e.g. due to exception - struct ClearStackOnExit { - explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} - ~ClearStackOnExit() { d_.ClearStack(); } - private: - ClearStackOnExit(const ClearStackOnExit&); - ClearStackOnExit& operator=(const ClearStackOnExit&); - GenericDocument& d_; - }; - - // callers of the following private Handler functions - // template <typename,typename,typename> friend class GenericReader; // for parsing - template <typename, typename> friend class GenericValue; // for deep copying - -public: - // Implementation of Handler - bool Null() { new (stack_.template Push<ValueType>()) ValueType(); return true; } - bool Bool(bool b) { new (stack_.template Push<ValueType>()) ValueType(b); return true; } - bool Int(int i) { new (stack_.template Push<ValueType>()) ValueType(i); return true; } - bool Uint(unsigned i) { new (stack_.template Push<ValueType>()) ValueType(i); return true; } - bool Int64(int64_t i) { new (stack_.template Push<ValueType>()) ValueType(i); return true; } - bool Uint64(uint64_t i) { new (stack_.template Push<ValueType>()) ValueType(i); return true; } - bool Double(double d) { new (stack_.template Push<ValueType>()) ValueType(d); return true; } - - bool RawNumber(const Ch* str, SizeType length, bool copy) { - if (copy) - new (stack_.template Push<ValueType>()) ValueType(str, length, GetAllocator()); - else - new (stack_.template Push<ValueType>()) ValueType(str, length); - return true; - } - - bool String(const Ch* str, SizeType length, bool copy) { - if (copy) - new (stack_.template Push<ValueType>()) ValueType(str, length, GetAllocator()); - else - new (stack_.template Push<ValueType>()) ValueType(str, length); - return true; - } - - bool StartObject() { new (stack_.template Push<ValueType>()) ValueType(kObjectType); return true; } - - bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount) { - typename ValueType::Member* members = stack_.template Pop<typename ValueType::Member>(memberCount); - stack_.template Top<ValueType>()->SetObjectRaw(members, memberCount, GetAllocator()); - return true; - } - - bool StartArray() { new (stack_.template Push<ValueType>()) ValueType(kArrayType); return true; } - - bool EndArray(SizeType elementCount) { - ValueType* elements = stack_.template Pop<ValueType>(elementCount); - stack_.template Top<ValueType>()->SetArrayRaw(elements, elementCount, GetAllocator()); - return true; - } - -private: - //! Prohibit copying - GenericDocument(const GenericDocument&); - //! Prohibit assignment - GenericDocument& operator=(const GenericDocument&); - - void ClearStack() { - if (Allocator::kNeedFree) - while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) - (stack_.template Pop<ValueType>(1))->~ValueType(); - else - stack_.Clear(); - stack_.ShrinkToFit(); - } - - void Destroy() { - RAPIDJSON_DELETE(ownAllocator_); - } - - static const size_t kDefaultStackCapacity = 1024; - Allocator* allocator_; - Allocator* ownAllocator_; - internal::Stack<StackAllocator> stack_; - ParseResult parseResult_; -}; - -//! GenericDocument with UTF8 encoding -typedef GenericDocument<UTF8<> > Document; - -// defined here due to the dependency on GenericDocument -template <typename Encoding, typename Allocator> -template <typename SourceAllocator> -inline -GenericValue<Encoding,Allocator>::GenericValue(const GenericValue<Encoding,SourceAllocator>& rhs, Allocator& allocator) -{ - switch (rhs.GetType()) { - case kObjectType: - case kArrayType: { // perform deep copy via SAX Handler - GenericDocument<Encoding,Allocator> d(&allocator); - rhs.Accept(d); - RawAssign(*d.stack_.template Pop<GenericValue>(1)); - } - break; - case kStringType: - if (rhs.data_.f.flags == kConstStringFlag) { - data_.f.flags = rhs.data_.f.flags; - data_ = *reinterpret_cast<const Data*>(&rhs.data_); - } else { - SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); - } - break; - default: - data_.f.flags = rhs.data_.f.flags; - data_ = *reinterpret_cast<const Data*>(&rhs.data_); - break; - } -} - -//! Helper class for accessing Value of array type. -/*! - Instance of this helper class is obtained by \c GenericValue::GetArray(). - In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. -*/ -template <bool Const, typename ValueT> -class GenericArray { -public: - typedef GenericArray<true, ValueT> ConstArray; - typedef GenericArray<false, ValueT> Array; - typedef ValueT PlainType; - typedef typename internal::MaybeAddConst<Const,PlainType>::Type ValueType; - typedef ValueType* ValueIterator; // This may be const or non-const iterator - typedef const ValueT* ConstValueIterator; - typedef typename ValueType::AllocatorType AllocatorType; - typedef typename ValueType::StringRefType StringRefType; - - template <typename, typename> - friend class GenericValue; - - GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} - GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } - ~GenericArray() {} - - SizeType Size() const { return value_.Size(); } - SizeType Capacity() const { return value_.Capacity(); } - bool Empty() const { return value_.Empty(); } - void Clear() const { value_.Clear(); } - ValueType& operator[](SizeType index) const { return value_[index]; } - ValueIterator Begin() const { return value_.Begin(); } - ValueIterator End() const { return value_.End(); } - GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } - GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } - template <typename T> RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } - GenericArray PopBack() const { value_.PopBack(); return *this; } - ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } - ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } - -#if RAPIDJSON_HAS_CXX11_RANGE_FOR - ValueIterator begin() const { return value_.Begin(); } - ValueIterator end() const { return value_.End(); } -#endif - -private: - GenericArray(); - GenericArray(ValueType& value) : value_(value) {} - ValueType& value_; -}; - -//! Helper class for accessing Value of object type. -/*! - Instance of this helper class is obtained by \c GenericValue::GetObject(). - In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. -*/ -template <bool Const, typename ValueT> -class GenericObject { -public: - typedef GenericObject<true, ValueT> ConstObject; - typedef GenericObject<false, ValueT> Object; - typedef ValueT PlainType; - typedef typename internal::MaybeAddConst<Const,PlainType>::Type ValueType; - typedef GenericMemberIterator<Const, typename ValueT::EncodingType, typename ValueT::AllocatorType> MemberIterator; // This may be const or non-const iterator - typedef GenericMemberIterator<true, typename ValueT::EncodingType, typename ValueT::AllocatorType> ConstMemberIterator; - typedef typename ValueType::AllocatorType AllocatorType; - typedef typename ValueType::StringRefType StringRefType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename ValueType::Ch Ch; - - template <typename, typename> - friend class GenericValue; - - GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} - GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } - ~GenericObject() {} - - SizeType MemberCount() const { return value_.MemberCount(); } - bool ObjectEmpty() const { return value_.ObjectEmpty(); } - template <typename T> ValueType& operator[](T* name) const { return value_[name]; } - template <typename SourceAllocator> ValueType& operator[](const GenericValue<EncodingType, SourceAllocator>& name) const { return value_[name]; } -#if RAPIDJSON_HAS_STDSTRING - ValueType& operator[](const std::basic_string<Ch>& name) const { return value_[name]; } -#endif - MemberIterator MemberBegin() const { return value_.MemberBegin(); } - MemberIterator MemberEnd() const { return value_.MemberEnd(); } - bool HasMember(const Ch* name) const { return value_.HasMember(name); } -#if RAPIDJSON_HAS_STDSTRING - bool HasMember(const std::basic_string<Ch>& name) const { return value_.HasMember(name); } -#endif - template <typename SourceAllocator> bool HasMember(const GenericValue<EncodingType, SourceAllocator>& name) const { return value_.HasMember(name); } - MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } - template <typename SourceAllocator> MemberIterator FindMember(const GenericValue<EncodingType, SourceAllocator>& name) const { return value_.FindMember(name); } -#if RAPIDJSON_HAS_STDSTRING - MemberIterator FindMember(const std::basic_string<Ch>& name) const { return value_.FindMember(name); } -#endif - GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#if RAPIDJSON_HAS_STDSTRING - GenericObject AddMember(ValueType& name, std::basic_string<Ch>& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#endif - template <typename T> RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - template <typename T> RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - void RemoveAllMembers() { return value_.RemoveAllMembers(); } - bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } -#if RAPIDJSON_HAS_STDSTRING - bool RemoveMember(const std::basic_string<Ch>& name) const { return value_.RemoveMember(name); } -#endif - template <typename SourceAllocator> bool RemoveMember(const GenericValue<EncodingType, SourceAllocator>& name) const { return value_.RemoveMember(name); } - MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } - MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } - MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } - bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } -#if RAPIDJSON_HAS_STDSTRING - bool EraseMember(const std::basic_string<Ch>& name) const { return EraseMember(ValueType(StringRef(name))); } -#endif - template <typename SourceAllocator> bool EraseMember(const GenericValue<EncodingType, SourceAllocator>& name) const { return value_.EraseMember(name); } - -#if RAPIDJSON_HAS_CXX11_RANGE_FOR - MemberIterator begin() const { return value_.MemberBegin(); } - MemberIterator end() const { return value_.MemberEnd(); } -#endif - -private: - GenericObject(); - GenericObject(ValueType& value) : value_(value) {} - ValueType& value_; -}; - -RAPIDJSON_NAMESPACE_END -RAPIDJSON_DIAG_POP - -#endif // RAPIDJSON_DOCUMENT_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/encodedstream.h b/ext/librethinkdbxx/src/rapidjson/encodedstream.h deleted file mode 100644 index 14506838..00000000 --- a/ext/librethinkdbxx/src/rapidjson/encodedstream.h +++ /dev/null @@ -1,299 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ENCODEDSTREAM_H_ -#define RAPIDJSON_ENCODEDSTREAM_H_ - -#include "stream.h" -#include "memorystream.h" - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Input byte stream wrapper with a statically bound encoding. -/*! - \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. - \tparam InputByteStream Type of input byte stream. For example, FileReadStream. -*/ -template <typename Encoding, typename InputByteStream> -class EncodedInputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); -public: - typedef typename Encoding::Ch Ch; - - EncodedInputStream(InputByteStream& is) : is_(is) { - current_ = Encoding::TakeBOM(is_); - } - - Ch Peek() const { return current_; } - Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } - size_t Tell() const { return is_.Tell(); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - EncodedInputStream(const EncodedInputStream&); - EncodedInputStream& operator=(const EncodedInputStream&); - - InputByteStream& is_; - Ch current_; -}; - -//! Specialized for UTF8 MemoryStream. -template <> -class EncodedInputStream<UTF8<>, MemoryStream> { -public: - typedef UTF8<>::Ch Ch; - - EncodedInputStream(MemoryStream& is) : is_(is) { - if (static_cast<unsigned char>(is_.Peek()) == 0xEFu) is_.Take(); - if (static_cast<unsigned char>(is_.Peek()) == 0xBBu) is_.Take(); - if (static_cast<unsigned char>(is_.Peek()) == 0xBFu) is_.Take(); - } - Ch Peek() const { return is_.Peek(); } - Ch Take() { return is_.Take(); } - size_t Tell() const { return is_.Tell(); } - - // Not implemented - void Put(Ch) {} - void Flush() {} - Ch* PutBegin() { return 0; } - size_t PutEnd(Ch*) { return 0; } - - MemoryStream& is_; - -private: - EncodedInputStream(const EncodedInputStream&); - EncodedInputStream& operator=(const EncodedInputStream&); -}; - -//! Output byte stream wrapper with statically bound encoding. -/*! - \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. - \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. -*/ -template <typename Encoding, typename OutputByteStream> -class EncodedOutputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); -public: - typedef typename Encoding::Ch Ch; - - EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { - if (putBOM) - Encoding::PutBOM(os_); - } - - void Put(Ch c) { Encoding::Put(os_, c); } - void Flush() { os_.Flush(); } - - // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} - Ch Take() { RAPIDJSON_ASSERT(false); return 0;} - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - EncodedOutputStream(const EncodedOutputStream&); - EncodedOutputStream& operator=(const EncodedOutputStream&); - - OutputByteStream& os_; -}; - -#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8<Ch>::x, UTF16LE<Ch>::x, UTF16BE<Ch>::x, UTF32LE<Ch>::x, UTF32BE<Ch>::x - -//! Input stream wrapper with dynamically bound encoding and automatic encoding detection. -/*! - \tparam CharType Type of character for reading. - \tparam InputByteStream type of input byte stream to be wrapped. -*/ -template <typename CharType, typename InputByteStream> -class AutoUTFInputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); -public: - typedef CharType Ch; - - //! Constructor. - /*! - \param is input stream to be wrapped. - \param type UTF encoding type if it is not detected from the stream. - */ - AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { - RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); - DetectType(); - static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; - takeFunc_ = f[type_]; - current_ = takeFunc_(*is_); - } - - UTFType GetType() const { return type_; } - bool HasBOM() const { return hasBOM_; } - - Ch Peek() const { return current_; } - Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } - size_t Tell() const { return is_->Tell(); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - AutoUTFInputStream(const AutoUTFInputStream&); - AutoUTFInputStream& operator=(const AutoUTFInputStream&); - - // Detect encoding type with BOM or RFC 4627 - void DetectType() { - // BOM (Byte Order Mark): - // 00 00 FE FF UTF-32BE - // FF FE 00 00 UTF-32LE - // FE FF UTF-16BE - // FF FE UTF-16LE - // EF BB BF UTF-8 - - const unsigned char* c = reinterpret_cast<const unsigned char *>(is_->Peek4()); - if (!c) - return; - - unsigned bom = static_cast<unsigned>(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); - hasBOM_ = false; - if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } - else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } - else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } - else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } - else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } - - // RFC 4627: Section 3 - // "Since the first two characters of a JSON text will always be ASCII - // characters [RFC0020], it is possible to determine whether an octet - // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking - // at the pattern of nulls in the first four octets." - // 00 00 00 xx UTF-32BE - // 00 xx 00 xx UTF-16BE - // xx 00 00 00 UTF-32LE - // xx 00 xx 00 UTF-16LE - // xx xx xx xx UTF-8 - - if (!hasBOM_) { - unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); - switch (pattern) { - case 0x08: type_ = kUTF32BE; break; - case 0x0A: type_ = kUTF16BE; break; - case 0x01: type_ = kUTF32LE; break; - case 0x05: type_ = kUTF16LE; break; - case 0x0F: type_ = kUTF8; break; - default: break; // Use type defined by user. - } - } - - // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. - if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); - if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); - } - - typedef Ch (*TakeFunc)(InputByteStream& is); - InputByteStream* is_; - UTFType type_; - Ch current_; - TakeFunc takeFunc_; - bool hasBOM_; -}; - -//! Output stream wrapper with dynamically bound encoding and automatic encoding detection. -/*! - \tparam CharType Type of character for writing. - \tparam OutputByteStream type of output byte stream to be wrapped. -*/ -template <typename CharType, typename OutputByteStream> -class AutoUTFOutputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); -public: - typedef CharType Ch; - - //! Constructor. - /*! - \param os output stream to be wrapped. - \param type UTF encoding type. - \param putBOM Whether to write BOM at the beginning of the stream. - */ - AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { - RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); - - // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. - if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); - if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); - - static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; - putFunc_ = f[type_]; - - if (putBOM) - PutBOM(); - } - - UTFType GetType() const { return type_; } - - void Put(Ch c) { putFunc_(*os_, c); } - void Flush() { os_->Flush(); } - - // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} - Ch Take() { RAPIDJSON_ASSERT(false); return 0;} - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - AutoUTFOutputStream(const AutoUTFOutputStream&); - AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); - - void PutBOM() { - typedef void (*PutBOMFunc)(OutputByteStream&); - static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; - f[type_](*os_); - } - - typedef void (*PutFunc)(OutputByteStream&, Ch); - - OutputByteStream* os_; - UTFType type_; - PutFunc putFunc_; -}; - -#undef RAPIDJSON_ENCODINGS_FUNC - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/encodings.h b/ext/librethinkdbxx/src/rapidjson/encodings.h deleted file mode 100644 index baa7c2b1..00000000 --- a/ext/librethinkdbxx/src/rapidjson/encodings.h +++ /dev/null @@ -1,716 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ENCODINGS_H_ -#define RAPIDJSON_ENCODINGS_H_ - -#include "rapidjson.h" - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data -RAPIDJSON_DIAG_OFF(4702) // unreachable code -#elif defined(__GNUC__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -RAPIDJSON_DIAG_OFF(overflow) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// Encoding - -/*! \class rapidjson::Encoding - \brief Concept for encoding of Unicode characters. - -\code -concept Encoding { - typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. - - enum { supportUnicode = 1 }; // or 0 if not supporting unicode - - //! \brief Encode a Unicode codepoint to an output stream. - //! \param os Output stream. - //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. - template<typename OutputStream> - static void Encode(OutputStream& os, unsigned codepoint); - - //! \brief Decode a Unicode codepoint from an input stream. - //! \param is Input stream. - //! \param codepoint Output of the unicode codepoint. - //! \return true if a valid codepoint can be decoded from the stream. - template <typename InputStream> - static bool Decode(InputStream& is, unsigned* codepoint); - - //! \brief Validate one Unicode codepoint from an encoded stream. - //! \param is Input stream to obtain codepoint. - //! \param os Output for copying one codepoint. - //! \return true if it is valid. - //! \note This function just validating and copying the codepoint without actually decode it. - template <typename InputStream, typename OutputStream> - static bool Validate(InputStream& is, OutputStream& os); - - // The following functions are deal with byte streams. - - //! Take a character from input byte stream, skip BOM if exist. - template <typename InputByteStream> - static CharType TakeBOM(InputByteStream& is); - - //! Take a character from input byte stream. - template <typename InputByteStream> - static Ch Take(InputByteStream& is); - - //! Put BOM to output byte stream. - template <typename OutputByteStream> - static void PutBOM(OutputByteStream& os); - - //! Put a character to output byte stream. - template <typename OutputByteStream> - static void Put(OutputByteStream& os, Ch c); -}; -\endcode -*/ - -/////////////////////////////////////////////////////////////////////////////// -// UTF8 - -//! UTF-8 encoding. -/*! http://en.wikipedia.org/wiki/UTF-8 - http://tools.ietf.org/html/rfc3629 - \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. - \note implements Encoding concept -*/ -template<typename CharType = char> -struct UTF8 { - typedef CharType Ch; - - enum { supportUnicode = 1 }; - - template<typename OutputStream> - static void Encode(OutputStream& os, unsigned codepoint) { - if (codepoint <= 0x7F) - os.Put(static_cast<Ch>(codepoint & 0xFF)); - else if (codepoint <= 0x7FF) { - os.Put(static_cast<Ch>(0xC0 | ((codepoint >> 6) & 0xFF))); - os.Put(static_cast<Ch>(0x80 | ((codepoint & 0x3F)))); - } - else if (codepoint <= 0xFFFF) { - os.Put(static_cast<Ch>(0xE0 | ((codepoint >> 12) & 0xFF))); - os.Put(static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); - os.Put(static_cast<Ch>(0x80 | (codepoint & 0x3F))); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - os.Put(static_cast<Ch>(0xF0 | ((codepoint >> 18) & 0xFF))); - os.Put(static_cast<Ch>(0x80 | ((codepoint >> 12) & 0x3F))); - os.Put(static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); - os.Put(static_cast<Ch>(0x80 | (codepoint & 0x3F))); - } - } - - template<typename OutputStream> - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - if (codepoint <= 0x7F) - PutUnsafe(os, static_cast<Ch>(codepoint & 0xFF)); - else if (codepoint <= 0x7FF) { - PutUnsafe(os, static_cast<Ch>(0xC0 | ((codepoint >> 6) & 0xFF))); - PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint & 0x3F)))); - } - else if (codepoint <= 0xFFFF) { - PutUnsafe(os, static_cast<Ch>(0xE0 | ((codepoint >> 12) & 0xFF))); - PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); - PutUnsafe(os, static_cast<Ch>(0x80 | (codepoint & 0x3F))); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - PutUnsafe(os, static_cast<Ch>(0xF0 | ((codepoint >> 18) & 0xFF))); - PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 12) & 0x3F))); - PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); - PutUnsafe(os, static_cast<Ch>(0x80 | (codepoint & 0x3F))); - } - } - - template <typename InputStream> - static bool Decode(InputStream& is, unsigned* codepoint) { -#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast<unsigned char>(c) & 0x3Fu) -#define TRANS(mask) result &= ((GetRange(static_cast<unsigned char>(c)) & mask) != 0) -#define TAIL() COPY(); TRANS(0x70) - typename InputStream::Ch c = is.Take(); - if (!(c & 0x80)) { - *codepoint = static_cast<unsigned char>(c); - return true; - } - - unsigned char type = GetRange(static_cast<unsigned char>(c)); - if (type >= 32) { - *codepoint = 0; - } else { - *codepoint = (0xFF >> type) & static_cast<unsigned char>(c); - } - bool result = true; - switch (type) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; - default: return false; - } -#undef COPY -#undef TRANS -#undef TAIL - } - - template <typename InputStream, typename OutputStream> - static bool Validate(InputStream& is, OutputStream& os) { -#define COPY() os.Put(c = is.Take()) -#define TRANS(mask) result &= ((GetRange(static_cast<unsigned char>(c)) & mask) != 0) -#define TAIL() COPY(); TRANS(0x70) - Ch c; - COPY(); - if (!(c & 0x80)) - return true; - - bool result = true; - switch (GetRange(static_cast<unsigned char>(c))) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; - default: return false; - } -#undef COPY -#undef TRANS -#undef TAIL - } - - static unsigned char GetRange(unsigned char c) { - // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. - static const unsigned char type[] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, - 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - }; - return type[c]; - } - - template <typename InputByteStream> - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - typename InputByteStream::Ch c = Take(is); - if (static_cast<unsigned char>(c) != 0xEFu) return c; - c = is.Take(); - if (static_cast<unsigned char>(c) != 0xBBu) return c; - c = is.Take(); - if (static_cast<unsigned char>(c) != 0xBFu) return c; - c = is.Take(); - return c; - } - - template <typename InputByteStream> - static Ch Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return static_cast<Ch>(is.Take()); - } - - template <typename OutputByteStream> - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(0xEFu)); - os.Put(static_cast<typename OutputByteStream::Ch>(0xBBu)); - os.Put(static_cast<typename OutputByteStream::Ch>(0xBFu)); - } - - template <typename OutputByteStream> - static void Put(OutputByteStream& os, Ch c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(c)); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// UTF16 - -//! UTF-16 encoding. -/*! http://en.wikipedia.org/wiki/UTF-16 - http://tools.ietf.org/html/rfc2781 - \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. - \note implements Encoding concept - - \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. - For streaming, use UTF16LE and UTF16BE, which handle endianness. -*/ -template<typename CharType = wchar_t> -struct UTF16 { - typedef CharType Ch; - RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); - - enum { supportUnicode = 1 }; - - template<typename OutputStream> - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - if (codepoint <= 0xFFFF) { - RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair - os.Put(static_cast<typename OutputStream::Ch>(codepoint)); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - unsigned v = codepoint - 0x10000; - os.Put(static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800)); - os.Put((v & 0x3FF) | 0xDC00); - } - } - - - template<typename OutputStream> - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - if (codepoint <= 0xFFFF) { - RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair - PutUnsafe(os, static_cast<typename OutputStream::Ch>(codepoint)); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - unsigned v = codepoint - 0x10000; - PutUnsafe(os, static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800)); - PutUnsafe(os, (v & 0x3FF) | 0xDC00); - } - } - - template <typename InputStream> - static bool Decode(InputStream& is, unsigned* codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - typename InputStream::Ch c = is.Take(); - if (c < 0xD800 || c > 0xDFFF) { - *codepoint = static_cast<unsigned>(c); - return true; - } - else if (c <= 0xDBFF) { - *codepoint = (static_cast<unsigned>(c) & 0x3FF) << 10; - c = is.Take(); - *codepoint |= (static_cast<unsigned>(c) & 0x3FF); - *codepoint += 0x10000; - return c >= 0xDC00 && c <= 0xDFFF; - } - return false; - } - - template <typename InputStream, typename OutputStream> - static bool Validate(InputStream& is, OutputStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - typename InputStream::Ch c; - os.Put(static_cast<typename OutputStream::Ch>(c = is.Take())); - if (c < 0xD800 || c > 0xDFFF) - return true; - else if (c <= 0xDBFF) { - os.Put(c = is.Take()); - return c >= 0xDC00 && c <= 0xDFFF; - } - return false; - } -}; - -//! UTF-16 little endian encoding. -template<typename CharType = wchar_t> -struct UTF16LE : UTF16<CharType> { - template <typename InputByteStream> - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast<uint16_t>(c) == 0xFEFFu ? Take(is) : c; - } - - template <typename InputByteStream> - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast<uint8_t>(is.Take()); - c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; - return static_cast<CharType>(c); - } - - template <typename OutputByteStream> - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); - } - - template <typename OutputByteStream> - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(static_cast<unsigned>(c) & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>((static_cast<unsigned>(c) >> 8) & 0xFFu)); - } -}; - -//! UTF-16 big endian encoding. -template<typename CharType = wchar_t> -struct UTF16BE : UTF16<CharType> { - template <typename InputByteStream> - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast<uint16_t>(c) == 0xFEFFu ? Take(is) : c; - } - - template <typename InputByteStream> - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; - c |= static_cast<uint8_t>(is.Take()); - return static_cast<CharType>(c); - } - - template <typename OutputByteStream> - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); - } - - template <typename OutputByteStream> - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>((static_cast<unsigned>(c) >> 8) & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>(static_cast<unsigned>(c) & 0xFFu)); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// UTF32 - -//! UTF-32 encoding. -/*! http://en.wikipedia.org/wiki/UTF-32 - \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. - \note implements Encoding concept - - \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. - For streaming, use UTF32LE and UTF32BE, which handle endianness. -*/ -template<typename CharType = unsigned> -struct UTF32 { - typedef CharType Ch; - RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); - - enum { supportUnicode = 1 }; - - template<typename OutputStream> - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - os.Put(codepoint); - } - - template<typename OutputStream> - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - PutUnsafe(os, codepoint); - } - - template <typename InputStream> - static bool Decode(InputStream& is, unsigned* codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); - Ch c = is.Take(); - *codepoint = c; - return c <= 0x10FFFF; - } - - template <typename InputStream, typename OutputStream> - static bool Validate(InputStream& is, OutputStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); - Ch c; - os.Put(c = is.Take()); - return c <= 0x10FFFF; - } -}; - -//! UTF-32 little endian enocoding. -template<typename CharType = unsigned> -struct UTF32LE : UTF32<CharType> { - template <typename InputByteStream> - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast<uint32_t>(c) == 0x0000FEFFu ? Take(is) : c; - } - - template <typename InputByteStream> - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast<uint8_t>(is.Take()); - c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; - c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 16; - c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 24; - return static_cast<CharType>(c); - } - - template <typename OutputByteStream> - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); - os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); - os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); - } - - template <typename OutputByteStream> - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(c & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>((c >> 8) & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>((c >> 16) & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>((c >> 24) & 0xFFu)); - } -}; - -//! UTF-32 big endian encoding. -template<typename CharType = unsigned> -struct UTF32BE : UTF32<CharType> { - template <typename InputByteStream> - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast<uint32_t>(c) == 0x0000FEFFu ? Take(is) : c; - } - - template <typename InputByteStream> - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 24; - c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 16; - c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; - c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())); - return static_cast<CharType>(c); - } - - template <typename OutputByteStream> - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); - os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); - os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); - } - - template <typename OutputByteStream> - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>((c >> 24) & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>((c >> 16) & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>((c >> 8) & 0xFFu)); - os.Put(static_cast<typename OutputByteStream::Ch>(c & 0xFFu)); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// ASCII - -//! ASCII encoding. -/*! http://en.wikipedia.org/wiki/ASCII - \tparam CharType Code unit for storing 7-bit ASCII data. Default is char. - \note implements Encoding concept -*/ -template<typename CharType = char> -struct ASCII { - typedef CharType Ch; - - enum { supportUnicode = 0 }; - - template<typename OutputStream> - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_ASSERT(codepoint <= 0x7F); - os.Put(static_cast<Ch>(codepoint & 0xFF)); - } - - template<typename OutputStream> - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_ASSERT(codepoint <= 0x7F); - PutUnsafe(os, static_cast<Ch>(codepoint & 0xFF)); - } - - template <typename InputStream> - static bool Decode(InputStream& is, unsigned* codepoint) { - uint8_t c = static_cast<uint8_t>(is.Take()); - *codepoint = c; - return c <= 0X7F; - } - - template <typename InputStream, typename OutputStream> - static bool Validate(InputStream& is, OutputStream& os) { - uint8_t c = static_cast<uint8_t>(is.Take()); - os.Put(static_cast<typename OutputStream::Ch>(c)); - return c <= 0x7F; - } - - template <typename InputByteStream> - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - uint8_t c = static_cast<uint8_t>(Take(is)); - return static_cast<Ch>(c); - } - - template <typename InputByteStream> - static Ch Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return static_cast<Ch>(is.Take()); - } - - template <typename OutputByteStream> - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - (void)os; - } - - template <typename OutputByteStream> - static void Put(OutputByteStream& os, Ch c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast<typename OutputByteStream::Ch>(c)); - } -}; - -/////////////////////////////////////////////////////////////////////////////// -// AutoUTF - -//! Runtime-specified UTF encoding type of a stream. -enum UTFType { - kUTF8 = 0, //!< UTF-8. - kUTF16LE = 1, //!< UTF-16 little endian. - kUTF16BE = 2, //!< UTF-16 big endian. - kUTF32LE = 3, //!< UTF-32 little endian. - kUTF32BE = 4 //!< UTF-32 big endian. -}; - -//! Dynamically select encoding according to stream's runtime-specified UTF encoding type. -/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). -*/ -template<typename CharType> -struct AutoUTF { - typedef CharType Ch; - - enum { supportUnicode = 1 }; - -#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8<Ch>::x, UTF16LE<Ch>::x, UTF16BE<Ch>::x, UTF32LE<Ch>::x, UTF32BE<Ch>::x - - template<typename OutputStream> - RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { - typedef void (*EncodeFunc)(OutputStream&, unsigned); - static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; - (*f[os.GetType()])(os, codepoint); - } - - template<typename OutputStream> - RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - typedef void (*EncodeFunc)(OutputStream&, unsigned); - static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; - (*f[os.GetType()])(os, codepoint); - } - - template <typename InputStream> - RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { - typedef bool (*DecodeFunc)(InputStream&, unsigned*); - static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; - return (*f[is.GetType()])(is, codepoint); - } - - template <typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - typedef bool (*ValidateFunc)(InputStream&, OutputStream&); - static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; - return (*f[is.GetType()])(is, os); - } - -#undef RAPIDJSON_ENCODINGS_FUNC -}; - -/////////////////////////////////////////////////////////////////////////////// -// Transcoder - -//! Encoding conversion. -template<typename SourceEncoding, typename TargetEncoding> -struct Transcoder { - //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. - template<typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { - unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) - return false; - TargetEncoding::Encode(os, codepoint); - return true; - } - - template<typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { - unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) - return false; - TargetEncoding::EncodeUnsafe(os, codepoint); - return true; - } - - //! Validate one Unicode codepoint from an encoded stream. - template<typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - return Transcode(is, os); // Since source/target encoding is different, must transcode. - } -}; - -// Forward declaration. -template<typename Stream> -inline void PutUnsafe(Stream& stream, typename Stream::Ch c); - -//! Specialization of Transcoder with same source and target encoding. -template<typename Encoding> -struct Transcoder<Encoding, Encoding> { - template<typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { - os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. - return true; - } - - template<typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { - PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. - return true; - } - - template<typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - return Encoding::Validate(is, os); // source/target encoding are the same - } -}; - -RAPIDJSON_NAMESPACE_END - -#if defined(__GNUC__) || defined(_MSC_VER) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/error/en.h b/ext/librethinkdbxx/src/rapidjson/error/en.h deleted file mode 100644 index 2db838bf..00000000 --- a/ext/librethinkdbxx/src/rapidjson/error/en.h +++ /dev/null @@ -1,74 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ERROR_EN_H_ -#define RAPIDJSON_ERROR_EN_H_ - -#include "error.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(covered-switch-default) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Maps error code of parsing into error message. -/*! - \ingroup RAPIDJSON_ERRORS - \param parseErrorCode Error code obtained in parsing. - \return the error message. - \note User can make a copy of this function for localization. - Using switch-case is safer for future modification of error codes. -*/ -inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { - switch (parseErrorCode) { - case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); - - case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); - case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); - - case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); - - case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); - case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); - case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); - - case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); - - case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); - case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); - case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); - case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); - case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); - - case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); - case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); - case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); - - case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); - case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); - - default: return RAPIDJSON_ERROR_STRING("Unknown error."); - } -} - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_ERROR_EN_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/error/error.h b/ext/librethinkdbxx/src/rapidjson/error/error.h deleted file mode 100644 index 95cb31a7..00000000 --- a/ext/librethinkdbxx/src/rapidjson/error/error.h +++ /dev/null @@ -1,155 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ERROR_ERROR_H_ -#define RAPIDJSON_ERROR_ERROR_H_ - -#include "../rapidjson.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -/*! \file error.h */ - -/*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ERROR_CHARTYPE - -//! Character type of error messages. -/*! \ingroup RAPIDJSON_ERRORS - The default character type is \c char. - On Windows, user can define this macro as \c TCHAR for supporting both - unicode/non-unicode settings. -*/ -#ifndef RAPIDJSON_ERROR_CHARTYPE -#define RAPIDJSON_ERROR_CHARTYPE char -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ERROR_STRING - -//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. -/*! \ingroup RAPIDJSON_ERRORS - By default this conversion macro does nothing. - On Windows, user can define this macro as \c _T(x) for supporting both - unicode/non-unicode settings. -*/ -#ifndef RAPIDJSON_ERROR_STRING -#define RAPIDJSON_ERROR_STRING(x) x -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// ParseErrorCode - -//! Error code of parsing. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericReader::Parse, GenericReader::GetParseErrorCode -*/ -enum ParseErrorCode { - kParseErrorNone = 0, //!< No error. - - kParseErrorDocumentEmpty, //!< The document is empty. - kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. - - kParseErrorValueInvalid, //!< Invalid value. - - kParseErrorObjectMissName, //!< Missing a name for object member. - kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. - kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. - - kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. - - kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. - kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. - kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. - kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. - kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. - - kParseErrorNumberTooBig, //!< Number too big to be stored in double. - kParseErrorNumberMissFraction, //!< Miss fraction part in number. - kParseErrorNumberMissExponent, //!< Miss exponent in number. - - kParseErrorTermination, //!< Parsing was terminated. - kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. -}; - -//! Result of parsing (wraps ParseErrorCode) -/*! - \ingroup RAPIDJSON_ERRORS - \code - Document doc; - ParseResult ok = doc.Parse("[42]"); - if (!ok) { - fprintf(stderr, "JSON parse error: %s (%u)", - GetParseError_En(ok.Code()), ok.Offset()); - exit(EXIT_FAILURE); - } - \endcode - \see GenericReader::Parse, GenericDocument::Parse -*/ -struct ParseResult { -public: - //! Default constructor, no error. - ParseResult() : code_(kParseErrorNone), offset_(0) {} - //! Constructor to set an error. - ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} - - //! Get the error code. - ParseErrorCode Code() const { return code_; } - //! Get the error offset, if \ref IsError(), 0 otherwise. - size_t Offset() const { return offset_; } - - //! Conversion to \c bool, returns \c true, iff !\ref IsError(). - operator bool() const { return !IsError(); } - //! Whether the result is an error. - bool IsError() const { return code_ != kParseErrorNone; } - - bool operator==(const ParseResult& that) const { return code_ == that.code_; } - bool operator==(ParseErrorCode code) const { return code_ == code; } - friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } - - //! Reset error code. - void Clear() { Set(kParseErrorNone); } - //! Update error code and offset. - void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } - -private: - ParseErrorCode code_; - size_t offset_; -}; - -//! Function pointer type of GetParseError(). -/*! \ingroup RAPIDJSON_ERRORS - - This is the prototype for \c GetParseError_X(), where \c X is a locale. - User can dynamically change locale in runtime, e.g.: -\code - GetParseErrorFunc GetParseError = GetParseError_En; // or whatever - const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); -\endcode -*/ -typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_ERROR_ERROR_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/filereadstream.h b/ext/librethinkdbxx/src/rapidjson/filereadstream.h deleted file mode 100644 index b56ea13b..00000000 --- a/ext/librethinkdbxx/src/rapidjson/filereadstream.h +++ /dev/null @@ -1,99 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_FILEREADSTREAM_H_ -#define RAPIDJSON_FILEREADSTREAM_H_ - -#include "stream.h" -#include <cstdio> - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(unreachable-code) -RAPIDJSON_DIAG_OFF(missing-noreturn) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! File byte stream for input using fread(). -/*! - \note implements Stream concept -*/ -class FileReadStream { -public: - typedef char Ch; //!< Character type (byte). - - //! Constructor. - /*! - \param fp File pointer opened for read. - \param buffer user-supplied buffer. - \param bufferSize size of buffer in bytes. Must >=4 bytes. - */ - FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { - RAPIDJSON_ASSERT(fp_ != 0); - RAPIDJSON_ASSERT(bufferSize >= 4); - Read(); - } - - Ch Peek() const { return *current_; } - Ch Take() { Ch c = *current_; Read(); return c; } - size_t Tell() const { return count_ + static_cast<size_t>(current_ - buffer_); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - return (current_ + 4 <= bufferLast_) ? current_ : 0; - } - -private: - void Read() { - if (current_ < bufferLast_) - ++current_; - else if (!eof_) { - count_ += readCount_; - readCount_ = fread(buffer_, 1, bufferSize_, fp_); - bufferLast_ = buffer_ + readCount_ - 1; - current_ = buffer_; - - if (readCount_ < bufferSize_) { - buffer_[readCount_] = '\0'; - ++bufferLast_; - eof_ = true; - } - } - } - - std::FILE* fp_; - Ch *buffer_; - size_t bufferSize_; - Ch *bufferLast_; - Ch *current_; - size_t readCount_; - size_t count_; //!< Number of characters read - bool eof_; -}; - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/filewritestream.h b/ext/librethinkdbxx/src/rapidjson/filewritestream.h deleted file mode 100644 index 6378dd60..00000000 --- a/ext/librethinkdbxx/src/rapidjson/filewritestream.h +++ /dev/null @@ -1,104 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_FILEWRITESTREAM_H_ -#define RAPIDJSON_FILEWRITESTREAM_H_ - -#include "stream.h" -#include <cstdio> - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(unreachable-code) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Wrapper of C file stream for input using fread(). -/*! - \note implements Stream concept -*/ -class FileWriteStream { -public: - typedef char Ch; //!< Character type. Only support char. - - FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { - RAPIDJSON_ASSERT(fp_ != 0); - } - - void Put(char c) { - if (current_ >= bufferEnd_) - Flush(); - - *current_++ = c; - } - - void PutN(char c, size_t n) { - size_t avail = static_cast<size_t>(bufferEnd_ - current_); - while (n > avail) { - std::memset(current_, c, avail); - current_ += avail; - Flush(); - n -= avail; - avail = static_cast<size_t>(bufferEnd_ - current_); - } - - if (n > 0) { - std::memset(current_, c, n); - current_ += n; - } - } - - void Flush() { - if (current_ != buffer_) { - size_t result = fwrite(buffer_, 1, static_cast<size_t>(current_ - buffer_), fp_); - if (result < static_cast<size_t>(current_ - buffer_)) { - // failure deliberately ignored at this time - // added to avoid warn_unused_result build errors - } - current_ = buffer_; - } - } - - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - // Prohibit copy constructor & assignment operator. - FileWriteStream(const FileWriteStream&); - FileWriteStream& operator=(const FileWriteStream&); - - std::FILE* fp_; - char *buffer_; - char *bufferEnd_; - char *current_; -}; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(FileWriteStream& stream, char c, size_t n) { - stream.PutN(c, n); -} - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/fwd.h b/ext/librethinkdbxx/src/rapidjson/fwd.h deleted file mode 100644 index e8104e84..00000000 --- a/ext/librethinkdbxx/src/rapidjson/fwd.h +++ /dev/null @@ -1,151 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_FWD_H_ -#define RAPIDJSON_FWD_H_ - -#include "rapidjson.h" - -RAPIDJSON_NAMESPACE_BEGIN - -// encodings.h - -template<typename CharType> struct UTF8; -template<typename CharType> struct UTF16; -template<typename CharType> struct UTF16BE; -template<typename CharType> struct UTF16LE; -template<typename CharType> struct UTF32; -template<typename CharType> struct UTF32BE; -template<typename CharType> struct UTF32LE; -template<typename CharType> struct ASCII; -template<typename CharType> struct AutoUTF; - -template<typename SourceEncoding, typename TargetEncoding> -struct Transcoder; - -// allocators.h - -class CrtAllocator; - -template <typename BaseAllocator> -class MemoryPoolAllocator; - -// stream.h - -template <typename Encoding> -struct GenericStringStream; - -typedef GenericStringStream<UTF8<char> > StringStream; - -template <typename Encoding> -struct GenericInsituStringStream; - -typedef GenericInsituStringStream<UTF8<char> > InsituStringStream; - -// stringbuffer.h - -template <typename Encoding, typename Allocator> -class GenericStringBuffer; - -typedef GenericStringBuffer<UTF8<char>, CrtAllocator> StringBuffer; - -// filereadstream.h - -class FileReadStream; - -// filewritestream.h - -class FileWriteStream; - -// memorybuffer.h - -template <typename Allocator> -struct GenericMemoryBuffer; - -typedef GenericMemoryBuffer<CrtAllocator> MemoryBuffer; - -// memorystream.h - -struct MemoryStream; - -// reader.h - -template<typename Encoding, typename Derived> -struct BaseReaderHandler; - -template <typename SourceEncoding, typename TargetEncoding, typename StackAllocator> -class GenericReader; - -typedef GenericReader<UTF8<char>, UTF8<char>, CrtAllocator> Reader; - -// writer.h - -template<typename OutputStream, typename SourceEncoding, typename TargetEncoding, typename StackAllocator, unsigned writeFlags> -class Writer; - -// prettywriter.h - -template<typename OutputStream, typename SourceEncoding, typename TargetEncoding, typename StackAllocator, unsigned writeFlags> -class PrettyWriter; - -// document.h - -template <typename Encoding, typename Allocator> -struct GenericMember; - -template <bool Const, typename Encoding, typename Allocator> -class GenericMemberIterator; - -template<typename CharType> -struct GenericStringRef; - -template <typename Encoding, typename Allocator> -class GenericValue; - -typedef GenericValue<UTF8<char>, MemoryPoolAllocator<CrtAllocator> > Value; - -template <typename Encoding, typename Allocator, typename StackAllocator> -class GenericDocument; - -typedef GenericDocument<UTF8<char>, MemoryPoolAllocator<CrtAllocator>, CrtAllocator> Document; - -// pointer.h - -template <typename ValueType, typename Allocator> -class GenericPointer; - -typedef GenericPointer<Value, CrtAllocator> Pointer; - -// schema.h - -template <typename SchemaDocumentType> -class IGenericRemoteSchemaDocumentProvider; - -template <typename ValueT, typename Allocator> -class GenericSchemaDocument; - -typedef GenericSchemaDocument<Value, CrtAllocator> SchemaDocument; -typedef IGenericRemoteSchemaDocumentProvider<SchemaDocument> IRemoteSchemaDocumentProvider; - -template < - typename SchemaDocumentType, - typename OutputHandler, - typename StateAllocator> -class GenericSchemaValidator; - -typedef GenericSchemaValidator<SchemaDocument, BaseReaderHandler<UTF8<char>, void>, CrtAllocator> SchemaValidator; - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_RAPIDJSONFWD_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h b/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h deleted file mode 100644 index 9d3e88c9..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h +++ /dev/null @@ -1,290 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_BIGINTEGER_H_ -#define RAPIDJSON_BIGINTEGER_H_ - -#include "../rapidjson.h" - -#if defined(_MSC_VER) && defined(_M_AMD64) -#include <intrin.h> // for _umul128 -#pragma intrinsic(_umul128) -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -class BigInteger { -public: - typedef uint64_t Type; - - BigInteger(const BigInteger& rhs) : count_(rhs.count_) { - std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); - } - - explicit BigInteger(uint64_t u) : count_(1) { - digits_[0] = u; - } - - BigInteger(const char* decimals, size_t length) : count_(1) { - RAPIDJSON_ASSERT(length > 0); - digits_[0] = 0; - size_t i = 0; - const size_t kMaxDigitPerIteration = 19; // 2^64 = 18446744073709551616 > 10^19 - while (length >= kMaxDigitPerIteration) { - AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration); - length -= kMaxDigitPerIteration; - i += kMaxDigitPerIteration; - } - - if (length > 0) - AppendDecimal64(decimals + i, decimals + i + length); - } - - BigInteger& operator=(const BigInteger &rhs) - { - if (this != &rhs) { - count_ = rhs.count_; - std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); - } - return *this; - } - - BigInteger& operator=(uint64_t u) { - digits_[0] = u; - count_ = 1; - return *this; - } - - BigInteger& operator+=(uint64_t u) { - Type backup = digits_[0]; - digits_[0] += u; - for (size_t i = 0; i < count_ - 1; i++) { - if (digits_[i] >= backup) - return *this; // no carry - backup = digits_[i + 1]; - digits_[i + 1] += 1; - } - - // Last carry - if (digits_[count_ - 1] < backup) - PushBack(1); - - return *this; - } - - BigInteger& operator*=(uint64_t u) { - if (u == 0) return *this = 0; - if (u == 1) return *this; - if (*this == 1) return *this = u; - - uint64_t k = 0; - for (size_t i = 0; i < count_; i++) { - uint64_t hi; - digits_[i] = MulAdd64(digits_[i], u, k, &hi); - k = hi; - } - - if (k > 0) - PushBack(k); - - return *this; - } - - BigInteger& operator*=(uint32_t u) { - if (u == 0) return *this = 0; - if (u == 1) return *this; - if (*this == 1) return *this = u; - - uint64_t k = 0; - for (size_t i = 0; i < count_; i++) { - const uint64_t c = digits_[i] >> 32; - const uint64_t d = digits_[i] & 0xFFFFFFFF; - const uint64_t uc = u * c; - const uint64_t ud = u * d; - const uint64_t p0 = ud + k; - const uint64_t p1 = uc + (p0 >> 32); - digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32); - k = p1 >> 32; - } - - if (k > 0) - PushBack(k); - - return *this; - } - - BigInteger& operator<<=(size_t shift) { - if (IsZero() || shift == 0) return *this; - - size_t offset = shift / kTypeBit; - size_t interShift = shift % kTypeBit; - RAPIDJSON_ASSERT(count_ + offset <= kCapacity); - - if (interShift == 0) { - std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type)); - count_ += offset; - } - else { - digits_[count_] = 0; - for (size_t i = count_; i > 0; i--) - digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift)); - digits_[offset] = digits_[0] << interShift; - count_ += offset; - if (digits_[count_]) - count_++; - } - - std::memset(digits_, 0, offset * sizeof(Type)); - - return *this; - } - - bool operator==(const BigInteger& rhs) const { - return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0; - } - - bool operator==(const Type rhs) const { - return count_ == 1 && digits_[0] == rhs; - } - - BigInteger& MultiplyPow5(unsigned exp) { - static const uint32_t kPow5[12] = { - 5, - 5 * 5, - 5 * 5 * 5, - 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 - }; - if (exp == 0) return *this; - for (; exp >= 27; exp -= 27) *this *= RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27 - for (; exp >= 13; exp -= 13) *this *= static_cast<uint32_t>(1220703125u); // 5^13 - if (exp > 0) *this *= kPow5[exp - 1]; - return *this; - } - - // Compute absolute difference of this and rhs. - // Assume this != rhs - bool Difference(const BigInteger& rhs, BigInteger* out) const { - int cmp = Compare(rhs); - RAPIDJSON_ASSERT(cmp != 0); - const BigInteger *a, *b; // Makes a > b - bool ret; - if (cmp < 0) { a = &rhs; b = this; ret = true; } - else { a = this; b = &rhs; ret = false; } - - Type borrow = 0; - for (size_t i = 0; i < a->count_; i++) { - Type d = a->digits_[i] - borrow; - if (i < b->count_) - d -= b->digits_[i]; - borrow = (d > a->digits_[i]) ? 1 : 0; - out->digits_[i] = d; - if (d != 0) - out->count_ = i + 1; - } - - return ret; - } - - int Compare(const BigInteger& rhs) const { - if (count_ != rhs.count_) - return count_ < rhs.count_ ? -1 : 1; - - for (size_t i = count_; i-- > 0;) - if (digits_[i] != rhs.digits_[i]) - return digits_[i] < rhs.digits_[i] ? -1 : 1; - - return 0; - } - - size_t GetCount() const { return count_; } - Type GetDigit(size_t index) const { RAPIDJSON_ASSERT(index < count_); return digits_[index]; } - bool IsZero() const { return count_ == 1 && digits_[0] == 0; } - -private: - void AppendDecimal64(const char* begin, const char* end) { - uint64_t u = ParseUint64(begin, end); - if (IsZero()) - *this = u; - else { - unsigned exp = static_cast<unsigned>(end - begin); - (MultiplyPow5(exp) <<= exp) += u; // *this = *this * 10^exp + u - } - } - - void PushBack(Type digit) { - RAPIDJSON_ASSERT(count_ < kCapacity); - digits_[count_++] = digit; - } - - static uint64_t ParseUint64(const char* begin, const char* end) { - uint64_t r = 0; - for (const char* p = begin; p != end; ++p) { - RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); - r = r * 10u + static_cast<unsigned>(*p - '0'); - } - return r; - } - - // Assume a * b + k < 2^128 - static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) { -#if defined(_MSC_VER) && defined(_M_AMD64) - uint64_t low = _umul128(a, b, outHigh) + k; - if (low < k) - (*outHigh)++; - return low; -#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) - __extension__ typedef unsigned __int128 uint128; - uint128 p = static_cast<uint128>(a) * static_cast<uint128>(b); - p += k; - *outHigh = static_cast<uint64_t>(p >> 64); - return static_cast<uint64_t>(p); -#else - const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32; - uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1; - x1 += (x0 >> 32); // can't give carry - x1 += x2; - if (x1 < x2) - x3 += (static_cast<uint64_t>(1) << 32); - uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF); - uint64_t hi = x3 + (x1 >> 32); - - lo += k; - if (lo < k) - hi++; - *outHigh = hi; - return lo; -#endif - } - - static const size_t kBitCount = 3328; // 64bit * 54 > 10^1000 - static const size_t kCapacity = kBitCount / sizeof(Type); - static const size_t kTypeBit = sizeof(Type) * 8; - - Type digits_[kCapacity]; - size_t count_; -}; - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_BIGINTEGER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h b/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h deleted file mode 100644 index c9fefdc6..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h +++ /dev/null @@ -1,258 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -// This is a C++ header-only implementation of Grisu2 algorithm from the publication: -// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with -// integers." ACM Sigplan Notices 45.6 (2010): 233-243. - -#ifndef RAPIDJSON_DIYFP_H_ -#define RAPIDJSON_DIYFP_H_ - -#include "../rapidjson.h" - -#if defined(_MSC_VER) && defined(_M_AMD64) -#include <intrin.h> -#pragma intrinsic(_BitScanReverse64) -#pragma intrinsic(_umul128) -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -struct DiyFp { - DiyFp() : f(), e() {} - - DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} - - explicit DiyFp(double d) { - union { - double d; - uint64_t u64; - } u = { d }; - - int biased_e = static_cast<int>((u.u64 & kDpExponentMask) >> kDpSignificandSize); - uint64_t significand = (u.u64 & kDpSignificandMask); - if (biased_e != 0) { - f = significand + kDpHiddenBit; - e = biased_e - kDpExponentBias; - } - else { - f = significand; - e = kDpMinExponent + 1; - } - } - - DiyFp operator-(const DiyFp& rhs) const { - return DiyFp(f - rhs.f, e); - } - - DiyFp operator*(const DiyFp& rhs) const { -#if defined(_MSC_VER) && defined(_M_AMD64) - uint64_t h; - uint64_t l = _umul128(f, rhs.f, &h); - if (l & (uint64_t(1) << 63)) // rounding - h++; - return DiyFp(h, e + rhs.e + 64); -#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) - __extension__ typedef unsigned __int128 uint128; - uint128 p = static_cast<uint128>(f) * static_cast<uint128>(rhs.f); - uint64_t h = static_cast<uint64_t>(p >> 64); - uint64_t l = static_cast<uint64_t>(p); - if (l & (uint64_t(1) << 63)) // rounding - h++; - return DiyFp(h, e + rhs.e + 64); -#else - const uint64_t M32 = 0xFFFFFFFF; - const uint64_t a = f >> 32; - const uint64_t b = f & M32; - const uint64_t c = rhs.f >> 32; - const uint64_t d = rhs.f & M32; - const uint64_t ac = a * c; - const uint64_t bc = b * c; - const uint64_t ad = a * d; - const uint64_t bd = b * d; - uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); - tmp += 1U << 31; /// mult_round - return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); -#endif - } - - DiyFp Normalize() const { -#if defined(_MSC_VER) && defined(_M_AMD64) - unsigned long index; - _BitScanReverse64(&index, f); - return DiyFp(f << (63 - index), e - (63 - index)); -#elif defined(__GNUC__) && __GNUC__ >= 4 - int s = __builtin_clzll(f); - return DiyFp(f << s, e - s); -#else - DiyFp res = *this; - while (!(res.f & (static_cast<uint64_t>(1) << 63))) { - res.f <<= 1; - res.e--; - } - return res; -#endif - } - - DiyFp NormalizeBoundary() const { - DiyFp res = *this; - while (!(res.f & (kDpHiddenBit << 1))) { - res.f <<= 1; - res.e--; - } - res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); - res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); - return res; - } - - void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { - DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); - DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); - mi.f <<= mi.e - pl.e; - mi.e = pl.e; - *plus = pl; - *minus = mi; - } - - double ToDouble() const { - union { - double d; - uint64_t u64; - }u; - const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : - static_cast<uint64_t>(e + kDpExponentBias); - u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); - return u.d; - } - - static const int kDiySignificandSize = 64; - static const int kDpSignificandSize = 52; - static const int kDpExponentBias = 0x3FF + kDpSignificandSize; - static const int kDpMaxExponent = 0x7FF - kDpExponentBias; - static const int kDpMinExponent = -kDpExponentBias; - static const int kDpDenormalExponent = -kDpExponentBias + 1; - static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); - static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); - static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); - - uint64_t f; - int e; -}; - -inline DiyFp GetCachedPowerByIndex(size_t index) { - // 10^-348, 10^-340, ..., 10^340 - static const uint64_t kCachedPowers_F[] = { - RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), - RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), - RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), - RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), - RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), - RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), - RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), - RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), - RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), - RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), - RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), - RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), - RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), - RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), - RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), - RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), - RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), - RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), - RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), - RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), - RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), - RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), - RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), - RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), - RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), - RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), - RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), - RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), - RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), - RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), - RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), - RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), - RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), - RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), - RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), - RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), - RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), - RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), - RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), - RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), - RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), - RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), - RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), - RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) - }; - static const int16_t kCachedPowers_E[] = { - -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, - -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, - -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, - -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, - -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, - 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, - 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, - 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, - 907, 933, 960, 986, 1013, 1039, 1066 - }; - return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); -} - -inline DiyFp GetCachedPower(int e, int* K) { - - //int k = static_cast<int>(ceil((-61 - e) * 0.30102999566398114)) + 374; - double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive - int k = static_cast<int>(dk); - if (dk - k > 0.0) - k++; - - unsigned index = static_cast<unsigned>((k >> 3) + 1); - *K = -(-348 + static_cast<int>(index << 3)); // decimal exponent no need lookup table - - return GetCachedPowerByIndex(index); -} - -inline DiyFp GetCachedPower10(int exp, int *outExp) { - unsigned index = (static_cast<unsigned>(exp) + 348u) / 8u; - *outExp = -348 + static_cast<int>(index) * 8; - return GetCachedPowerByIndex(index); - } - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -RAPIDJSON_DIAG_OFF(padded) -#endif - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_DIYFP_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h b/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h deleted file mode 100644 index 8d6350e6..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h +++ /dev/null @@ -1,245 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -// This is a C++ header-only implementation of Grisu2 algorithm from the publication: -// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with -// integers." ACM Sigplan Notices 45.6 (2010): 233-243. - -#ifndef RAPIDJSON_DTOA_ -#define RAPIDJSON_DTOA_ - -#include "itoa.h" // GetDigitsLut() -#include "diyfp.h" -#include "ieee754.h" - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 -#endif - -inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { - while (rest < wp_w && delta - rest >= ten_kappa && - (rest + ten_kappa < wp_w || /// closer - wp_w - rest > rest + ten_kappa - wp_w)) { - buffer[len - 1]--; - rest += ten_kappa; - } -} - -inline unsigned CountDecimalDigit32(uint32_t n) { - // Simple pure C++ implementation was faster than __builtin_clz version in this situation. - if (n < 10) return 1; - if (n < 100) return 2; - if (n < 1000) return 3; - if (n < 10000) return 4; - if (n < 100000) return 5; - if (n < 1000000) return 6; - if (n < 10000000) return 7; - if (n < 100000000) return 8; - // Will not reach 10 digits in DigitGen() - //if (n < 1000000000) return 9; - //return 10; - return 9; -} - -inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { - static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; - const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); - const DiyFp wp_w = Mp - W; - uint32_t p1 = static_cast<uint32_t>(Mp.f >> -one.e); - uint64_t p2 = Mp.f & (one.f - 1); - unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] - *len = 0; - - while (kappa > 0) { - uint32_t d = 0; - switch (kappa) { - case 9: d = p1 / 100000000; p1 %= 100000000; break; - case 8: d = p1 / 10000000; p1 %= 10000000; break; - case 7: d = p1 / 1000000; p1 %= 1000000; break; - case 6: d = p1 / 100000; p1 %= 100000; break; - case 5: d = p1 / 10000; p1 %= 10000; break; - case 4: d = p1 / 1000; p1 %= 1000; break; - case 3: d = p1 / 100; p1 %= 100; break; - case 2: d = p1 / 10; p1 %= 10; break; - case 1: d = p1; p1 = 0; break; - default:; - } - if (d || *len) - buffer[(*len)++] = static_cast<char>('0' + static_cast<char>(d)); - kappa--; - uint64_t tmp = (static_cast<uint64_t>(p1) << -one.e) + p2; - if (tmp <= delta) { - *K += kappa; - GrisuRound(buffer, *len, delta, tmp, static_cast<uint64_t>(kPow10[kappa]) << -one.e, wp_w.f); - return; - } - } - - // kappa = 0 - for (;;) { - p2 *= 10; - delta *= 10; - char d = static_cast<char>(p2 >> -one.e); - if (d || *len) - buffer[(*len)++] = static_cast<char>('0' + d); - p2 &= one.f - 1; - kappa--; - if (p2 < delta) { - *K += kappa; - int index = -static_cast<int>(kappa); - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast<int>(kappa)] : 0)); - return; - } - } -} - -inline void Grisu2(double value, char* buffer, int* length, int* K) { - const DiyFp v(value); - DiyFp w_m, w_p; - v.NormalizedBoundaries(&w_m, &w_p); - - const DiyFp c_mk = GetCachedPower(w_p.e, K); - const DiyFp W = v.Normalize() * c_mk; - DiyFp Wp = w_p * c_mk; - DiyFp Wm = w_m * c_mk; - Wm.f++; - Wp.f--; - DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); -} - -inline char* WriteExponent(int K, char* buffer) { - if (K < 0) { - *buffer++ = '-'; - K = -K; - } - - if (K >= 100) { - *buffer++ = static_cast<char>('0' + static_cast<char>(K / 100)); - K %= 100; - const char* d = GetDigitsLut() + K * 2; - *buffer++ = d[0]; - *buffer++ = d[1]; - } - else if (K >= 10) { - const char* d = GetDigitsLut() + K * 2; - *buffer++ = d[0]; - *buffer++ = d[1]; - } - else - *buffer++ = static_cast<char>('0' + static_cast<char>(K)); - - return buffer; -} - -inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { - const int kk = length + k; // 10^(kk-1) <= v < 10^kk - - if (0 <= k && kk <= 21) { - // 1234e7 -> 12340000000 - for (int i = length; i < kk; i++) - buffer[i] = '0'; - buffer[kk] = '.'; - buffer[kk + 1] = '0'; - return &buffer[kk + 2]; - } - else if (0 < kk && kk <= 21) { - // 1234e-2 -> 12.34 - std::memmove(&buffer[kk + 1], &buffer[kk], static_cast<size_t>(length - kk)); - buffer[kk] = '.'; - if (0 > k + maxDecimalPlaces) { - // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 - // Remove extra trailing zeros (at least one) after truncation. - for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) - if (buffer[i] != '0') - return &buffer[i + 1]; - return &buffer[kk + 2]; // Reserve one zero - } - else - return &buffer[length + 1]; - } - else if (-6 < kk && kk <= 0) { - // 1234e-6 -> 0.001234 - const int offset = 2 - kk; - std::memmove(&buffer[offset], &buffer[0], static_cast<size_t>(length)); - buffer[0] = '0'; - buffer[1] = '.'; - for (int i = 2; i < offset; i++) - buffer[i] = '0'; - if (length - kk > maxDecimalPlaces) { - // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 - // Remove extra trailing zeros (at least one) after truncation. - for (int i = maxDecimalPlaces + 1; i > 2; i--) - if (buffer[i] != '0') - return &buffer[i + 1]; - return &buffer[3]; // Reserve one zero - } - else - return &buffer[length + offset]; - } - else if (kk < -maxDecimalPlaces) { - // Truncate to zero - buffer[0] = '0'; - buffer[1] = '.'; - buffer[2] = '0'; - return &buffer[3]; - } - else if (length == 1) { - // 1e30 - buffer[1] = 'e'; - return WriteExponent(kk - 1, &buffer[2]); - } - else { - // 1234e30 -> 1.234e33 - std::memmove(&buffer[2], &buffer[1], static_cast<size_t>(length - 1)); - buffer[1] = '.'; - buffer[length + 1] = 'e'; - return WriteExponent(kk - 1, &buffer[0 + length + 2]); - } -} - -inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { - RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); - Double d(value); - if (d.IsZero()) { - if (d.Sign()) - *buffer++ = '-'; // -0.0, Issue #289 - buffer[0] = '0'; - buffer[1] = '.'; - buffer[2] = '0'; - return &buffer[3]; - } - else { - if (value < 0) { - *buffer++ = '-'; - value = -value; - } - int length, K; - Grisu2(value, buffer, &length, &K); - return Prettify(buffer, length, K, maxDecimalPlaces); - } -} - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_DTOA_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h b/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h deleted file mode 100644 index 82bb0b99..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h +++ /dev/null @@ -1,78 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_IEEE754_ -#define RAPIDJSON_IEEE754_ - -#include "../rapidjson.h" - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -class Double { -public: - Double() {} - Double(double d) : d_(d) {} - Double(uint64_t u) : u_(u) {} - - double Value() const { return d_; } - uint64_t Uint64Value() const { return u_; } - - double NextPositiveDouble() const { - RAPIDJSON_ASSERT(!Sign()); - return Double(u_ + 1).Value(); - } - - bool Sign() const { return (u_ & kSignMask) != 0; } - uint64_t Significand() const { return u_ & kSignificandMask; } - int Exponent() const { return static_cast<int>(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } - - bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } - bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } - bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } - bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } - bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } - - uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } - int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } - uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } - - static unsigned EffectiveSignificandSize(int order) { - if (order >= -1021) - return 53; - else if (order <= -1074) - return 0; - else - return static_cast<unsigned>(order) + 1074; - } - -private: - static const int kSignificandSize = 52; - static const int kExponentBias = 0x3FF; - static const int kDenormalExponent = 1 - kExponentBias; - static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); - static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); - static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); - static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); - - union { - double d_; - uint64_t u_; - }; -}; - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_IEEE754_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/itoa.h b/ext/librethinkdbxx/src/rapidjson/internal/itoa.h deleted file mode 100644 index 01a4e7e7..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/itoa.h +++ /dev/null @@ -1,304 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ITOA_ -#define RAPIDJSON_ITOA_ - -#include "../rapidjson.h" - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -inline const char* GetDigitsLut() { - static const char cDigitsLut[200] = { - '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', - '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', - '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', - '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', - '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', - '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', - '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', - '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', - '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', - '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' - }; - return cDigitsLut; -} - -inline char* u32toa(uint32_t value, char* buffer) { - const char* cDigitsLut = GetDigitsLut(); - - if (value < 10000) { - const uint32_t d1 = (value / 100) << 1; - const uint32_t d2 = (value % 100) << 1; - - if (value >= 1000) - *buffer++ = cDigitsLut[d1]; - if (value >= 100) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 10) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - } - else if (value < 100000000) { - // value = bbbbcccc - const uint32_t b = value / 10000; - const uint32_t c = value % 10000; - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - if (value >= 10000000) - *buffer++ = cDigitsLut[d1]; - if (value >= 1000000) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 100000) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - } - else { - // value = aabbbbcccc in decimal - - const uint32_t a = value / 100000000; // 1 to 42 - value %= 100000000; - - if (a >= 10) { - const unsigned i = a << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else - *buffer++ = static_cast<char>('0' + static_cast<char>(a)); - - const uint32_t b = value / 10000; // 0 to 9999 - const uint32_t c = value % 10000; // 0 to 9999 - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - *buffer++ = cDigitsLut[d1]; - *buffer++ = cDigitsLut[d1 + 1]; - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - } - return buffer; -} - -inline char* i32toa(int32_t value, char* buffer) { - uint32_t u = static_cast<uint32_t>(value); - if (value < 0) { - *buffer++ = '-'; - u = ~u + 1; - } - - return u32toa(u, buffer); -} - -inline char* u64toa(uint64_t value, char* buffer) { - const char* cDigitsLut = GetDigitsLut(); - const uint64_t kTen8 = 100000000; - const uint64_t kTen9 = kTen8 * 10; - const uint64_t kTen10 = kTen8 * 100; - const uint64_t kTen11 = kTen8 * 1000; - const uint64_t kTen12 = kTen8 * 10000; - const uint64_t kTen13 = kTen8 * 100000; - const uint64_t kTen14 = kTen8 * 1000000; - const uint64_t kTen15 = kTen8 * 10000000; - const uint64_t kTen16 = kTen8 * kTen8; - - if (value < kTen8) { - uint32_t v = static_cast<uint32_t>(value); - if (v < 10000) { - const uint32_t d1 = (v / 100) << 1; - const uint32_t d2 = (v % 100) << 1; - - if (v >= 1000) - *buffer++ = cDigitsLut[d1]; - if (v >= 100) - *buffer++ = cDigitsLut[d1 + 1]; - if (v >= 10) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - } - else { - // value = bbbbcccc - const uint32_t b = v / 10000; - const uint32_t c = v % 10000; - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - if (value >= 10000000) - *buffer++ = cDigitsLut[d1]; - if (value >= 1000000) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 100000) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - } - } - else if (value < kTen16) { - const uint32_t v0 = static_cast<uint32_t>(value / kTen8); - const uint32_t v1 = static_cast<uint32_t>(value % kTen8); - - const uint32_t b0 = v0 / 10000; - const uint32_t c0 = v0 % 10000; - - const uint32_t d1 = (b0 / 100) << 1; - const uint32_t d2 = (b0 % 100) << 1; - - const uint32_t d3 = (c0 / 100) << 1; - const uint32_t d4 = (c0 % 100) << 1; - - const uint32_t b1 = v1 / 10000; - const uint32_t c1 = v1 % 10000; - - const uint32_t d5 = (b1 / 100) << 1; - const uint32_t d6 = (b1 % 100) << 1; - - const uint32_t d7 = (c1 / 100) << 1; - const uint32_t d8 = (c1 % 100) << 1; - - if (value >= kTen15) - *buffer++ = cDigitsLut[d1]; - if (value >= kTen14) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= kTen13) - *buffer++ = cDigitsLut[d2]; - if (value >= kTen12) - *buffer++ = cDigitsLut[d2 + 1]; - if (value >= kTen11) - *buffer++ = cDigitsLut[d3]; - if (value >= kTen10) - *buffer++ = cDigitsLut[d3 + 1]; - if (value >= kTen9) - *buffer++ = cDigitsLut[d4]; - if (value >= kTen8) - *buffer++ = cDigitsLut[d4 + 1]; - - *buffer++ = cDigitsLut[d5]; - *buffer++ = cDigitsLut[d5 + 1]; - *buffer++ = cDigitsLut[d6]; - *buffer++ = cDigitsLut[d6 + 1]; - *buffer++ = cDigitsLut[d7]; - *buffer++ = cDigitsLut[d7 + 1]; - *buffer++ = cDigitsLut[d8]; - *buffer++ = cDigitsLut[d8 + 1]; - } - else { - const uint32_t a = static_cast<uint32_t>(value / kTen16); // 1 to 1844 - value %= kTen16; - - if (a < 10) - *buffer++ = static_cast<char>('0' + static_cast<char>(a)); - else if (a < 100) { - const uint32_t i = a << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else if (a < 1000) { - *buffer++ = static_cast<char>('0' + static_cast<char>(a / 100)); - - const uint32_t i = (a % 100) << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else { - const uint32_t i = (a / 100) << 1; - const uint32_t j = (a % 100) << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - *buffer++ = cDigitsLut[j]; - *buffer++ = cDigitsLut[j + 1]; - } - - const uint32_t v0 = static_cast<uint32_t>(value / kTen8); - const uint32_t v1 = static_cast<uint32_t>(value % kTen8); - - const uint32_t b0 = v0 / 10000; - const uint32_t c0 = v0 % 10000; - - const uint32_t d1 = (b0 / 100) << 1; - const uint32_t d2 = (b0 % 100) << 1; - - const uint32_t d3 = (c0 / 100) << 1; - const uint32_t d4 = (c0 % 100) << 1; - - const uint32_t b1 = v1 / 10000; - const uint32_t c1 = v1 % 10000; - - const uint32_t d5 = (b1 / 100) << 1; - const uint32_t d6 = (b1 % 100) << 1; - - const uint32_t d7 = (c1 / 100) << 1; - const uint32_t d8 = (c1 % 100) << 1; - - *buffer++ = cDigitsLut[d1]; - *buffer++ = cDigitsLut[d1 + 1]; - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - *buffer++ = cDigitsLut[d5]; - *buffer++ = cDigitsLut[d5 + 1]; - *buffer++ = cDigitsLut[d6]; - *buffer++ = cDigitsLut[d6 + 1]; - *buffer++ = cDigitsLut[d7]; - *buffer++ = cDigitsLut[d7 + 1]; - *buffer++ = cDigitsLut[d8]; - *buffer++ = cDigitsLut[d8 + 1]; - } - - return buffer; -} - -inline char* i64toa(int64_t value, char* buffer) { - uint64_t u = static_cast<uint64_t>(value); - if (value < 0) { - *buffer++ = '-'; - u = ~u + 1; - } - - return u64toa(u, buffer); -} - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_ITOA_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/meta.h b/ext/librethinkdbxx/src/rapidjson/internal/meta.h deleted file mode 100644 index 5a9aaa42..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/meta.h +++ /dev/null @@ -1,181 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_INTERNAL_META_H_ -#define RAPIDJSON_INTERNAL_META_H_ - -#include "../rapidjson.h" - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif -#if defined(_MSC_VER) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(6334) -#endif - -#if RAPIDJSON_HAS_CXX11_TYPETRAITS -#include <type_traits> -#endif - -//@cond RAPIDJSON_INTERNAL -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -// Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching -template <typename T> struct Void { typedef void Type; }; - -/////////////////////////////////////////////////////////////////////////////// -// BoolType, TrueType, FalseType -// -template <bool Cond> struct BoolType { - static const bool Value = Cond; - typedef BoolType Type; -}; -typedef BoolType<true> TrueType; -typedef BoolType<false> FalseType; - - -/////////////////////////////////////////////////////////////////////////////// -// SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr -// - -template <bool C> struct SelectIfImpl { template <typename T1, typename T2> struct Apply { typedef T1 Type; }; }; -template <> struct SelectIfImpl<false> { template <typename T1, typename T2> struct Apply { typedef T2 Type; }; }; -template <bool C, typename T1, typename T2> struct SelectIfCond : SelectIfImpl<C>::template Apply<T1,T2> {}; -template <typename C, typename T1, typename T2> struct SelectIf : SelectIfCond<C::Value, T1, T2> {}; - -template <bool Cond1, bool Cond2> struct AndExprCond : FalseType {}; -template <> struct AndExprCond<true, true> : TrueType {}; -template <bool Cond1, bool Cond2> struct OrExprCond : TrueType {}; -template <> struct OrExprCond<false, false> : FalseType {}; - -template <typename C> struct BoolExpr : SelectIf<C,TrueType,FalseType>::Type {}; -template <typename C> struct NotExpr : SelectIf<C,FalseType,TrueType>::Type {}; -template <typename C1, typename C2> struct AndExpr : AndExprCond<C1::Value, C2::Value>::Type {}; -template <typename C1, typename C2> struct OrExpr : OrExprCond<C1::Value, C2::Value>::Type {}; - - -/////////////////////////////////////////////////////////////////////////////// -// AddConst, MaybeAddConst, RemoveConst -template <typename T> struct AddConst { typedef const T Type; }; -template <bool Constify, typename T> struct MaybeAddConst : SelectIfCond<Constify, const T, T> {}; -template <typename T> struct RemoveConst { typedef T Type; }; -template <typename T> struct RemoveConst<const T> { typedef T Type; }; - - -/////////////////////////////////////////////////////////////////////////////// -// IsSame, IsConst, IsMoreConst, IsPointer -// -template <typename T, typename U> struct IsSame : FalseType {}; -template <typename T> struct IsSame<T, T> : TrueType {}; - -template <typename T> struct IsConst : FalseType {}; -template <typename T> struct IsConst<const T> : TrueType {}; - -template <typename CT, typename T> -struct IsMoreConst - : AndExpr<IsSame<typename RemoveConst<CT>::Type, typename RemoveConst<T>::Type>, - BoolType<IsConst<CT>::Value >= IsConst<T>::Value> >::Type {}; - -template <typename T> struct IsPointer : FalseType {}; -template <typename T> struct IsPointer<T*> : TrueType {}; - -/////////////////////////////////////////////////////////////////////////////// -// IsBaseOf -// -#if RAPIDJSON_HAS_CXX11_TYPETRAITS - -template <typename B, typename D> struct IsBaseOf - : BoolType< ::std::is_base_of<B,D>::value> {}; - -#else // simplified version adopted from Boost - -template<typename B, typename D> struct IsBaseOfImpl { - RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); - RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); - - typedef char (&Yes)[1]; - typedef char (&No) [2]; - - template <typename T> - static Yes Check(const D*, T); - static No Check(const B*, int); - - struct Host { - operator const B*() const; - operator const D*(); - }; - - enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; -}; - -template <typename B, typename D> struct IsBaseOf - : OrExpr<IsSame<B, D>, BoolExpr<IsBaseOfImpl<B, D> > >::Type {}; - -#endif // RAPIDJSON_HAS_CXX11_TYPETRAITS - - -////////////////////////////////////////////////////////////////////////// -// EnableIf / DisableIf -// -template <bool Condition, typename T = void> struct EnableIfCond { typedef T Type; }; -template <typename T> struct EnableIfCond<false, T> { /* empty */ }; - -template <bool Condition, typename T = void> struct DisableIfCond { typedef T Type; }; -template <typename T> struct DisableIfCond<true, T> { /* empty */ }; - -template <typename Condition, typename T = void> -struct EnableIf : EnableIfCond<Condition::Value, T> {}; - -template <typename Condition, typename T = void> -struct DisableIf : DisableIfCond<Condition::Value, T> {}; - -// SFINAE helpers -struct SfinaeTag {}; -template <typename T> struct RemoveSfinaeTag; -template <typename T> struct RemoveSfinaeTag<SfinaeTag&(*)(T)> { typedef T Type; }; - -#define RAPIDJSON_REMOVEFPTR_(type) \ - typename ::RAPIDJSON_NAMESPACE::internal::RemoveSfinaeTag \ - < ::RAPIDJSON_NAMESPACE::internal::SfinaeTag&(*) type>::Type - -#define RAPIDJSON_ENABLEIF(cond) \ - typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ - <RAPIDJSON_REMOVEFPTR_(cond)>::Type * = NULL - -#define RAPIDJSON_DISABLEIF(cond) \ - typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ - <RAPIDJSON_REMOVEFPTR_(cond)>::Type * = NULL - -#define RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ - typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ - <RAPIDJSON_REMOVEFPTR_(cond), \ - RAPIDJSON_REMOVEFPTR_(returntype)>::Type - -#define RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ - typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ - <RAPIDJSON_REMOVEFPTR_(cond), \ - RAPIDJSON_REMOVEFPTR_(returntype)>::Type - -} // namespace internal -RAPIDJSON_NAMESPACE_END -//@endcond - -#if defined(__GNUC__) || defined(_MSC_VER) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_INTERNAL_META_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/pow10.h b/ext/librethinkdbxx/src/rapidjson/internal/pow10.h deleted file mode 100644 index 02f475d7..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/pow10.h +++ /dev/null @@ -1,55 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_POW10_ -#define RAPIDJSON_POW10_ - -#include "../rapidjson.h" - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -//! Computes integer powers of 10 in double (10.0^n). -/*! This function uses lookup table for fast and accurate results. - \param n non-negative exponent. Must <= 308. - \return 10.0^n -*/ -inline double Pow10(int n) { - static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes - 1e+0, - 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, - 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, - 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, - 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, - 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, - 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, - 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, - 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, - 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, - 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, - 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, - 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, - 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, - 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, - 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, - 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 - }; - RAPIDJSON_ASSERT(n >= 0 && n <= 308); - return e[n]; -} - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_POW10_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/regex.h b/ext/librethinkdbxx/src/rapidjson/internal/regex.h deleted file mode 100644 index 422a5240..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/regex.h +++ /dev/null @@ -1,701 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_INTERNAL_REGEX_H_ -#define RAPIDJSON_INTERNAL_REGEX_H_ - -#include "../allocators.h" -#include "../stream.h" -#include "stack.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(implicit-fallthrough) -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -#ifndef RAPIDJSON_REGEX_VERBOSE -#define RAPIDJSON_REGEX_VERBOSE 0 -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -/////////////////////////////////////////////////////////////////////////////// -// GenericRegex - -static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 -static const SizeType kRegexInvalidRange = ~SizeType(0); - -//! Regular expression engine with subset of ECMAscript grammar. -/*! - Supported regular expression syntax: - - \c ab Concatenation - - \c a|b Alternation - - \c a? Zero or one - - \c a* Zero or more - - \c a+ One or more - - \c a{3} Exactly 3 times - - \c a{3,} At least 3 times - - \c a{3,5} 3 to 5 times - - \c (ab) Grouping - - \c ^a At the beginning - - \c a$ At the end - - \c . Any character - - \c [abc] Character classes - - \c [a-c] Character class range - - \c [a-z0-9_] Character class combination - - \c [^abc] Negated character classes - - \c [^a-c] Negated character class range - - \c [\b] Backspace (U+0008) - - \c \\| \\\\ ... Escape characters - - \c \\f Form feed (U+000C) - - \c \\n Line feed (U+000A) - - \c \\r Carriage return (U+000D) - - \c \\t Tab (U+0009) - - \c \\v Vertical tab (U+000B) - - \note This is a Thompson NFA engine, implemented with reference to - Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", - https://swtch.com/~rsc/regexp/regexp1.html -*/ -template <typename Encoding, typename Allocator = CrtAllocator> -class GenericRegex { -public: - typedef typename Encoding::Ch Ch; - - GenericRegex(const Ch* source, Allocator* allocator = 0) : - states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), - stateSet_(), state0_(allocator, 0), state1_(allocator, 0), anchorBegin_(), anchorEnd_() - { - GenericStringStream<Encoding> ss(source); - DecodedStream<GenericStringStream<Encoding> > ds(ss); - Parse(ds); - } - - ~GenericRegex() { - Allocator::Free(stateSet_); - } - - bool IsValid() const { - return root_ != kRegexInvalidState; - } - - template <typename InputStream> - bool Match(InputStream& is) const { - return SearchWithAnchoring(is, true, true); - } - - bool Match(const Ch* s) const { - GenericStringStream<Encoding> is(s); - return Match(is); - } - - template <typename InputStream> - bool Search(InputStream& is) const { - return SearchWithAnchoring(is, anchorBegin_, anchorEnd_); - } - - bool Search(const Ch* s) const { - GenericStringStream<Encoding> is(s); - return Search(is); - } - -private: - enum Operator { - kZeroOrOne, - kZeroOrMore, - kOneOrMore, - kConcatenation, - kAlternation, - kLeftParenthesis - }; - - static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' - static const unsigned kRangeCharacterClass = 0xFFFFFFFE; - static const unsigned kRangeNegationFlag = 0x80000000; - - struct Range { - unsigned start; // - unsigned end; - SizeType next; - }; - - struct State { - SizeType out; //!< Equals to kInvalid for matching state - SizeType out1; //!< Equals to non-kInvalid for split - SizeType rangeStart; - unsigned codepoint; - }; - - struct Frag { - Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} - SizeType start; - SizeType out; //!< link-list of all output states - SizeType minIndex; - }; - - template <typename SourceStream> - class DecodedStream { - public: - DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } - unsigned Peek() { return codepoint_; } - unsigned Take() { - unsigned c = codepoint_; - if (c) // No further decoding when '\0' - Decode(); - return c; - } - - private: - void Decode() { - if (!Encoding::Decode(ss_, &codepoint_)) - codepoint_ = 0; - } - - SourceStream& ss_; - unsigned codepoint_; - }; - - State& GetState(SizeType index) { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom<State>()[index]; - } - - const State& GetState(SizeType index) const { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom<State>()[index]; - } - - Range& GetRange(SizeType index) { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom<Range>()[index]; - } - - const Range& GetRange(SizeType index) const { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom<Range>()[index]; - } - - template <typename InputStream> - void Parse(DecodedStream<InputStream>& ds) { - Allocator allocator; - Stack<Allocator> operandStack(&allocator, 256); // Frag - Stack<Allocator> operatorStack(&allocator, 256); // Operator - Stack<Allocator> atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) - - *atomCountStack.template Push<unsigned>() = 0; - - unsigned codepoint; - while (ds.Peek() != 0) { - switch (codepoint = ds.Take()) { - case '^': - anchorBegin_ = true; - break; - - case '$': - anchorEnd_ = true; - break; - - case '|': - while (!operatorStack.Empty() && *operatorStack.template Top<Operator>() < kAlternation) - if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1))) - return; - *operatorStack.template Push<Operator>() = kAlternation; - *atomCountStack.template Top<unsigned>() = 0; - break; - - case '(': - *operatorStack.template Push<Operator>() = kLeftParenthesis; - *atomCountStack.template Push<unsigned>() = 0; - break; - - case ')': - while (!operatorStack.Empty() && *operatorStack.template Top<Operator>() != kLeftParenthesis) - if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1))) - return; - if (operatorStack.Empty()) - return; - operatorStack.template Pop<Operator>(1); - atomCountStack.template Pop<unsigned>(1); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '?': - if (!Eval(operandStack, kZeroOrOne)) - return; - break; - - case '*': - if (!Eval(operandStack, kZeroOrMore)) - return; - break; - - case '+': - if (!Eval(operandStack, kOneOrMore)) - return; - break; - - case '{': - { - unsigned n, m; - if (!ParseUnsigned(ds, &n)) - return; - - if (ds.Peek() == ',') { - ds.Take(); - if (ds.Peek() == '}') - m = kInfinityQuantifier; - else if (!ParseUnsigned(ds, &m) || m < n) - return; - } - else - m = n; - - if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') - return; - ds.Take(); - } - break; - - case '.': - PushOperand(operandStack, kAnyCharacterClass); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '[': - { - SizeType range; - if (!ParseRange(ds, &range)) - return; - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); - GetState(s).rangeStart = range; - *operandStack.template Push<Frag>() = Frag(s, s, s); - } - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '\\': // Escape character - if (!CharacterEscape(ds, &codepoint)) - return; // Unsupported escape character - // fall through to default - - default: // Pattern character - PushOperand(operandStack, codepoint); - ImplicitConcatenation(atomCountStack, operatorStack); - } - } - - while (!operatorStack.Empty()) - if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1))) - return; - - // Link the operand to matching state. - if (operandStack.GetSize() == sizeof(Frag)) { - Frag* e = operandStack.template Pop<Frag>(1); - Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); - root_ = e->start; - -#if RAPIDJSON_REGEX_VERBOSE - printf("root: %d\n", root_); - for (SizeType i = 0; i < stateCount_ ; i++) { - State& s = GetState(i); - printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); - } - printf("\n"); -#endif - } - - // Preallocate buffer for SearchWithAnchoring() - RAPIDJSON_ASSERT(stateSet_ == 0); - if (stateCount_ > 0) { - stateSet_ = static_cast<unsigned*>(states_.GetAllocator().Malloc(GetStateSetSize())); - state0_.template Reserve<SizeType>(stateCount_); - state1_.template Reserve<SizeType>(stateCount_); - } - } - - SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { - State* s = states_.template Push<State>(); - s->out = out; - s->out1 = out1; - s->codepoint = codepoint; - s->rangeStart = kRegexInvalidRange; - return stateCount_++; - } - - void PushOperand(Stack<Allocator>& operandStack, unsigned codepoint) { - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); - *operandStack.template Push<Frag>() = Frag(s, s, s); - } - - void ImplicitConcatenation(Stack<Allocator>& atomCountStack, Stack<Allocator>& operatorStack) { - if (*atomCountStack.template Top<unsigned>()) - *operatorStack.template Push<Operator>() = kConcatenation; - (*atomCountStack.template Top<unsigned>())++; - } - - SizeType Append(SizeType l1, SizeType l2) { - SizeType old = l1; - while (GetState(l1).out != kRegexInvalidState) - l1 = GetState(l1).out; - GetState(l1).out = l2; - return old; - } - - void Patch(SizeType l, SizeType s) { - for (SizeType next; l != kRegexInvalidState; l = next) { - next = GetState(l).out; - GetState(l).out = s; - } - } - - bool Eval(Stack<Allocator>& operandStack, Operator op) { - switch (op) { - case kConcatenation: - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); - { - Frag e2 = *operandStack.template Pop<Frag>(1); - Frag e1 = *operandStack.template Pop<Frag>(1); - Patch(e1.out, e2.start); - *operandStack.template Push<Frag>() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); - } - return true; - - case kAlternation: - if (operandStack.GetSize() >= sizeof(Frag) * 2) { - Frag e2 = *operandStack.template Pop<Frag>(1); - Frag e1 = *operandStack.template Pop<Frag>(1); - SizeType s = NewState(e1.start, e2.start, 0); - *operandStack.template Push<Frag>() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); - return true; - } - return false; - - case kZeroOrOne: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop<Frag>(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - *operandStack.template Push<Frag>() = Frag(s, Append(e.out, s), e.minIndex); - return true; - } - return false; - - case kZeroOrMore: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop<Frag>(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push<Frag>() = Frag(s, s, e.minIndex); - return true; - } - return false; - - default: - RAPIDJSON_ASSERT(op == kOneOrMore); - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop<Frag>(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push<Frag>() = Frag(e.start, s, e.minIndex); - return true; - } - return false; - } - } - - bool EvalQuantifier(Stack<Allocator>& operandStack, unsigned n, unsigned m) { - RAPIDJSON_ASSERT(n <= m); - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); - - if (n == 0) { - if (m == 0) // a{0} not support - return false; - else if (m == kInfinityQuantifier) - Eval(operandStack, kZeroOrMore); // a{0,} -> a* - else { - Eval(operandStack, kZeroOrOne); // a{0,5} -> a? - for (unsigned i = 0; i < m - 1; i++) - CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? - for (unsigned i = 0; i < m - 1; i++) - Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? - } - return true; - } - - for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a - CloneTopOperand(operandStack); - - if (m == kInfinityQuantifier) - Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ - else if (m > n) { - CloneTopOperand(operandStack); // a{3,5} -> a a a a - Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? - for (unsigned i = n; i < m - 1; i++) - CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? - for (unsigned i = n; i < m; i++) - Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? - } - - for (unsigned i = 0; i < n - 1; i++) - Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? - - return true; - } - - static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } - - void CloneTopOperand(Stack<Allocator>& operandStack) { - const Frag src = *operandStack.template Top<Frag>(); // Copy constructor to prevent invalidation - SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) - State* s = states_.template Push<State>(count); - memcpy(s, &GetState(src.minIndex), count * sizeof(State)); - for (SizeType j = 0; j < count; j++) { - if (s[j].out != kRegexInvalidState) - s[j].out += count; - if (s[j].out1 != kRegexInvalidState) - s[j].out1 += count; - } - *operandStack.template Push<Frag>() = Frag(src.start + count, src.out + count, src.minIndex + count); - stateCount_ += count; - } - - template <typename InputStream> - bool ParseUnsigned(DecodedStream<InputStream>& ds, unsigned* u) { - unsigned r = 0; - if (ds.Peek() < '0' || ds.Peek() > '9') - return false; - while (ds.Peek() >= '0' && ds.Peek() <= '9') { - if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 - return false; // overflow - r = r * 10 + (ds.Take() - '0'); - } - *u = r; - return true; - } - - template <typename InputStream> - bool ParseRange(DecodedStream<InputStream>& ds, SizeType* range) { - bool isBegin = true; - bool negate = false; - int step = 0; - SizeType start = kRegexInvalidRange; - SizeType current = kRegexInvalidRange; - unsigned codepoint; - while ((codepoint = ds.Take()) != 0) { - if (isBegin) { - isBegin = false; - if (codepoint == '^') { - negate = true; - continue; - } - } - - switch (codepoint) { - case ']': - if (start == kRegexInvalidRange) - return false; // Error: nothing inside [] - if (step == 2) { // Add trailing '-' - SizeType r = NewRange('-'); - RAPIDJSON_ASSERT(current != kRegexInvalidRange); - GetRange(current).next = r; - } - if (negate) - GetRange(start).start |= kRangeNegationFlag; - *range = start; - return true; - - case '\\': - if (ds.Peek() == 'b') { - ds.Take(); - codepoint = 0x0008; // Escape backspace character - } - else if (!CharacterEscape(ds, &codepoint)) - return false; - // fall through to default - - default: - switch (step) { - case 1: - if (codepoint == '-') { - step++; - break; - } - // fall through to step 0 for other characters - - case 0: - { - SizeType r = NewRange(codepoint); - if (current != kRegexInvalidRange) - GetRange(current).next = r; - if (start == kRegexInvalidRange) - start = r; - current = r; - } - step = 1; - break; - - default: - RAPIDJSON_ASSERT(step == 2); - GetRange(current).end = codepoint; - step = 0; - } - } - } - return false; - } - - SizeType NewRange(unsigned codepoint) { - Range* r = ranges_.template Push<Range>(); - r->start = r->end = codepoint; - r->next = kRegexInvalidRange; - return rangeCount_++; - } - - template <typename InputStream> - bool CharacterEscape(DecodedStream<InputStream>& ds, unsigned* escapedCodepoint) { - unsigned codepoint; - switch (codepoint = ds.Take()) { - case '^': - case '$': - case '|': - case '(': - case ')': - case '?': - case '*': - case '+': - case '.': - case '[': - case ']': - case '{': - case '}': - case '\\': - *escapedCodepoint = codepoint; return true; - case 'f': *escapedCodepoint = 0x000C; return true; - case 'n': *escapedCodepoint = 0x000A; return true; - case 'r': *escapedCodepoint = 0x000D; return true; - case 't': *escapedCodepoint = 0x0009; return true; - case 'v': *escapedCodepoint = 0x000B; return true; - default: - return false; // Unsupported escape character - } - } - - template <typename InputStream> - bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) const { - RAPIDJSON_ASSERT(IsValid()); - DecodedStream<InputStream> ds(is); - - state0_.Clear(); - Stack<Allocator> *current = &state0_, *next = &state1_; - const size_t stateSetSize = GetStateSetSize(); - std::memset(stateSet_, 0, stateSetSize); - - bool matched = AddState(*current, root_); - unsigned codepoint; - while (!current->Empty() && (codepoint = ds.Take()) != 0) { - std::memset(stateSet_, 0, stateSetSize); - next->Clear(); - matched = false; - for (const SizeType* s = current->template Bottom<SizeType>(); s != current->template End<SizeType>(); ++s) { - const State& sr = GetState(*s); - if (sr.codepoint == codepoint || - sr.codepoint == kAnyCharacterClass || - (sr.codepoint == kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) - { - matched = AddState(*next, sr.out) || matched; - if (!anchorEnd && matched) - return true; - } - if (!anchorBegin) - AddState(*next, root_); - } - internal::Swap(current, next); - } - - return matched; - } - - size_t GetStateSetSize() const { - return (stateCount_ + 31) / 32 * 4; - } - - // Return whether the added states is a match state - bool AddState(Stack<Allocator>& l, SizeType index) const { - RAPIDJSON_ASSERT(index != kRegexInvalidState); - - const State& s = GetState(index); - if (s.out1 != kRegexInvalidState) { // Split - bool matched = AddState(l, s.out); - return AddState(l, s.out1) || matched; - } - else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) { - stateSet_[index >> 5] |= (1 << (index & 31)); - *l.template PushUnsafe<SizeType>() = index; - } - return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. - } - - bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { - bool yes = (GetRange(rangeIndex).start & kRangeNegationFlag) == 0; - while (rangeIndex != kRegexInvalidRange) { - const Range& r = GetRange(rangeIndex); - if (codepoint >= (r.start & ~kRangeNegationFlag) && codepoint <= r.end) - return yes; - rangeIndex = r.next; - } - return !yes; - } - - Stack<Allocator> states_; - Stack<Allocator> ranges_; - SizeType root_; - SizeType stateCount_; - SizeType rangeCount_; - - static const unsigned kInfinityQuantifier = ~0u; - - // For SearchWithAnchoring() - uint32_t* stateSet_; // allocated by states_.GetAllocator() - mutable Stack<Allocator> state0_; - mutable Stack<Allocator> state1_; - bool anchorBegin_; - bool anchorEnd_; -}; - -typedef GenericRegex<UTF8<> > Regex; - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/stack.h b/ext/librethinkdbxx/src/rapidjson/internal/stack.h deleted file mode 100644 index 022c9aab..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/stack.h +++ /dev/null @@ -1,230 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_INTERNAL_STACK_H_ -#define RAPIDJSON_INTERNAL_STACK_H_ - -#include "../allocators.h" -#include "swap.h" - -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -/////////////////////////////////////////////////////////////////////////////// -// Stack - -//! A type-unsafe stack for storing different types of data. -/*! \tparam Allocator Allocator for allocating stack memory. -*/ -template <typename Allocator> -class Stack { -public: - // Optimization note: Do not allocate memory for stack_ in constructor. - // Do it lazily when first Push() -> Expand() -> Resize(). - Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - Stack(Stack&& rhs) - : allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - stack_(rhs.stack_), - stackTop_(rhs.stackTop_), - stackEnd_(rhs.stackEnd_), - initialCapacity_(rhs.initialCapacity_) - { - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.stack_ = 0; - rhs.stackTop_ = 0; - rhs.stackEnd_ = 0; - rhs.initialCapacity_ = 0; - } -#endif - - ~Stack() { - Destroy(); - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - Stack& operator=(Stack&& rhs) { - if (&rhs != this) - { - Destroy(); - - allocator_ = rhs.allocator_; - ownAllocator_ = rhs.ownAllocator_; - stack_ = rhs.stack_; - stackTop_ = rhs.stackTop_; - stackEnd_ = rhs.stackEnd_; - initialCapacity_ = rhs.initialCapacity_; - - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.stack_ = 0; - rhs.stackTop_ = 0; - rhs.stackEnd_ = 0; - rhs.initialCapacity_ = 0; - } - return *this; - } -#endif - - void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { - internal::Swap(allocator_, rhs.allocator_); - internal::Swap(ownAllocator_, rhs.ownAllocator_); - internal::Swap(stack_, rhs.stack_); - internal::Swap(stackTop_, rhs.stackTop_); - internal::Swap(stackEnd_, rhs.stackEnd_); - internal::Swap(initialCapacity_, rhs.initialCapacity_); - } - - void Clear() { stackTop_ = stack_; } - - void ShrinkToFit() { - if (Empty()) { - // If the stack is empty, completely deallocate the memory. - Allocator::Free(stack_); - stack_ = 0; - stackTop_ = 0; - stackEnd_ = 0; - } - else - Resize(GetSize()); - } - - // Optimization note: try to minimize the size of this function for force inline. - // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. - template<typename T> - RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { - // Expand the stack if needed - if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) - Expand<T>(count); - } - - template<typename T> - RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { - Reserve<T>(count); - return PushUnsafe<T>(count); - } - - template<typename T> - RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { - RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); - T* ret = reinterpret_cast<T*>(stackTop_); - stackTop_ += sizeof(T) * count; - return ret; - } - - template<typename T> - T* Pop(size_t count) { - RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); - stackTop_ -= count * sizeof(T); - return reinterpret_cast<T*>(stackTop_); - } - - template<typename T> - T* Top() { - RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); - return reinterpret_cast<T*>(stackTop_ - sizeof(T)); - } - - template<typename T> - const T* Top() const { - RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); - return reinterpret_cast<T*>(stackTop_ - sizeof(T)); - } - - template<typename T> - T* End() { return reinterpret_cast<T*>(stackTop_); } - - template<typename T> - const T* End() const { return reinterpret_cast<T*>(stackTop_); } - - template<typename T> - T* Bottom() { return reinterpret_cast<T*>(stack_); } - - template<typename T> - const T* Bottom() const { return reinterpret_cast<T*>(stack_); } - - bool HasAllocator() const { - return allocator_ != 0; - } - - Allocator& GetAllocator() { - RAPIDJSON_ASSERT(allocator_); - return *allocator_; - } - - bool Empty() const { return stackTop_ == stack_; } - size_t GetSize() const { return static_cast<size_t>(stackTop_ - stack_); } - size_t GetCapacity() const { return static_cast<size_t>(stackEnd_ - stack_); } - -private: - template<typename T> - void Expand(size_t count) { - // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. - size_t newCapacity; - if (stack_ == 0) { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - newCapacity = initialCapacity_; - } else { - newCapacity = GetCapacity(); - newCapacity += (newCapacity + 1) / 2; - } - size_t newSize = GetSize() + sizeof(T) * count; - if (newCapacity < newSize) - newCapacity = newSize; - - Resize(newCapacity); - } - - void Resize(size_t newCapacity) { - const size_t size = GetSize(); // Backup the current size - stack_ = static_cast<char*>(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); - stackTop_ = stack_ + size; - stackEnd_ = stack_ + newCapacity; - } - - void Destroy() { - Allocator::Free(stack_); - RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack - } - - // Prohibit copy constructor & assignment operator. - Stack(const Stack&); - Stack& operator=(const Stack&); - - Allocator* allocator_; - Allocator* ownAllocator_; - char *stack_; - char *stackTop_; - char *stackEnd_; - size_t initialCapacity_; -}; - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_STACK_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h b/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h deleted file mode 100644 index 2edfae52..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h +++ /dev/null @@ -1,55 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ -#define RAPIDJSON_INTERNAL_STRFUNC_H_ - -#include "../stream.h" - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -//! Custom strlen() which works on different character types. -/*! \tparam Ch Character type (e.g. char, wchar_t, short) - \param s Null-terminated input string. - \return Number of characters in the string. - \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. -*/ -template <typename Ch> -inline SizeType StrLen(const Ch* s) { - const Ch* p = s; - while (*p) ++p; - return SizeType(p - s); -} - -//! Returns number of code points in a encoded string. -template<typename Encoding> -bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { - GenericStringStream<Encoding> is(s); - const typename Encoding::Ch* end = s + length; - SizeType count = 0; - while (is.src_ < end) { - unsigned codepoint; - if (!Encoding::Decode(is, &codepoint)) - return false; - count++; - } - *outCount = count; - return true; -} - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_INTERNAL_STRFUNC_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/strtod.h b/ext/librethinkdbxx/src/rapidjson/internal/strtod.h deleted file mode 100644 index 289c413b..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/strtod.h +++ /dev/null @@ -1,269 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_STRTOD_ -#define RAPIDJSON_STRTOD_ - -#include "ieee754.h" -#include "biginteger.h" -#include "diyfp.h" -#include "pow10.h" - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -inline double FastPath(double significand, int exp) { - if (exp < -308) - return 0.0; - else if (exp >= 0) - return significand * internal::Pow10(exp); - else - return significand / internal::Pow10(-exp); -} - -inline double StrtodNormalPrecision(double d, int p) { - if (p < -308) { - // Prevent expSum < -308, making Pow10(p) = 0 - d = FastPath(d, -308); - d = FastPath(d, p + 308); - } - else - d = FastPath(d, p); - return d; -} - -template <typename T> -inline T Min3(T a, T b, T c) { - T m = a; - if (m > b) m = b; - if (m > c) m = c; - return m; -} - -inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { - const Double db(b); - const uint64_t bInt = db.IntegerSignificand(); - const int bExp = db.IntegerExponent(); - const int hExp = bExp - 1; - - int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; - - // Adjust for decimal exponent - if (dExp >= 0) { - dS_Exp2 += dExp; - dS_Exp5 += dExp; - } - else { - bS_Exp2 -= dExp; - bS_Exp5 -= dExp; - hS_Exp2 -= dExp; - hS_Exp5 -= dExp; - } - - // Adjust for binary exponent - if (bExp >= 0) - bS_Exp2 += bExp; - else { - dS_Exp2 -= bExp; - hS_Exp2 -= bExp; - } - - // Adjust for half ulp exponent - if (hExp >= 0) - hS_Exp2 += hExp; - else { - dS_Exp2 -= hExp; - bS_Exp2 -= hExp; - } - - // Remove common power of two factor from all three scaled values - int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); - dS_Exp2 -= common_Exp2; - bS_Exp2 -= common_Exp2; - hS_Exp2 -= common_Exp2; - - BigInteger dS = d; - dS.MultiplyPow5(static_cast<unsigned>(dS_Exp5)) <<= static_cast<unsigned>(dS_Exp2); - - BigInteger bS(bInt); - bS.MultiplyPow5(static_cast<unsigned>(bS_Exp5)) <<= static_cast<unsigned>(bS_Exp2); - - BigInteger hS(1); - hS.MultiplyPow5(static_cast<unsigned>(hS_Exp5)) <<= static_cast<unsigned>(hS_Exp2); - - BigInteger delta(0); - dS.Difference(bS, &delta); - - return delta.Compare(hS); -} - -inline bool StrtodFast(double d, int p, double* result) { - // Use fast path for string-to-double conversion if possible - // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - if (p > 22 && p < 22 + 16) { - // Fast Path Cases In Disguise - d *= internal::Pow10(p - 22); - p = 22; - } - - if (p >= -22 && p <= 22 && d <= 9007199254740991.0) { // 2^53 - 1 - *result = FastPath(d, p); - return true; - } - else - return false; -} - -// Compute an approximation and see if it is within 1/2 ULP -inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) { - uint64_t significand = 0; - size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 - for (; i < length; i++) { - if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || - (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) - break; - significand = significand * 10u + static_cast<unsigned>(decimals[i] - '0'); - } - - if (i < length && decimals[i] >= '5') // Rounding - significand++; - - size_t remaining = length - i; - const unsigned kUlpShift = 3; - const unsigned kUlp = 1 << kUlpShift; - int64_t error = (remaining == 0) ? 0 : kUlp / 2; - - DiyFp v(significand, 0); - v = v.Normalize(); - error <<= -v.e; - - const int dExp = static_cast<int>(decimalPosition) - static_cast<int>(i) + exp; - - int actualExp; - DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); - if (actualExp != dExp) { - static const DiyFp kPow10[] = { - DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 - DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 - DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 - DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 - DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 - DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 - DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 - }; - int adjustment = dExp - actualExp - 1; - RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); - v = v * kPow10[adjustment]; - if (length + static_cast<unsigned>(adjustment)> 19u) // has more digits than decimal digits in 64-bit - error += kUlp / 2; - } - - v = v * cachedPower; - - error += kUlp + (error == 0 ? 0 : 1); - - const int oldExp = v.e; - v = v.Normalize(); - error <<= oldExp - v.e; - - const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); - unsigned precisionSize = 64 - effectiveSignificandSize; - if (precisionSize + kUlpShift >= 64) { - unsigned scaleExp = (precisionSize + kUlpShift) - 63; - v.f >>= scaleExp; - v.e += scaleExp; - error = (error >> scaleExp) + 1 + static_cast<int>(kUlp); - precisionSize -= scaleExp; - } - - DiyFp rounded(v.f >> precisionSize, v.e + static_cast<int>(precisionSize)); - const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; - const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; - if (precisionBits >= halfWay + static_cast<unsigned>(error)) { - rounded.f++; - if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) - rounded.f >>= 1; - rounded.e++; - } - } - - *result = rounded.ToDouble(); - - return halfWay - static_cast<unsigned>(error) >= precisionBits || precisionBits >= halfWay + static_cast<unsigned>(error); -} - -inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { - const BigInteger dInt(decimals, length); - const int dExp = static_cast<int>(decimalPosition) - static_cast<int>(length) + exp; - Double a(approx); - int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); - if (cmp < 0) - return a.Value(); // within half ULP - else if (cmp == 0) { - // Round towards even - if (a.Significand() & 1) - return a.NextPositiveDouble(); - else - return a.Value(); - } - else // adjustment - return a.NextPositiveDouble(); -} - -inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { - RAPIDJSON_ASSERT(d >= 0.0); - RAPIDJSON_ASSERT(length >= 1); - - double result; - if (StrtodFast(d, p, &result)) - return result; - - // Trim leading zeros - while (*decimals == '0' && length > 1) { - length--; - decimals++; - decimalPosition--; - } - - // Trim trailing zeros - while (decimals[length - 1] == '0' && length > 1) { - length--; - decimalPosition--; - exp++; - } - - // Trim right-most digits - const int kMaxDecimalDigit = 780; - if (static_cast<int>(length) > kMaxDecimalDigit) { - int delta = (static_cast<int>(length) - kMaxDecimalDigit); - exp += delta; - decimalPosition -= static_cast<unsigned>(delta); - length = kMaxDecimalDigit; - } - - // If too small, underflow to zero - if (int(length) + exp < -324) - return 0.0; - - if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) - return result; - - // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison - return StrtodBigInteger(result, decimals, length, decimalPosition, exp); -} - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_STRTOD_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/swap.h b/ext/librethinkdbxx/src/rapidjson/internal/swap.h deleted file mode 100644 index 666e49f9..00000000 --- a/ext/librethinkdbxx/src/rapidjson/internal/swap.h +++ /dev/null @@ -1,46 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_INTERNAL_SWAP_H_ -#define RAPIDJSON_INTERNAL_SWAP_H_ - -#include "../rapidjson.h" - -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -//! Custom swap() to avoid dependency on C++ <algorithm> header -/*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. - \note This has the same semantics as std::swap(). -*/ -template <typename T> -inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { - T tmp = a; - a = b; - b = tmp; -} - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_INTERNAL_SWAP_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h b/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h deleted file mode 100644 index f5fe2897..00000000 --- a/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h +++ /dev/null @@ -1,115 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_ISTREAMWRAPPER_H_ -#define RAPIDJSON_ISTREAMWRAPPER_H_ - -#include "stream.h" -#include <iosfwd> - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. -/*! - The classes can be wrapped including but not limited to: - - - \c std::istringstream - - \c std::stringstream - - \c std::wistringstream - - \c std::wstringstream - - \c std::ifstream - - \c std::fstream - - \c std::wifstream - - \c std::wfstream - - \tparam StreamType Class derived from \c std::basic_istream. -*/ - -template <typename StreamType> -class BasicIStreamWrapper { -public: - typedef typename StreamType::char_type Ch; - BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} - - Ch Peek() const { - typename StreamType::int_type c = stream_.peek(); - return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast<Ch>(c) : '\0'; - } - - Ch Take() { - typename StreamType::int_type c = stream_.get(); - if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { - count_++; - return static_cast<Ch>(c); - } - else - return '\0'; - } - - // tellg() may return -1 when failed. So we count by ourself. - size_t Tell() const { return count_; } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. - int i; - bool hasError = false; - for (i = 0; i < 4; ++i) { - typename StreamType::int_type c = stream_.get(); - if (c == StreamType::traits_type::eof()) { - hasError = true; - stream_.clear(); - break; - } - peekBuffer_[i] = static_cast<Ch>(c); - } - for (--i; i >= 0; --i) - stream_.putback(peekBuffer_[i]); - return !hasError ? peekBuffer_ : 0; - } - -private: - BasicIStreamWrapper(const BasicIStreamWrapper&); - BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); - - StreamType& stream_; - size_t count_; //!< Number of characters read. Note: - mutable Ch peekBuffer_[4]; -}; - -typedef BasicIStreamWrapper<std::istream> IStreamWrapper; -typedef BasicIStreamWrapper<std::wistream> WIStreamWrapper; - -#if defined(__clang__) || defined(_MSC_VER) -RAPIDJSON_DIAG_POP -#endif - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_ISTREAMWRAPPER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/memorybuffer.h b/ext/librethinkdbxx/src/rapidjson/memorybuffer.h deleted file mode 100644 index 39bee1de..00000000 --- a/ext/librethinkdbxx/src/rapidjson/memorybuffer.h +++ /dev/null @@ -1,70 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_MEMORYBUFFER_H_ -#define RAPIDJSON_MEMORYBUFFER_H_ - -#include "stream.h" -#include "internal/stack.h" - -RAPIDJSON_NAMESPACE_BEGIN - -//! Represents an in-memory output byte stream. -/*! - This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. - - It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. - - Differences between MemoryBuffer and StringBuffer: - 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. - 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. - - \tparam Allocator type for allocating memory buffer. - \note implements Stream concept -*/ -template <typename Allocator = CrtAllocator> -struct GenericMemoryBuffer { - typedef char Ch; // byte - - GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} - - void Put(Ch c) { *stack_.template Push<Ch>() = c; } - void Flush() {} - - void Clear() { stack_.Clear(); } - void ShrinkToFit() { stack_.ShrinkToFit(); } - Ch* Push(size_t count) { return stack_.template Push<Ch>(count); } - void Pop(size_t count) { stack_.template Pop<Ch>(count); } - - const Ch* GetBuffer() const { - return stack_.template Bottom<Ch>(); - } - - size_t GetSize() const { return stack_.GetSize(); } - - static const size_t kDefaultCapacity = 256; - mutable internal::Stack<Allocator> stack_; -}; - -typedef GenericMemoryBuffer<> MemoryBuffer; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { - std::memset(memoryBuffer.stack_.Push<char>(n), c, n * sizeof(c)); -} - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/memorystream.h b/ext/librethinkdbxx/src/rapidjson/memorystream.h deleted file mode 100644 index 1d71d8a4..00000000 --- a/ext/librethinkdbxx/src/rapidjson/memorystream.h +++ /dev/null @@ -1,71 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_MEMORYSTREAM_H_ -#define RAPIDJSON_MEMORYSTREAM_H_ - -#include "stream.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(unreachable-code) -RAPIDJSON_DIAG_OFF(missing-noreturn) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Represents an in-memory input byte stream. -/*! - This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. - - It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. - - Differences between MemoryStream and StringStream: - 1. StringStream has encoding but MemoryStream is a byte stream. - 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. - 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). - \note implements Stream concept -*/ -struct MemoryStream { - typedef char Ch; // byte - - MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} - - Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } - Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } - size_t Tell() const { return static_cast<size_t>(src_ - begin_); } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - return Tell() + 4 <= size_ ? src_ : 0; - } - - const Ch* src_; //!< Current read position. - const Ch* begin_; //!< Original head of the string. - const Ch* end_; //!< End of stream. - size_t size_; //!< Size of the stream. -}; - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h b/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h deleted file mode 100644 index 18111286..00000000 --- a/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h +++ /dev/null @@ -1,316 +0,0 @@ -// ISO C9x compliant inttypes.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -// The above software in this distribution may have been modified by -// THL A29 Limited ("Tencent Modifications"). -// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_INTTYPES_H_ // [ -#define _MSC_INTTYPES_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -#include "stdint.h" - -// miloyip: VC supports inttypes.h since VC2013 -#if _MSC_VER >= 1800 -#include <inttypes.h> -#else - -// 7.8 Format conversion of integer types - -typedef struct { - intmax_t quot; - intmax_t rem; -} imaxdiv_t; - -// 7.8.1 Macros for format specifiers - -#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 - -// The fprintf macros for signed integers are: -#define PRId8 "d" -#define PRIi8 "i" -#define PRIdLEAST8 "d" -#define PRIiLEAST8 "i" -#define PRIdFAST8 "d" -#define PRIiFAST8 "i" - -#define PRId16 "hd" -#define PRIi16 "hi" -#define PRIdLEAST16 "hd" -#define PRIiLEAST16 "hi" -#define PRIdFAST16 "hd" -#define PRIiFAST16 "hi" - -#define PRId32 "I32d" -#define PRIi32 "I32i" -#define PRIdLEAST32 "I32d" -#define PRIiLEAST32 "I32i" -#define PRIdFAST32 "I32d" -#define PRIiFAST32 "I32i" - -#define PRId64 "I64d" -#define PRIi64 "I64i" -#define PRIdLEAST64 "I64d" -#define PRIiLEAST64 "I64i" -#define PRIdFAST64 "I64d" -#define PRIiFAST64 "I64i" - -#define PRIdMAX "I64d" -#define PRIiMAX "I64i" - -#define PRIdPTR "Id" -#define PRIiPTR "Ii" - -// The fprintf macros for unsigned integers are: -#define PRIo8 "o" -#define PRIu8 "u" -#define PRIx8 "x" -#define PRIX8 "X" -#define PRIoLEAST8 "o" -#define PRIuLEAST8 "u" -#define PRIxLEAST8 "x" -#define PRIXLEAST8 "X" -#define PRIoFAST8 "o" -#define PRIuFAST8 "u" -#define PRIxFAST8 "x" -#define PRIXFAST8 "X" - -#define PRIo16 "ho" -#define PRIu16 "hu" -#define PRIx16 "hx" -#define PRIX16 "hX" -#define PRIoLEAST16 "ho" -#define PRIuLEAST16 "hu" -#define PRIxLEAST16 "hx" -#define PRIXLEAST16 "hX" -#define PRIoFAST16 "ho" -#define PRIuFAST16 "hu" -#define PRIxFAST16 "hx" -#define PRIXFAST16 "hX" - -#define PRIo32 "I32o" -#define PRIu32 "I32u" -#define PRIx32 "I32x" -#define PRIX32 "I32X" -#define PRIoLEAST32 "I32o" -#define PRIuLEAST32 "I32u" -#define PRIxLEAST32 "I32x" -#define PRIXLEAST32 "I32X" -#define PRIoFAST32 "I32o" -#define PRIuFAST32 "I32u" -#define PRIxFAST32 "I32x" -#define PRIXFAST32 "I32X" - -#define PRIo64 "I64o" -#define PRIu64 "I64u" -#define PRIx64 "I64x" -#define PRIX64 "I64X" -#define PRIoLEAST64 "I64o" -#define PRIuLEAST64 "I64u" -#define PRIxLEAST64 "I64x" -#define PRIXLEAST64 "I64X" -#define PRIoFAST64 "I64o" -#define PRIuFAST64 "I64u" -#define PRIxFAST64 "I64x" -#define PRIXFAST64 "I64X" - -#define PRIoMAX "I64o" -#define PRIuMAX "I64u" -#define PRIxMAX "I64x" -#define PRIXMAX "I64X" - -#define PRIoPTR "Io" -#define PRIuPTR "Iu" -#define PRIxPTR "Ix" -#define PRIXPTR "IX" - -// The fscanf macros for signed integers are: -#define SCNd8 "d" -#define SCNi8 "i" -#define SCNdLEAST8 "d" -#define SCNiLEAST8 "i" -#define SCNdFAST8 "d" -#define SCNiFAST8 "i" - -#define SCNd16 "hd" -#define SCNi16 "hi" -#define SCNdLEAST16 "hd" -#define SCNiLEAST16 "hi" -#define SCNdFAST16 "hd" -#define SCNiFAST16 "hi" - -#define SCNd32 "ld" -#define SCNi32 "li" -#define SCNdLEAST32 "ld" -#define SCNiLEAST32 "li" -#define SCNdFAST32 "ld" -#define SCNiFAST32 "li" - -#define SCNd64 "I64d" -#define SCNi64 "I64i" -#define SCNdLEAST64 "I64d" -#define SCNiLEAST64 "I64i" -#define SCNdFAST64 "I64d" -#define SCNiFAST64 "I64i" - -#define SCNdMAX "I64d" -#define SCNiMAX "I64i" - -#ifdef _WIN64 // [ -# define SCNdPTR "I64d" -# define SCNiPTR "I64i" -#else // _WIN64 ][ -# define SCNdPTR "ld" -# define SCNiPTR "li" -#endif // _WIN64 ] - -// The fscanf macros for unsigned integers are: -#define SCNo8 "o" -#define SCNu8 "u" -#define SCNx8 "x" -#define SCNX8 "X" -#define SCNoLEAST8 "o" -#define SCNuLEAST8 "u" -#define SCNxLEAST8 "x" -#define SCNXLEAST8 "X" -#define SCNoFAST8 "o" -#define SCNuFAST8 "u" -#define SCNxFAST8 "x" -#define SCNXFAST8 "X" - -#define SCNo16 "ho" -#define SCNu16 "hu" -#define SCNx16 "hx" -#define SCNX16 "hX" -#define SCNoLEAST16 "ho" -#define SCNuLEAST16 "hu" -#define SCNxLEAST16 "hx" -#define SCNXLEAST16 "hX" -#define SCNoFAST16 "ho" -#define SCNuFAST16 "hu" -#define SCNxFAST16 "hx" -#define SCNXFAST16 "hX" - -#define SCNo32 "lo" -#define SCNu32 "lu" -#define SCNx32 "lx" -#define SCNX32 "lX" -#define SCNoLEAST32 "lo" -#define SCNuLEAST32 "lu" -#define SCNxLEAST32 "lx" -#define SCNXLEAST32 "lX" -#define SCNoFAST32 "lo" -#define SCNuFAST32 "lu" -#define SCNxFAST32 "lx" -#define SCNXFAST32 "lX" - -#define SCNo64 "I64o" -#define SCNu64 "I64u" -#define SCNx64 "I64x" -#define SCNX64 "I64X" -#define SCNoLEAST64 "I64o" -#define SCNuLEAST64 "I64u" -#define SCNxLEAST64 "I64x" -#define SCNXLEAST64 "I64X" -#define SCNoFAST64 "I64o" -#define SCNuFAST64 "I64u" -#define SCNxFAST64 "I64x" -#define SCNXFAST64 "I64X" - -#define SCNoMAX "I64o" -#define SCNuMAX "I64u" -#define SCNxMAX "I64x" -#define SCNXMAX "I64X" - -#ifdef _WIN64 // [ -# define SCNoPTR "I64o" -# define SCNuPTR "I64u" -# define SCNxPTR "I64x" -# define SCNXPTR "I64X" -#else // _WIN64 ][ -# define SCNoPTR "lo" -# define SCNuPTR "lu" -# define SCNxPTR "lx" -# define SCNXPTR "lX" -#endif // _WIN64 ] - -#endif // __STDC_FORMAT_MACROS ] - -// 7.8.2 Functions for greatest-width integer types - -// 7.8.2.1 The imaxabs function -#define imaxabs _abs64 - -// 7.8.2.2 The imaxdiv function - -// This is modified version of div() function from Microsoft's div.c found -// in %MSVC.NET%\crt\src\div.c -#ifdef STATIC_IMAXDIV // [ -static -#else // STATIC_IMAXDIV ][ -_inline -#endif // STATIC_IMAXDIV ] -imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) -{ - imaxdiv_t result; - - result.quot = numer / denom; - result.rem = numer % denom; - - if (numer < 0 && result.rem > 0) { - // did division wrong; must fix up - ++result.quot; - result.rem -= denom; - } - - return result; -} - -// 7.8.2.3 The strtoimax and strtoumax functions -#define strtoimax _strtoi64 -#define strtoumax _strtoui64 - -// 7.8.2.4 The wcstoimax and wcstoumax functions -#define wcstoimax _wcstoi64 -#define wcstoumax _wcstoui64 - -#endif // _MSC_VER >= 1800 - -#endif // _MSC_INTTYPES_H_ ] diff --git a/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h b/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h deleted file mode 100644 index 3d4477b9..00000000 --- a/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h +++ /dev/null @@ -1,300 +0,0 @@ -// ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -// The above software in this distribution may have been modified by -// THL A29 Limited ("Tencent Modifications"). -// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_STDINT_H_ // [ -#define _MSC_STDINT_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. -#if _MSC_VER >= 1600 // [ -#include <stdint.h> - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -#undef INT8_C -#undef INT16_C -#undef INT32_C -#undef INT64_C -#undef UINT8_C -#undef UINT16_C -#undef UINT32_C -#undef UINT64_C - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with <boost/cstdint.hpp>. -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#else // ] _MSC_VER >= 1700 [ - -#include <limits.h> - -// For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we have to wrap <wchar.h> include with 'extern "C++" {}' -// or compiler would give many errors like this: -// error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#if defined(__cplusplus) && !defined(_M_ARM) -extern "C" { -#endif -# include <wchar.h> -#if defined(__cplusplus) && !defined(_M_ARM) -} -#endif - -// Define _W64 macros to mark types changing their size, like intptr_t. -#ifndef _W64 -# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 -# define _W64 __w64 -# else -# define _W64 -# endif -#endif - - -// 7.18.1 Integer types - -// 7.18.1.1 Exact-width integer types - -// Visual Studio 6 and Embedded Visual C++ 4 doesn't -// realize that, e.g. char has the same size as __int8 -// so we give up on __intX for them. -#if (_MSC_VER < 1300) - typedef signed char int8_t; - typedef signed short int16_t; - typedef signed int int32_t; - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; -#else - typedef signed __int8 int8_t; - typedef signed __int16 int16_t; - typedef signed __int32 int32_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; -#endif -typedef signed __int64 int64_t; -typedef unsigned __int64 uint64_t; - - -// 7.18.1.2 Minimum-width integer types -typedef int8_t int_least8_t; -typedef int16_t int_least16_t; -typedef int32_t int_least32_t; -typedef int64_t int_least64_t; -typedef uint8_t uint_least8_t; -typedef uint16_t uint_least16_t; -typedef uint32_t uint_least32_t; -typedef uint64_t uint_least64_t; - -// 7.18.1.3 Fastest minimum-width integer types -typedef int8_t int_fast8_t; -typedef int16_t int_fast16_t; -typedef int32_t int_fast32_t; -typedef int64_t int_fast64_t; -typedef uint8_t uint_fast8_t; -typedef uint16_t uint_fast16_t; -typedef uint32_t uint_fast32_t; -typedef uint64_t uint_fast64_t; - -// 7.18.1.4 Integer types capable of holding object pointers -#ifdef _WIN64 // [ - typedef signed __int64 intptr_t; - typedef unsigned __int64 uintptr_t; -#else // _WIN64 ][ - typedef _W64 signed int intptr_t; - typedef _W64 unsigned int uintptr_t; -#endif // _WIN64 ] - -// 7.18.1.5 Greatest-width integer types -typedef int64_t intmax_t; -typedef uint64_t uintmax_t; - - -// 7.18.2 Limits of specified-width integer types - -#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 - -// 7.18.2.1 Limits of exact-width integer types -#define INT8_MIN ((int8_t)_I8_MIN) -#define INT8_MAX _I8_MAX -#define INT16_MIN ((int16_t)_I16_MIN) -#define INT16_MAX _I16_MAX -#define INT32_MIN ((int32_t)_I32_MIN) -#define INT32_MAX _I32_MAX -#define INT64_MIN ((int64_t)_I64_MIN) -#define INT64_MAX _I64_MAX -#define UINT8_MAX _UI8_MAX -#define UINT16_MAX _UI16_MAX -#define UINT32_MAX _UI32_MAX -#define UINT64_MAX _UI64_MAX - -// 7.18.2.2 Limits of minimum-width integer types -#define INT_LEAST8_MIN INT8_MIN -#define INT_LEAST8_MAX INT8_MAX -#define INT_LEAST16_MIN INT16_MIN -#define INT_LEAST16_MAX INT16_MAX -#define INT_LEAST32_MIN INT32_MIN -#define INT_LEAST32_MAX INT32_MAX -#define INT_LEAST64_MIN INT64_MIN -#define INT_LEAST64_MAX INT64_MAX -#define UINT_LEAST8_MAX UINT8_MAX -#define UINT_LEAST16_MAX UINT16_MAX -#define UINT_LEAST32_MAX UINT32_MAX -#define UINT_LEAST64_MAX UINT64_MAX - -// 7.18.2.3 Limits of fastest minimum-width integer types -#define INT_FAST8_MIN INT8_MIN -#define INT_FAST8_MAX INT8_MAX -#define INT_FAST16_MIN INT16_MIN -#define INT_FAST16_MAX INT16_MAX -#define INT_FAST32_MIN INT32_MIN -#define INT_FAST32_MAX INT32_MAX -#define INT_FAST64_MIN INT64_MIN -#define INT_FAST64_MAX INT64_MAX -#define UINT_FAST8_MAX UINT8_MAX -#define UINT_FAST16_MAX UINT16_MAX -#define UINT_FAST32_MAX UINT32_MAX -#define UINT_FAST64_MAX UINT64_MAX - -// 7.18.2.4 Limits of integer types capable of holding object pointers -#ifdef _WIN64 // [ -# define INTPTR_MIN INT64_MIN -# define INTPTR_MAX INT64_MAX -# define UINTPTR_MAX UINT64_MAX -#else // _WIN64 ][ -# define INTPTR_MIN INT32_MIN -# define INTPTR_MAX INT32_MAX -# define UINTPTR_MAX UINT32_MAX -#endif // _WIN64 ] - -// 7.18.2.5 Limits of greatest-width integer types -#define INTMAX_MIN INT64_MIN -#define INTMAX_MAX INT64_MAX -#define UINTMAX_MAX UINT64_MAX - -// 7.18.3 Limits of other integer types - -#ifdef _WIN64 // [ -# define PTRDIFF_MIN _I64_MIN -# define PTRDIFF_MAX _I64_MAX -#else // _WIN64 ][ -# define PTRDIFF_MIN _I32_MIN -# define PTRDIFF_MAX _I32_MAX -#endif // _WIN64 ] - -#define SIG_ATOMIC_MIN INT_MIN -#define SIG_ATOMIC_MAX INT_MAX - -#ifndef SIZE_MAX // [ -# ifdef _WIN64 // [ -# define SIZE_MAX _UI64_MAX -# else // _WIN64 ][ -# define SIZE_MAX _UI32_MAX -# endif // _WIN64 ] -#endif // SIZE_MAX ] - -// WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h> -#ifndef WCHAR_MIN // [ -# define WCHAR_MIN 0 -#endif // WCHAR_MIN ] -#ifndef WCHAR_MAX // [ -# define WCHAR_MAX _UI16_MAX -#endif // WCHAR_MAX ] - -#define WINT_MIN 0 -#define WINT_MAX _UI16_MAX - -#endif // __STDC_LIMIT_MACROS ] - - -// 7.18.4 Limits of other integer types - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with <boost/cstdint.hpp>. -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#endif // _MSC_VER >= 1600 ] - -#endif // _MSC_STDINT_H_ ] diff --git a/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h b/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h deleted file mode 100644 index 6f4667c0..00000000 --- a/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h +++ /dev/null @@ -1,81 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ -#define RAPIDJSON_OSTREAMWRAPPER_H_ - -#include "stream.h" -#include <iosfwd> - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. -/*! - The classes can be wrapped including but not limited to: - - - \c std::ostringstream - - \c std::stringstream - - \c std::wpstringstream - - \c std::wstringstream - - \c std::ifstream - - \c std::fstream - - \c std::wofstream - - \c std::wfstream - - \tparam StreamType Class derived from \c std::basic_ostream. -*/ - -template <typename StreamType> -class BasicOStreamWrapper { -public: - typedef typename StreamType::char_type Ch; - BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} - - void Put(Ch c) { - stream_.put(c); - } - - void Flush() { - stream_.flush(); - } - - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - BasicOStreamWrapper(const BasicOStreamWrapper&); - BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); - - StreamType& stream_; -}; - -typedef BasicOStreamWrapper<std::ostream> OStreamWrapper; -typedef BasicOStreamWrapper<std::wostream> WOStreamWrapper; - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/pointer.h b/ext/librethinkdbxx/src/rapidjson/pointer.h deleted file mode 100644 index 0206ac1c..00000000 --- a/ext/librethinkdbxx/src/rapidjson/pointer.h +++ /dev/null @@ -1,1358 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_POINTER_H_ -#define RAPIDJSON_POINTER_H_ - -#include "document.h" -#include "internal/itoa.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(switch-enum) -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token - -//! Error code of parsing. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode -*/ -enum PointerParseErrorCode { - kPointerParseErrorNone = 0, //!< The parse is successful - - kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' - kPointerParseErrorInvalidEscape, //!< Invalid escape - kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment - kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericPointer - -//! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. -/*! - This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" - (https://tools.ietf.org/html/rfc6901). - - A JSON pointer is for identifying a specific value in a JSON document - (GenericDocument). It can simplify coding of DOM tree manipulation, because it - can access multiple-level depth of DOM tree with single API call. - - After it parses a string representation (e.g. "/foo/0" or URI fragment - representation (e.g. "#/foo/0") into its internal representation (tokens), - it can be used to resolve a specific value in multiple documents, or sub-tree - of documents. - - Contrary to GenericValue, Pointer can be copy constructed and copy assigned. - Apart from assignment, a Pointer cannot be modified after construction. - - Although Pointer is very convenient, please aware that constructing Pointer - involves parsing and dynamic memory allocation. A special constructor with user- - supplied tokens eliminates these. - - GenericPointer depends on GenericDocument and GenericValue. - - \tparam ValueType The value type of the DOM tree. E.g. GenericValue<UTF8<> > - \tparam Allocator The allocator type for allocating memory for internal representation. - - \note GenericPointer uses same encoding of ValueType. - However, Allocator of GenericPointer is independent of Allocator of Value. -*/ -template <typename ValueType, typename Allocator = CrtAllocator> -class GenericPointer { -public: - typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value - typedef typename ValueType::Ch Ch; //!< Character type from Value - - //! A token is the basic units of internal representation. - /*! - A JSON pointer string representation "/foo/123" is parsed to two tokens: - "foo" and 123. 123 will be represented in both numeric form and string form. - They are resolved according to the actual value type (object or array). - - For token that are not numbers, or the numeric value is out of bound - (greater than limits of SizeType), they are only treated as string form - (i.e. the token's index will be equal to kPointerInvalidIndex). - - This struct is public so that user can create a Pointer without parsing and - allocation, using a special constructor. - */ - struct Token { - const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. - SizeType length; //!< Length of the name. - SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. - }; - - //!@name Constructors and destructor. - //@{ - - //! Default constructor. - GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} - - //! Constructor that parses a string or URI fragment representation. - /*! - \param source A null-terminated, string or URI fragment representation of JSON pointer. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - */ - explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source, internal::StrLen(source)); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Constructor that parses a string or URI fragment representation. - /*! - \param source A string or URI fragment representation of JSON pointer. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - explicit GenericPointer(const std::basic_string<Ch>& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source.c_str(), source.size()); - } -#endif - - //! Constructor that parses a string or URI fragment representation, with length of the source string. - /*! - \param source A string or URI fragment representation of JSON pointer. - \param length Length of source. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - \note Slightly faster than the overload without length. - */ - GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source, length); - } - - //! Constructor with user-supplied tokens. - /*! - This constructor let user supplies const array of tokens. - This prevents the parsing process and eliminates allocation. - This is preferred for memory constrained environments. - - \param tokens An constant array of tokens representing the JSON pointer. - \param tokenCount Number of tokens. - - \b Example - \code - #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } - #define INDEX(i) { #i, sizeof(#i) - 1, i } - - static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; - static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); - // Equivalent to static const Pointer p("/foo/123"); - - #undef NAME - #undef INDEX - \endcode - */ - GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast<Token*>(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} - - //! Copy constructor. - GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - *this = rhs; - } - - //! Destructor. - ~GenericPointer() { - if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. - Allocator::Free(tokens_); - RAPIDJSON_DELETE(ownAllocator_); - } - - //! Assignment operator. - GenericPointer& operator=(const GenericPointer& rhs) { - if (this != &rhs) { - // Do not delete ownAllcator - if (nameBuffer_) - Allocator::Free(tokens_); - - tokenCount_ = rhs.tokenCount_; - parseErrorOffset_ = rhs.parseErrorOffset_; - parseErrorCode_ = rhs.parseErrorCode_; - - if (rhs.nameBuffer_) - CopyFromRaw(rhs); // Normally parsed tokens. - else { - tokens_ = rhs.tokens_; // User supplied const tokens. - nameBuffer_ = 0; - } - } - return *this; - } - - //@} - - //!@name Append token - //@{ - - //! Append a token and return a new Pointer - /*! - \param token Token to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const Token& token, Allocator* allocator = 0) const { - GenericPointer r; - r.allocator_ = allocator; - Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); - std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); - r.tokens_[tokenCount_].name = p; - r.tokens_[tokenCount_].length = token.length; - r.tokens_[tokenCount_].index = token.index; - return r; - } - - //! Append a name token with length, and return a new Pointer - /*! - \param name Name to be appended. - \param length Length of name. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { - Token token = { name, length, kPointerInvalidIndex }; - return Append(token, allocator); - } - - //! Append a name token without length, and return a new Pointer - /*! - \param name Name (const Ch*) to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr<internal::IsSame<typename internal::RemoveConst<T>::Type, Ch> >), (GenericPointer)) - Append(T* name, Allocator* allocator = 0) const { - return Append(name, StrLen(name), allocator); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Append a name token, and return a new Pointer - /*! - \param name Name to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const std::basic_string<Ch>& name, Allocator* allocator = 0) const { - return Append(name.c_str(), static_cast<SizeType>(name.size()), allocator); - } -#endif - - //! Append a index token, and return a new Pointer - /*! - \param index Index to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(SizeType index, Allocator* allocator = 0) const { - char buffer[21]; - char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); - SizeType length = static_cast<SizeType>(end - buffer); - buffer[length] = '\0'; - - if (sizeof(Ch) == 1) { - Token token = { reinterpret_cast<Ch*>(buffer), length, index }; - return Append(token, allocator); - } - else { - Ch name[21]; - for (size_t i = 0; i <= length; i++) - name[i] = buffer[i]; - Token token = { name, length, index }; - return Append(token, allocator); - } - } - - //! Append a token by value, and return a new Pointer - /*! - \param token token to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { - if (token.IsString()) - return Append(token.GetString(), token.GetStringLength(), allocator); - else { - RAPIDJSON_ASSERT(token.IsUint64()); - RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); - return Append(static_cast<SizeType>(token.GetUint64()), allocator); - } - } - - //!@name Handling Parse Error - //@{ - - //! Check whether this is a valid pointer. - bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } - - //! Get the parsing error offset in code unit. - size_t GetParseErrorOffset() const { return parseErrorOffset_; } - - //! Get the parsing error code. - PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } - - //@} - - //! Get the allocator of this pointer. - Allocator& GetAllocator() { return *allocator_; } - - //!@name Tokens - //@{ - - //! Get the token array (const version only). - const Token* GetTokens() const { return tokens_; } - - //! Get the number of tokens. - size_t GetTokenCount() const { return tokenCount_; } - - //@} - - //!@name Equality/inequality operators - //@{ - - //! Equality operator. - /*! - \note When any pointers are invalid, always returns false. - */ - bool operator==(const GenericPointer& rhs) const { - if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) - return false; - - for (size_t i = 0; i < tokenCount_; i++) { - if (tokens_[i].index != rhs.tokens_[i].index || - tokens_[i].length != rhs.tokens_[i].length || - (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) - { - return false; - } - } - - return true; - } - - //! Inequality operator. - /*! - \note When any pointers are invalid, always returns true. - */ - bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } - - //@} - - //!@name Stringify - //@{ - - //! Stringify the pointer into string representation. - /*! - \tparam OutputStream Type of output stream. - \param os The output stream. - */ - template<typename OutputStream> - bool Stringify(OutputStream& os) const { - return Stringify<false, OutputStream>(os); - } - - //! Stringify the pointer into URI fragment representation. - /*! - \tparam OutputStream Type of output stream. - \param os The output stream. - */ - template<typename OutputStream> - bool StringifyUriFragment(OutputStream& os) const { - return Stringify<true, OutputStream>(os); - } - - //@} - - //!@name Create value - //@{ - - //! Create a value in a subtree. - /*! - If the value is not exist, it creates all parent values and a JSON Null value. - So it always succeed and return the newly created or existing value. - - Remind that it may change types of parents according to tokens, so it - potentially removes previously stored values. For example, if a document - was an array, and "/foo" is used to create a value, then the document - will be changed to an object, and all existing array elements are lost. - - \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \param alreadyExist If non-null, it stores whether the resolved value is already exist. - \return The resolved newly created (a JSON Null value), or already exists value. - */ - ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { - RAPIDJSON_ASSERT(IsValid()); - ValueType* v = &root; - bool exist = true; - for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - if (v->IsArray() && t->name[0] == '-' && t->length == 1) { - v->PushBack(ValueType().Move(), allocator); - v = &((*v)[v->Size() - 1]); - exist = false; - } - else { - if (t->index == kPointerInvalidIndex) { // must be object name - if (!v->IsObject()) - v->SetObject(); // Change to Object - } - else { // object name or array index - if (!v->IsArray() && !v->IsObject()) - v->SetArray(); // Change to Array - } - - if (v->IsArray()) { - if (t->index >= v->Size()) { - v->Reserve(t->index + 1, allocator); - while (t->index >= v->Size()) - v->PushBack(ValueType().Move(), allocator); - exist = false; - } - v = &((*v)[t->index]); - } - else { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef<Ch>(t->name, t->length)); - if (m == v->MemberEnd()) { - v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); - v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end - exist = false; - } - else - v = &m->value; - } - } - } - - if (alreadyExist) - *alreadyExist = exist; - - return *v; - } - - //! Creates a value in a document. - /*! - \param document A document to be resolved. - \param alreadyExist If non-null, it stores whether the resolved value is already exist. - \return The resolved newly created, or already exists value. - */ - template <typename stackAllocator> - ValueType& Create(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, bool* alreadyExist = 0) const { - return Create(document, document.GetAllocator(), alreadyExist); - } - - //@} - - //!@name Query value - //@{ - - //! Query a value in a subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. - \return Pointer to the value if it can be resolved. Otherwise null. - - \note - There are only 3 situations when a value cannot be resolved: - 1. A value in the path is not an array nor object. - 2. An object value does not contain the token. - 3. A token is out of range of an array value. - - Use unresolvedTokenIndex to retrieve the token index. - */ - ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { - RAPIDJSON_ASSERT(IsValid()); - ValueType* v = &root; - for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - switch (v->GetType()) { - case kObjectType: - { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef<Ch>(t->name, t->length)); - if (m == v->MemberEnd()) - break; - v = &m->value; - } - continue; - case kArrayType: - if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - break; - v = &((*v)[t->index]); - continue; - default: - break; - } - - // Error: unresolved token - if (unresolvedTokenIndex) - *unresolvedTokenIndex = static_cast<size_t>(t - tokens_); - return 0; - } - return v; - } - - //! Query a const value in a const subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \return Pointer to the value if it can be resolved. Otherwise null. - */ - const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { - return Get(const_cast<ValueType&>(root), unresolvedTokenIndex); - } - - //@} - - //!@name Query a value with default - //@{ - - //! Query a value in a subtree with default value. - /*! - Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. - So that this function always succeed. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param defaultValue Default value to be cloned if the value was not exists. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); - } - - //! Query a value in a subtree with default null-terminated string. - ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.SetString(defaultValue, allocator); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Query a value in a subtree with default std::basic_string. - ValueType& GetWithDefault(ValueType& root, const std::basic_string<Ch>& defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.SetString(defaultValue, allocator); - } -#endif - - //! Query a value in a subtree with default primitive value. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (ValueType&)) - GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { - return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); - } - - //! Query a value in a document with default value. - template <typename stackAllocator> - ValueType& GetWithDefault(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, const ValueType& defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - - //! Query a value in a document with default null-terminated string. - template <typename stackAllocator> - ValueType& GetWithDefault(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, const Ch* defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Query a value in a document with default std::basic_string. - template <typename stackAllocator> - ValueType& GetWithDefault(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, const std::basic_string<Ch>& defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } -#endif - - //! Query a value in a document with default primitive value. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template <typename T, typename stackAllocator> - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (ValueType&)) - GetWithDefault(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, T defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - - //@} - - //!@name Set a value - //@{ - - //! Set a value in a subtree, with move semantics. - /*! - It creates all parents if they are not exist or types are different to the tokens. - So this function always succeeds but potentially remove existing values. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param value Value to be set. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = value; - } - - //! Set a value in a subtree, with copy semantics. - ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator).CopyFrom(value, allocator); - } - - //! Set a null-terminated string in a subtree. - ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value, allocator).Move(); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Set a std::basic_string in a subtree. - ValueType& Set(ValueType& root, const std::basic_string<Ch>& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value, allocator).Move(); - } -#endif - - //! Set a primitive value in a subtree. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template <typename T> - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (ValueType&)) - Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value).Move(); - } - - //! Set a value in a document, with move semantics. - template <typename stackAllocator> - ValueType& Set(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, ValueType& value) const { - return Create(document) = value; - } - - //! Set a value in a document, with copy semantics. - template <typename stackAllocator> - ValueType& Set(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, const ValueType& value) const { - return Create(document).CopyFrom(value, document.GetAllocator()); - } - - //! Set a null-terminated string in a document. - template <typename stackAllocator> - ValueType& Set(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, const Ch* value) const { - return Create(document) = ValueType(value, document.GetAllocator()).Move(); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Sets a std::basic_string in a document. - template <typename stackAllocator> - ValueType& Set(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, const std::basic_string<Ch>& value) const { - return Create(document) = ValueType(value, document.GetAllocator()).Move(); - } -#endif - - //! Set a primitive value in a document. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template <typename T, typename stackAllocator> - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T>, internal::IsGenericValue<T> >), (ValueType&)) - Set(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, T value) const { - return Create(document) = value; - } - - //@} - - //!@name Swap a value - //@{ - - //! Swap a value with a value in a subtree. - /*! - It creates all parents if they are not exist or types are different to the tokens. - So this function always succeeds but potentially remove existing values. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param value Value to be swapped. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator).Swap(value); - } - - //! Swap a value with a value in a document. - template <typename stackAllocator> - ValueType& Swap(GenericDocument<EncodingType, typename ValueType::AllocatorType, stackAllocator>& document, ValueType& value) const { - return Create(document).Swap(value); - } - - //@} - - //! Erase a value in a subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \return Whether the resolved value is found and erased. - - \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. - */ - bool Erase(ValueType& root) const { - RAPIDJSON_ASSERT(IsValid()); - if (tokenCount_ == 0) // Cannot erase the root - return false; - - ValueType* v = &root; - const Token* last = tokens_ + (tokenCount_ - 1); - for (const Token *t = tokens_; t != last; ++t) { - switch (v->GetType()) { - case kObjectType: - { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef<Ch>(t->name, t->length)); - if (m == v->MemberEnd()) - return false; - v = &m->value; - } - break; - case kArrayType: - if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - return false; - v = &((*v)[t->index]); - break; - default: - return false; - } - } - - switch (v->GetType()) { - case kObjectType: - return v->EraseMember(GenericStringRef<Ch>(last->name, last->length)); - case kArrayType: - if (last->index == kPointerInvalidIndex || last->index >= v->Size()) - return false; - v->Erase(v->Begin() + last->index); - return true; - default: - return false; - } - } - -private: - //! Clone the content from rhs to this. - /*! - \param rhs Source pointer. - \param extraToken Extra tokens to be allocated. - \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. - \return Start of non-occupied name buffer, for storing extra names. - */ - Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { - if (!allocator_) // allocator is independently owned. - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - - size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens - for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) - nameBufferSize += t->length; - - tokenCount_ = rhs.tokenCount_ + extraToken; - tokens_ = static_cast<Token *>(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); - nameBuffer_ = reinterpret_cast<Ch *>(tokens_ + tokenCount_); - if (rhs.tokenCount_ > 0) { - std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); - } - if (nameBufferSize > 0) { - std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); - } - - // Adjust pointers to name buffer - std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; - for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) - t->name += diff; - - return nameBuffer_ + nameBufferSize; - } - - //! Check whether a character should be percent-encoded. - /*! - According to RFC 3986 2.3 Unreserved Characters. - \param c The character (code unit) to be tested. - */ - bool NeedPercentEncode(Ch c) const { - return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); - } - - //! Parse a JSON String or its URI fragment representation into tokens. -#ifndef __clang__ // -Wdocumentation - /*! - \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. - \param length Length of the source string. - \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. - */ -#endif - void Parse(const Ch* source, size_t length) { - RAPIDJSON_ASSERT(source != NULL); - RAPIDJSON_ASSERT(nameBuffer_ == 0); - RAPIDJSON_ASSERT(tokens_ == 0); - - // Create own allocator if user did not supply. - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - - // Count number of '/' as tokenCount - tokenCount_ = 0; - for (const Ch* s = source; s != source + length; s++) - if (*s == '/') - tokenCount_++; - - Token* token = tokens_ = static_cast<Token *>(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); - Ch* name = nameBuffer_ = reinterpret_cast<Ch *>(tokens_ + tokenCount_); - size_t i = 0; - - // Detect if it is a URI fragment - bool uriFragment = false; - if (source[i] == '#') { - uriFragment = true; - i++; - } - - if (i != length && source[i] != '/') { - parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; - goto error; - } - - while (i < length) { - RAPIDJSON_ASSERT(source[i] == '/'); - i++; // consumes '/' - - token->name = name; - bool isNumber = true; - - while (i < length && source[i] != '/') { - Ch c = source[i]; - if (uriFragment) { - // Decoding percent-encoding for URI fragment - if (c == '%') { - PercentDecodeStream is(&source[i], source + length); - GenericInsituStringStream<EncodingType> os(name); - Ch* begin = os.PutBegin(); - if (!Transcoder<UTF8<>, EncodingType>().Validate(is, os) || !is.IsValid()) { - parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; - goto error; - } - size_t len = os.PutEnd(begin); - i += is.Tell() - 1; - if (len == 1) - c = *name; - else { - name += len; - isNumber = false; - i++; - continue; - } - } - else if (NeedPercentEncode(c)) { - parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; - goto error; - } - } - - i++; - - // Escaping "~0" -> '~', "~1" -> '/' - if (c == '~') { - if (i < length) { - c = source[i]; - if (c == '0') c = '~'; - else if (c == '1') c = '/'; - else { - parseErrorCode_ = kPointerParseErrorInvalidEscape; - goto error; - } - i++; - } - else { - parseErrorCode_ = kPointerParseErrorInvalidEscape; - goto error; - } - } - - // First check for index: all of characters are digit - if (c < '0' || c > '9') - isNumber = false; - - *name++ = c; - } - token->length = static_cast<SizeType>(name - token->name); - if (token->length == 0) - isNumber = false; - *name++ = '\0'; // Null terminator - - // Second check for index: more than one digit cannot have leading zero - if (isNumber && token->length > 1 && token->name[0] == '0') - isNumber = false; - - // String to SizeType conversion - SizeType n = 0; - if (isNumber) { - for (size_t j = 0; j < token->length; j++) { - SizeType m = n * 10 + static_cast<SizeType>(token->name[j] - '0'); - if (m < n) { // overflow detection - isNumber = false; - break; - } - n = m; - } - } - - token->index = isNumber ? n : kPointerInvalidIndex; - token++; - } - - RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer - parseErrorCode_ = kPointerParseErrorNone; - return; - - error: - Allocator::Free(tokens_); - nameBuffer_ = 0; - tokens_ = 0; - tokenCount_ = 0; - parseErrorOffset_ = i; - return; - } - - //! Stringify to string or URI fragment representation. - /*! - \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. - \tparam OutputStream type of output stream. - \param os The output stream. - */ - template<bool uriFragment, typename OutputStream> - bool Stringify(OutputStream& os) const { - RAPIDJSON_ASSERT(IsValid()); - - if (uriFragment) - os.Put('#'); - - for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - os.Put('/'); - for (size_t j = 0; j < t->length; j++) { - Ch c = t->name[j]; - if (c == '~') { - os.Put('~'); - os.Put('0'); - } - else if (c == '/') { - os.Put('~'); - os.Put('1'); - } - else if (uriFragment && NeedPercentEncode(c)) { - // Transcode to UTF8 sequence - GenericStringStream<typename ValueType::EncodingType> source(&t->name[j]); - PercentEncodeStream<OutputStream> target(os); - if (!Transcoder<EncodingType, UTF8<> >().Validate(source, target)) - return false; - j += source.Tell() - 1; - } - else - os.Put(c); - } - } - return true; - } - - //! A helper stream for decoding a percent-encoded sequence into code unit. - /*! - This stream decodes %XY triplet into code unit (0-255). - If it encounters invalid characters, it sets output code unit as 0 and - mark invalid, and to be checked by IsValid(). - */ - class PercentDecodeStream { - public: - typedef typename ValueType::Ch Ch; - - //! Constructor - /*! - \param source Start of the stream - \param end Past-the-end of the stream. - */ - PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} - - Ch Take() { - if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet - valid_ = false; - return 0; - } - src_++; - Ch c = 0; - for (int j = 0; j < 2; j++) { - c = static_cast<Ch>(c << 4); - Ch h = *src_; - if (h >= '0' && h <= '9') c = static_cast<Ch>(c + h - '0'); - else if (h >= 'A' && h <= 'F') c = static_cast<Ch>(c + h - 'A' + 10); - else if (h >= 'a' && h <= 'f') c = static_cast<Ch>(c + h - 'a' + 10); - else { - valid_ = false; - return 0; - } - src_++; - } - return c; - } - - size_t Tell() const { return static_cast<size_t>(src_ - head_); } - bool IsValid() const { return valid_; } - - private: - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. - const Ch* end_; //!< Past-the-end position. - bool valid_; //!< Whether the parsing is valid. - }; - - //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. - template <typename OutputStream> - class PercentEncodeStream { - public: - PercentEncodeStream(OutputStream& os) : os_(os) {} - void Put(char c) { // UTF-8 must be byte - unsigned char u = static_cast<unsigned char>(c); - static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; - os_.Put('%'); - os_.Put(hexDigits[u >> 4]); - os_.Put(hexDigits[u & 15]); - } - private: - OutputStream& os_; - }; - - Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. - Allocator* ownAllocator_; //!< Allocator owned by this Pointer. - Ch* nameBuffer_; //!< A buffer containing all names in tokens. - Token* tokens_; //!< A list of tokens. - size_t tokenCount_; //!< Number of tokens in tokens_. - size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. - PointerParseErrorCode parseErrorCode_; //!< Parsing error code. -}; - -//! GenericPointer for Value (UTF-8, default allocator). -typedef GenericPointer<Value> Pointer; - -//!@name Helper functions for GenericPointer -//@{ - -////////////////////////////////////////////////////////////////////////////// - -template <typename T> -typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, typename T::AllocatorType& a) { - return pointer.Create(root, a); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).Create(root, a); -} - -// No allocator parameter - -template <typename DocumentType> -typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer) { - return pointer.Create(document); -} - -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).Create(document); -} - -////////////////////////////////////////////////////////////////////////////// - -template <typename T> -typename T::ValueType* GetValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); -} - -template <typename T> -const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer<typename T::ValueType>& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer<typename T::ValueType>(source, N - 1).Get(root, unresolvedTokenIndex); -} - -template <typename T, typename CharType, size_t N> -const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer<typename T::ValueType>(source, N - 1).Get(root, unresolvedTokenIndex); -} - -////////////////////////////////////////////////////////////////////////////// - -template <typename T> -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer<typename T::ValueType>& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} - -template <typename T> -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer<typename T::ValueType>& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename T> -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer<typename T::ValueType>& pointer, const std::basic_string<typename T::Ch>& defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} -#endif - -template <typename T, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename T::ValueType&)) -GetValueByPointerWithDefault(T& root, const GenericPointer<typename T::ValueType>& pointer, T2 defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).GetWithDefault(root, defaultValue, a); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).GetWithDefault(root, defaultValue, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename T, typename CharType, size_t N> -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string<typename T::Ch>& defaultValue, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).GetWithDefault(root, defaultValue, a); -} -#endif - -template <typename T, typename CharType, size_t N, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename T::ValueType&)) -GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).GetWithDefault(root, defaultValue, a); -} - -// No allocator parameter - -template <typename DocumentType> -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, const typename DocumentType::ValueType& defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} - -template <typename DocumentType> -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, const typename DocumentType::Ch* defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename DocumentType> -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, const std::basic_string<typename DocumentType::Ch>& defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} -#endif - -template <typename DocumentType, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename DocumentType::ValueType&)) -GetValueByPointerWithDefault(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, T2 defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} - -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).GetWithDefault(document, defaultValue); -} - -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).GetWithDefault(document, defaultValue); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string<typename DocumentType::Ch>& defaultValue) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).GetWithDefault(document, defaultValue); -} -#endif - -template <typename DocumentType, typename CharType, size_t N, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename DocumentType::ValueType&)) -GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).GetWithDefault(document, defaultValue); -} - -////////////////////////////////////////////////////////////////////////////// - -template <typename T> -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -template <typename T> -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -template <typename T> -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename T> -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, const std::basic_string<typename T::Ch>& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} -#endif - -template <typename T, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename T::ValueType&)) -SetValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, T2 value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).Set(root, value, a); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).Set(root, value, a); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).Set(root, value, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename T, typename CharType, size_t N> -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string<typename T::Ch>& value, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).Set(root, value, a); -} -#endif - -template <typename T, typename CharType, size_t N, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename T::ValueType&)) -SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).Set(root, value, a); -} - -// No allocator parameter - -template <typename DocumentType> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, typename DocumentType::ValueType& value) { - return pointer.Set(document, value); -} - -template <typename DocumentType> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, const typename DocumentType::ValueType& value) { - return pointer.Set(document, value); -} - -template <typename DocumentType> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, const typename DocumentType::Ch* value) { - return pointer.Set(document, value); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename DocumentType> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, const std::basic_string<typename DocumentType::Ch>& value) { - return pointer.Set(document, value); -} -#endif - -template <typename DocumentType, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename DocumentType::ValueType&)) -SetValueByPointer(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, T2 value) { - return pointer.Set(document, value); -} - -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).Set(document, value); -} - -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).Set(document, value); -} - -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).Set(document, value); -} - -#if RAPIDJSON_HAS_STDSTRING -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string<typename DocumentType::Ch>& value) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).Set(document, value); -} -#endif - -template <typename DocumentType, typename CharType, size_t N, typename T2> -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr<internal::IsPointer<T2>, internal::IsGenericValue<T2> >), (typename DocumentType::ValueType&)) -SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).Set(document, value); -} - -////////////////////////////////////////////////////////////////////////////// - -template <typename T> -typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Swap(root, value, a); -} - -template <typename T, typename CharType, size_t N> -typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer<typename T::ValueType>(source, N - 1).Swap(root, value, a); -} - -template <typename DocumentType> -typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer<typename DocumentType::ValueType>& pointer, typename DocumentType::ValueType& value) { - return pointer.Swap(document, value); -} - -template <typename DocumentType, typename CharType, size_t N> -typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { - return GenericPointer<typename DocumentType::ValueType>(source, N - 1).Swap(document, value); -} - -////////////////////////////////////////////////////////////////////////////// - -template <typename T> -bool EraseValueByPointer(T& root, const GenericPointer<typename T::ValueType>& pointer) { - return pointer.Erase(root); -} - -template <typename T, typename CharType, size_t N> -bool EraseValueByPointer(T& root, const CharType(&source)[N]) { - return GenericPointer<typename T::ValueType>(source, N - 1).Erase(root); -} - -//@} - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_POINTER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/prettywriter.h b/ext/librethinkdbxx/src/rapidjson/prettywriter.h deleted file mode 100644 index 75dc474f..00000000 --- a/ext/librethinkdbxx/src/rapidjson/prettywriter.h +++ /dev/null @@ -1,249 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_PRETTYWRITER_H_ -#define RAPIDJSON_PRETTYWRITER_H_ - -#include "writer.h" - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Combination of PrettyWriter format flags. -/*! \see PrettyWriter::SetFormatOptions - */ -enum PrettyFormatOptions { - kFormatDefault = 0, //!< Default pretty formatting. - kFormatSingleLineArray = 1 //!< Format arrays on a single line. -}; - -//! Writer with indentation and spacing. -/*! - \tparam OutputStream Type of ouptut os. - \tparam SourceEncoding Encoding of source string. - \tparam TargetEncoding Encoding of output stream. - \tparam StackAllocator Type of allocator for allocating memory of stack. -*/ -template<typename OutputStream, typename SourceEncoding = UTF8<>, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> -class PrettyWriter : public Writer<OutputStream, SourceEncoding, TargetEncoding, StackAllocator, writeFlags> { -public: - typedef Writer<OutputStream, SourceEncoding, TargetEncoding, StackAllocator> Base; - typedef typename Base::Ch Ch; - - //! Constructor - /*! \param os Output stream. - \param allocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of stack. - */ - explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} - - - explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} - - //! Set custom indentation. - /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). - \param indentCharCount Number of indent characters for each indentation level. - \note The default indentation is 4 spaces. - */ - PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { - RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); - indentChar_ = indentChar; - indentCharCount_ = indentCharCount; - return *this; - } - - //! Set pretty writer formatting options. - /*! \param options Formatting options. - */ - PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { - formatOptions_ = options; - return *this; - } - - /*! @name Implementation of Handler - \see Handler - */ - //@{ - - bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } - bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } - bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } - bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } - bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } - bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } - bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } - - bool RawNumber(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - PrettyPrefix(kNumberType); - return Base::WriteString(str, length); - } - - bool String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - PrettyPrefix(kStringType); - return Base::WriteString(str, length); - } - -#if RAPIDJSON_HAS_STDSTRING - bool String(const std::basic_string<Ch>& str) { - return String(str.data(), SizeType(str.size())); - } -#endif - - bool StartObject() { - PrettyPrefix(kObjectType); - new (Base::level_stack_.template Push<typename Base::Level>()) typename Base::Level(false); - return Base::WriteStartObject(); - } - - bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(!Base::level_stack_.template Top<typename Base::Level>()->inArray); - bool empty = Base::level_stack_.template Pop<typename Base::Level>(1)->valueCount == 0; - - if (!empty) { - Base::os_->Put('\n'); - WriteIndent(); - } - bool ret = Base::WriteEndObject(); - (void)ret; - RAPIDJSON_ASSERT(ret == true); - if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); - return true; - } - - bool StartArray() { - PrettyPrefix(kArrayType); - new (Base::level_stack_.template Push<typename Base::Level>()) typename Base::Level(true); - return Base::WriteStartArray(); - } - - bool EndArray(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(Base::level_stack_.template Top<typename Base::Level>()->inArray); - bool empty = Base::level_stack_.template Pop<typename Base::Level>(1)->valueCount == 0; - - if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { - Base::os_->Put('\n'); - WriteIndent(); - } - bool ret = Base::WriteEndArray(); - (void)ret; - RAPIDJSON_ASSERT(ret == true); - if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); - return true; - } - - //@} - - /*! @name Convenience extensions */ - //@{ - - //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - - //@} - - //! Write a raw JSON value. - /*! - For user to write a stringified JSON as a value. - - \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. - \param length Length of the json. - \param type Type of the root of json. - \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. - */ - bool RawValue(const Ch* json, size_t length, Type type) { PrettyPrefix(type); return Base::WriteRawValue(json, length); } - -protected: - void PrettyPrefix(Type type) { - (void)type; - if (Base::level_stack_.GetSize() != 0) { // this value is not at root - typename Base::Level* level = Base::level_stack_.template Top<typename Base::Level>(); - - if (level->inArray) { - if (level->valueCount > 0) { - Base::os_->Put(','); // add comma if it is not the first element in array - if (formatOptions_ & kFormatSingleLineArray) - Base::os_->Put(' '); - } - - if (!(formatOptions_ & kFormatSingleLineArray)) { - Base::os_->Put('\n'); - WriteIndent(); - } - } - else { // in object - if (level->valueCount > 0) { - if (level->valueCount % 2 == 0) { - Base::os_->Put(','); - Base::os_->Put('\n'); - } - else { - Base::os_->Put(':'); - Base::os_->Put(' '); - } - } - else - Base::os_->Put('\n'); - - if (level->valueCount % 2 == 0) - WriteIndent(); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else { - RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. - Base::hasRoot_ = true; - } - } - - void WriteIndent() { - size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, static_cast<typename TargetEncoding::Ch>(indentChar_), count); - } - - Ch indentChar_; - unsigned indentCharCount_; - PrettyFormatOptions formatOptions_; - -private: - // Prohibit copy constructor & assignment operator. - PrettyWriter(const PrettyWriter&); - PrettyWriter& operator=(const PrettyWriter&); -}; - -RAPIDJSON_NAMESPACE_END - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/rapidjson.h b/ext/librethinkdbxx/src/rapidjson/rapidjson.h deleted file mode 100644 index d666f202..00000000 --- a/ext/librethinkdbxx/src/rapidjson/rapidjson.h +++ /dev/null @@ -1,615 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_RAPIDJSON_H_ -#define RAPIDJSON_RAPIDJSON_H_ - -/*!\file rapidjson.h - \brief common definitions and configuration - - \see RAPIDJSON_CONFIG - */ - -/*! \defgroup RAPIDJSON_CONFIG RapidJSON configuration - \brief Configuration macros for library features - - Some RapidJSON features are configurable to adapt the library to a wide - variety of platforms, environments and usage scenarios. Most of the - features can be configured in terms of overriden or predefined - preprocessor macros at compile-time. - - Some additional customization is available in the \ref RAPIDJSON_ERRORS APIs. - - \note These macros should be given on the compiler command-line - (where applicable) to avoid inconsistent values when compiling - different translation units of a single application. - */ - -#include <cstdlib> // malloc(), realloc(), free(), size_t -#include <cstring> // memset(), memcpy(), memmove(), memcmp() - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_VERSION_STRING -// -// ALWAYS synchronize the following 3 macros with corresponding variables in /CMakeLists.txt. -// - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -// token stringification -#define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) -#define RAPIDJSON_DO_STRINGIFY(x) #x -//!@endcond - -/*! \def RAPIDJSON_MAJOR_VERSION - \ingroup RAPIDJSON_CONFIG - \brief Major version of RapidJSON in integer. -*/ -/*! \def RAPIDJSON_MINOR_VERSION - \ingroup RAPIDJSON_CONFIG - \brief Minor version of RapidJSON in integer. -*/ -/*! \def RAPIDJSON_PATCH_VERSION - \ingroup RAPIDJSON_CONFIG - \brief Patch version of RapidJSON in integer. -*/ -/*! \def RAPIDJSON_VERSION_STRING - \ingroup RAPIDJSON_CONFIG - \brief Version of RapidJSON in "<major>.<minor>.<patch>" string format. -*/ -#define RAPIDJSON_MAJOR_VERSION 1 -#define RAPIDJSON_MINOR_VERSION 0 -#define RAPIDJSON_PATCH_VERSION 2 -#define RAPIDJSON_VERSION_STRING \ - RAPIDJSON_STRINGIFY(RAPIDJSON_MAJOR_VERSION.RAPIDJSON_MINOR_VERSION.RAPIDJSON_PATCH_VERSION) - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_NAMESPACE_(BEGIN|END) -/*! \def RAPIDJSON_NAMESPACE - \ingroup RAPIDJSON_CONFIG - \brief provide custom rapidjson namespace - - In order to avoid symbol clashes and/or "One Definition Rule" errors - between multiple inclusions of (different versions of) RapidJSON in - a single binary, users can customize the name of the main RapidJSON - namespace. - - In case of a single nesting level, defining \c RAPIDJSON_NAMESPACE - to a custom name (e.g. \c MyRapidJSON) is sufficient. If multiple - levels are needed, both \ref RAPIDJSON_NAMESPACE_BEGIN and \ref - RAPIDJSON_NAMESPACE_END need to be defined as well: - - \code - // in some .cpp file - #define RAPIDJSON_NAMESPACE my::rapidjson - #define RAPIDJSON_NAMESPACE_BEGIN namespace my { namespace rapidjson { - #define RAPIDJSON_NAMESPACE_END } } - #include "rapidjson/..." - \endcode - - \see rapidjson - */ -/*! \def RAPIDJSON_NAMESPACE_BEGIN - \ingroup RAPIDJSON_CONFIG - \brief provide custom rapidjson namespace (opening expression) - \see RAPIDJSON_NAMESPACE -*/ -/*! \def RAPIDJSON_NAMESPACE_END - \ingroup RAPIDJSON_CONFIG - \brief provide custom rapidjson namespace (closing expression) - \see RAPIDJSON_NAMESPACE -*/ -#ifndef RAPIDJSON_NAMESPACE -#define RAPIDJSON_NAMESPACE rapidjson -#endif -#ifndef RAPIDJSON_NAMESPACE_BEGIN -#define RAPIDJSON_NAMESPACE_BEGIN namespace RAPIDJSON_NAMESPACE { -#endif -#ifndef RAPIDJSON_NAMESPACE_END -#define RAPIDJSON_NAMESPACE_END } -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_HAS_STDSTRING - -#ifndef RAPIDJSON_HAS_STDSTRING -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation -#else -#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default -#endif -/*! \def RAPIDJSON_HAS_STDSTRING - \ingroup RAPIDJSON_CONFIG - \brief Enable RapidJSON support for \c std::string - - By defining this preprocessor symbol to \c 1, several convenience functions for using - \ref rapidjson::GenericValue with \c std::string are enabled, especially - for construction and comparison. - - \hideinitializer -*/ -#endif // !defined(RAPIDJSON_HAS_STDSTRING) - -#if RAPIDJSON_HAS_STDSTRING -#include <string> -#endif // RAPIDJSON_HAS_STDSTRING - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_NO_INT64DEFINE - -/*! \def RAPIDJSON_NO_INT64DEFINE - \ingroup RAPIDJSON_CONFIG - \brief Use external 64-bit integer types. - - RapidJSON requires the 64-bit integer types \c int64_t and \c uint64_t types - to be available at global scope. - - If users have their own definition, define RAPIDJSON_NO_INT64DEFINE to - prevent RapidJSON from defining its own types. -*/ -#ifndef RAPIDJSON_NO_INT64DEFINE -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 -#include "msinttypes/stdint.h" -#include "msinttypes/inttypes.h" -#else -// Other compilers should have this. -#include <stdint.h> -#include <inttypes.h> -#endif -//!@endcond -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_NO_INT64DEFINE -#endif -#endif // RAPIDJSON_NO_INT64TYPEDEF - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_FORCEINLINE - -#ifndef RAPIDJSON_FORCEINLINE -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && defined(NDEBUG) -#define RAPIDJSON_FORCEINLINE __forceinline -#elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) -#define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) -#else -#define RAPIDJSON_FORCEINLINE -#endif -//!@endcond -#endif // RAPIDJSON_FORCEINLINE - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ENDIAN -#define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine -#define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine - -//! Endianness of the machine. -/*! - \def RAPIDJSON_ENDIAN - \ingroup RAPIDJSON_CONFIG - - GCC 4.6 provided macro for detecting endianness of the target machine. But other - compilers may not have this. User can define RAPIDJSON_ENDIAN to either - \ref RAPIDJSON_LITTLEENDIAN or \ref RAPIDJSON_BIGENDIAN. - - Default detection implemented with reference to - \li https://gcc.gnu.org/onlinedocs/gcc-4.6.0/cpp/Common-Predefined-Macros.html - \li http://www.boost.org/doc/libs/1_42_0/boost/detail/endian.hpp -*/ -#ifndef RAPIDJSON_ENDIAN -// Detect with GCC 4.6's macro -# ifdef __BYTE_ORDER__ -# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -# else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. -# endif // __BYTE_ORDER__ -// Detect with GLIBC's endian.h -# elif defined(__GLIBC__) -# include <endian.h> -# if (__BYTE_ORDER == __LITTLE_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif (__BYTE_ORDER == __BIG_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -# else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. -# endif // __GLIBC__ -// Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro -# elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -// Detect with architecture macros -# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) -# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN -# elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif defined(_MSC_VER) && defined(_M_ARM) -# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif defined(RAPIDJSON_DOXYGEN_RUNNING) -# define RAPIDJSON_ENDIAN -# else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. -# endif -#endif // RAPIDJSON_ENDIAN - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_64BIT - -//! Whether using 64-bit architecture -#ifndef RAPIDJSON_64BIT -#if defined(__LP64__) || defined(_WIN64) || defined(__EMSCRIPTEN__) -#define RAPIDJSON_64BIT 1 -#else -#define RAPIDJSON_64BIT 0 -#endif -#endif // RAPIDJSON_64BIT - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ALIGN - -//! Data alignment of the machine. -/*! \ingroup RAPIDJSON_CONFIG - \param x pointer to align - - Some machines require strict data alignment. Currently the default uses 4 bytes - alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. - User can customize by defining the RAPIDJSON_ALIGN function macro. -*/ -#ifndef RAPIDJSON_ALIGN -#if RAPIDJSON_64BIT == 1 -#define RAPIDJSON_ALIGN(x) (((x) + static_cast<uint64_t>(7u)) & ~static_cast<uint64_t>(7u)) -#else -#define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) -#endif -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_UINT64_C2 - -//! Construct a 64-bit literal by a pair of 32-bit integer. -/*! - 64-bit literal with or without ULL suffix is prone to compiler warnings. - UINT64_C() is C macro which cause compilation problems. - Use this macro to define 64-bit constants by a pair of 32-bit integer. -*/ -#ifndef RAPIDJSON_UINT64_C2 -#define RAPIDJSON_UINT64_C2(high32, low32) ((static_cast<uint64_t>(high32) << 32) | static_cast<uint64_t>(low32)) -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_48BITPOINTER_OPTIMIZATION - -//! Use only lower 48-bit address for some pointers. -/*! - \ingroup RAPIDJSON_CONFIG - - This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. - The higher 16-bit can be used for storing other data. - \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. -*/ -#ifndef RAPIDJSON_48BITPOINTER_OPTIMIZATION -#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) -#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 -#else -#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 -#endif -#endif // RAPIDJSON_48BITPOINTER_OPTIMIZATION - -#if RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 -#if RAPIDJSON_64BIT != 1 -#error RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when RAPIDJSON_64BIT=1 -#endif -#define RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast<type *>((reinterpret_cast<uintptr_t>(p) & static_cast<uintptr_t>(RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast<uintptr_t>(reinterpret_cast<const void*>(x)))) -#define RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast<type *>(reinterpret_cast<uintptr_t>(p) & static_cast<uintptr_t>(RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) -#else -#define RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) -#define RAPIDJSON_GETPOINTER(type, p) (p) -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD - -/*! \def RAPIDJSON_SIMD - \ingroup RAPIDJSON_CONFIG - \brief Enable SSE2/SSE4.2 optimization. - - RapidJSON supports optimized implementations for some parsing operations - based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible - processors. - - To enable these optimizations, two different symbols can be defined; - \code - // Enable SSE2 optimization. - #define RAPIDJSON_SSE2 - - // Enable SSE4.2 optimization. - #define RAPIDJSON_SSE42 - \endcode - - \c RAPIDJSON_SSE42 takes precedence, if both are defined. - - If any of these symbols is defined, RapidJSON defines the macro - \c RAPIDJSON_SIMD to indicate the availability of the optimized code. -*/ -#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ - || defined(RAPIDJSON_DOXYGEN_RUNNING) -#define RAPIDJSON_SIMD -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_NO_SIZETYPEDEFINE - -#ifndef RAPIDJSON_NO_SIZETYPEDEFINE -/*! \def RAPIDJSON_NO_SIZETYPEDEFINE - \ingroup RAPIDJSON_CONFIG - \brief User-provided \c SizeType definition. - - In order to avoid using 32-bit size types for indexing strings and arrays, - define this preprocessor symbol and provide the type rapidjson::SizeType - before including RapidJSON: - \code - #define RAPIDJSON_NO_SIZETYPEDEFINE - namespace rapidjson { typedef ::std::size_t SizeType; } - #include "rapidjson/..." - \endcode - - \see rapidjson::SizeType -*/ -#ifdef RAPIDJSON_DOXYGEN_RUNNING -#define RAPIDJSON_NO_SIZETYPEDEFINE -#endif -RAPIDJSON_NAMESPACE_BEGIN -//! Size type (for string lengths, array sizes, etc.) -/*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms, - instead of using \c size_t. Users may override the SizeType by defining - \ref RAPIDJSON_NO_SIZETYPEDEFINE. -*/ -typedef unsigned SizeType; -RAPIDJSON_NAMESPACE_END -#endif - -// always import std::size_t to rapidjson namespace -RAPIDJSON_NAMESPACE_BEGIN -using std::size_t; -RAPIDJSON_NAMESPACE_END - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_ASSERT - -//! Assertion. -/*! \ingroup RAPIDJSON_CONFIG - By default, rapidjson uses C \c assert() for internal assertions. - User can override it by defining RAPIDJSON_ASSERT(x) macro. - - \note Parsing errors are handled and can be customized by the - \ref RAPIDJSON_ERRORS APIs. -*/ -#ifndef RAPIDJSON_ASSERT -#include <cassert> -#define RAPIDJSON_ASSERT(x) assert(x) -#endif // RAPIDJSON_ASSERT - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_STATIC_ASSERT - -// Adopt from boost -#ifndef RAPIDJSON_STATIC_ASSERT -#ifndef __clang__ -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#endif -RAPIDJSON_NAMESPACE_BEGIN -template <bool x> struct STATIC_ASSERTION_FAILURE; -template <> struct STATIC_ASSERTION_FAILURE<true> { enum { value = 1 }; }; -template<int x> struct StaticAssertTest {}; -RAPIDJSON_NAMESPACE_END - -#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) -#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) -#define RAPIDJSON_DO_JOIN2(X, Y) X##Y - -#if defined(__GNUC__) -#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) -#else -#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE -#endif -#ifndef __clang__ -//!@endcond -#endif - -/*! \def RAPIDJSON_STATIC_ASSERT - \brief (Internal) macro to check for conditions at compile-time - \param x compile-time condition - \hideinitializer - */ -#define RAPIDJSON_STATIC_ASSERT(x) \ - typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ - sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE<bool(x) >)> \ - RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE -#endif - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY - -//! Compiler branching hint for expression with high probability to be true. -/*! - \ingroup RAPIDJSON_CONFIG - \param x Boolean expression likely to be true. -*/ -#ifndef RAPIDJSON_LIKELY -#if defined(__GNUC__) || defined(__clang__) -#define RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) -#else -#define RAPIDJSON_LIKELY(x) (x) -#endif -#endif - -//! Compiler branching hint for expression with low probability to be true. -/*! - \ingroup RAPIDJSON_CONFIG - \param x Boolean expression unlikely to be true. -*/ -#ifndef RAPIDJSON_UNLIKELY -#if defined(__GNUC__) || defined(__clang__) -#define RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define RAPIDJSON_UNLIKELY(x) (x) -#endif -#endif - -/////////////////////////////////////////////////////////////////////////////// -// Helpers - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN - -#define RAPIDJSON_MULTILINEMACRO_BEGIN do { -#define RAPIDJSON_MULTILINEMACRO_END \ -} while((void)0, 0) - -// adopted from Boost -#define RAPIDJSON_VERSION_CODE(x,y,z) \ - (((x)*100000) + ((y)*100) + (z)) - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF - -#if defined(__GNUC__) -#define RAPIDJSON_GNUC \ - RAPIDJSON_VERSION_CODE(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) -#endif - -#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,2,0)) - -#define RAPIDJSON_PRAGMA(x) _Pragma(RAPIDJSON_STRINGIFY(x)) -#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(GCC diagnostic x) -#define RAPIDJSON_DIAG_OFF(x) \ - RAPIDJSON_DIAG_PRAGMA(ignored RAPIDJSON_STRINGIFY(RAPIDJSON_JOIN(-W,x))) - -// push/pop support in Clang and GCC>=4.6 -#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) -#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) -#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) -#else // GCC >= 4.2, < 4.6 -#define RAPIDJSON_DIAG_PUSH /* ignored */ -#define RAPIDJSON_DIAG_POP /* ignored */ -#endif - -#elif defined(_MSC_VER) - -// pragma (MSVC specific) -#define RAPIDJSON_PRAGMA(x) __pragma(x) -#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(warning(x)) - -#define RAPIDJSON_DIAG_OFF(x) RAPIDJSON_DIAG_PRAGMA(disable: x) -#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) -#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) - -#else - -#define RAPIDJSON_DIAG_OFF(x) /* ignored */ -#define RAPIDJSON_DIAG_PUSH /* ignored */ -#define RAPIDJSON_DIAG_POP /* ignored */ - -#endif // RAPIDJSON_DIAG_* - -/////////////////////////////////////////////////////////////////////////////// -// C++11 features - -#ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS -#if defined(__clang__) -#if __has_feature(cxx_rvalue_references) && \ - (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 -#else -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 -#endif -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ - (defined(_MSC_VER) && _MSC_VER >= 1600) - -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 -#else -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 -#endif -#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - -#ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT -#if defined(__clang__) -#define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) -// (defined(_MSC_VER) && _MSC_VER >= ????) // not yet supported -#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 -#else -#define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 -#endif -#endif -#if RAPIDJSON_HAS_CXX11_NOEXCEPT -#define RAPIDJSON_NOEXCEPT noexcept -#else -#define RAPIDJSON_NOEXCEPT /* noexcept */ -#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT - -// no automatic detection, yet -#ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS -#define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 -#endif - -#ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR -#if defined(__clang__) -#define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ - (defined(_MSC_VER) && _MSC_VER >= 1700) -#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 -#else -#define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 -#endif -#endif // RAPIDJSON_HAS_CXX11_RANGE_FOR - -//!@endcond - -/////////////////////////////////////////////////////////////////////////////// -// new/delete - -#ifndef RAPIDJSON_NEW -///! customization point for global \c new -#define RAPIDJSON_NEW(x) new x -#endif -#ifndef RAPIDJSON_DELETE -///! customization point for global \c delete -#define RAPIDJSON_DELETE(x) delete x -#endif - -/////////////////////////////////////////////////////////////////////////////// -// Type - -/*! \namespace rapidjson - \brief main RapidJSON namespace - \see RAPIDJSON_NAMESPACE -*/ -RAPIDJSON_NAMESPACE_BEGIN - -//! Type of JSON value -enum Type { - kNullType = 0, //!< null - kFalseType = 1, //!< false - kTrueType = 2, //!< true - kObjectType = 3, //!< object - kArrayType = 4, //!< array - kStringType = 5, //!< string - kNumberType = 6 //!< number -}; - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/reader.h b/ext/librethinkdbxx/src/rapidjson/reader.h deleted file mode 100644 index 19f8849b..00000000 --- a/ext/librethinkdbxx/src/rapidjson/reader.h +++ /dev/null @@ -1,1879 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_READER_H_ -#define RAPIDJSON_READER_H_ - -/*! \file reader.h */ - -#include "allocators.h" -#include "stream.h" -#include "encodedstream.h" -#include "internal/meta.h" -#include "internal/stack.h" -#include "internal/strtod.h" -#include <limits> - -#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) -#include <intrin.h> -#pragma intrinsic(_BitScanForward) -#endif -#ifdef RAPIDJSON_SSE42 -#include <nmmintrin.h> -#elif defined(RAPIDJSON_SSE2) -#include <emmintrin.h> -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -RAPIDJSON_DIAG_OFF(4702) // unreachable code -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(old-style-cast) -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(switch-enum) -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define RAPIDJSON_NOTHING /* deliberately empty */ -#ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN -#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ - RAPIDJSON_MULTILINEMACRO_BEGIN \ - if (RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ - RAPIDJSON_MULTILINEMACRO_END -#endif -#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(RAPIDJSON_NOTHING) -//!@endcond - -/*! \def RAPIDJSON_PARSE_ERROR_NORETURN - \ingroup RAPIDJSON_ERRORS - \brief Macro to indicate a parse error. - \param parseErrorCode \ref rapidjson::ParseErrorCode of the error - \param offset position of the error in JSON input (\c size_t) - - This macros can be used as a customization point for the internal - error handling mechanism of RapidJSON. - - A common usage model is to throw an exception instead of requiring the - caller to explicitly check the \ref rapidjson::GenericReader::Parse's - return value: - - \code - #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ - throw ParseException(parseErrorCode, #parseErrorCode, offset) - - #include <stdexcept> // std::runtime_error - #include "rapidjson/error/error.h" // rapidjson::ParseResult - - struct ParseException : std::runtime_error, rapidjson::ParseResult { - ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) - : std::runtime_error(msg), ParseResult(code, offset) {} - }; - - #include "rapidjson/reader.h" - \endcode - - \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse - */ -#ifndef RAPIDJSON_PARSE_ERROR_NORETURN -#define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ - RAPIDJSON_MULTILINEMACRO_BEGIN \ - RAPIDJSON_ASSERT(!HasParseError()); /* Error can only be assigned once */ \ - SetParseError(parseErrorCode, offset); \ - RAPIDJSON_MULTILINEMACRO_END -#endif - -/*! \def RAPIDJSON_PARSE_ERROR - \ingroup RAPIDJSON_ERRORS - \brief (Internal) macro to indicate and handle a parse error. - \param parseErrorCode \ref rapidjson::ParseErrorCode of the error - \param offset position of the error in JSON input (\c size_t) - - Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. - - \see RAPIDJSON_PARSE_ERROR_NORETURN - \hideinitializer - */ -#ifndef RAPIDJSON_PARSE_ERROR -#define RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ - RAPIDJSON_MULTILINEMACRO_BEGIN \ - RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset); \ - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; \ - RAPIDJSON_MULTILINEMACRO_END -#endif - -#include "error/error.h" // ParseErrorCode, ParseResult - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// ParseFlag - -/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS - \ingroup RAPIDJSON_CONFIG - \brief User-defined kParseDefaultFlags definition. - - User can define this as any \c ParseFlag combinations. -*/ -#ifndef RAPIDJSON_PARSE_DEFAULT_FLAGS -#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseNoFlags -#endif - -//! Combination of parseFlags -/*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream - */ -enum ParseFlag { - kParseNoFlags = 0, //!< No flags are set. - kParseInsituFlag = 1, //!< In-situ(destructive) parsing. - kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. - kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. - kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. - kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). - kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. - kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. - kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. - kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. - kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS -}; - -/////////////////////////////////////////////////////////////////////////////// -// Handler - -/*! \class rapidjson::Handler - \brief Concept for receiving events from GenericReader upon parsing. - The functions return true if no error occurs. If they return false, - the event publisher should terminate the process. -\code -concept Handler { - typename Ch; - - bool Null(); - bool Bool(bool b); - bool Int(int i); - bool Uint(unsigned i); - bool Int64(int64_t i); - bool Uint64(uint64_t i); - bool Double(double d); - /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool RawNumber(const Ch* str, SizeType length, bool copy); - bool String(const Ch* str, SizeType length, bool copy); - bool StartObject(); - bool Key(const Ch* str, SizeType length, bool copy); - bool EndObject(SizeType memberCount); - bool StartArray(); - bool EndArray(SizeType elementCount); -}; -\endcode -*/ -/////////////////////////////////////////////////////////////////////////////// -// BaseReaderHandler - -//! Default implementation of Handler. -/*! This can be used as base class of any reader handler. - \note implements Handler concept -*/ -template<typename Encoding = UTF8<>, typename Derived = void> -struct BaseReaderHandler { - typedef typename Encoding::Ch Ch; - - typedef typename internal::SelectIf<internal::IsSame<Derived, void>, BaseReaderHandler, Derived>::Type Override; - - bool Default() { return true; } - bool Null() { return static_cast<Override&>(*this).Default(); } - bool Bool(bool) { return static_cast<Override&>(*this).Default(); } - bool Int(int) { return static_cast<Override&>(*this).Default(); } - bool Uint(unsigned) { return static_cast<Override&>(*this).Default(); } - bool Int64(int64_t) { return static_cast<Override&>(*this).Default(); } - bool Uint64(uint64_t) { return static_cast<Override&>(*this).Default(); } - bool Double(double) { return static_cast<Override&>(*this).Default(); } - /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast<Override&>(*this).String(str, len, copy); } - bool String(const Ch*, SizeType, bool) { return static_cast<Override&>(*this).Default(); } - bool StartObject() { return static_cast<Override&>(*this).Default(); } - bool Key(const Ch* str, SizeType len, bool copy) { return static_cast<Override&>(*this).String(str, len, copy); } - bool EndObject(SizeType) { return static_cast<Override&>(*this).Default(); } - bool StartArray() { return static_cast<Override&>(*this).Default(); } - bool EndArray(SizeType) { return static_cast<Override&>(*this).Default(); } -}; - -/////////////////////////////////////////////////////////////////////////////// -// StreamLocalCopy - -namespace internal { - -template<typename Stream, int = StreamTraits<Stream>::copyOptimization> -class StreamLocalCopy; - -//! Do copy optimization. -template<typename Stream> -class StreamLocalCopy<Stream, 1> { -public: - StreamLocalCopy(Stream& original) : s(original), original_(original) {} - ~StreamLocalCopy() { original_ = s; } - - Stream s; - -private: - StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; - - Stream& original_; -}; - -//! Keep reference. -template<typename Stream> -class StreamLocalCopy<Stream, 0> { -public: - StreamLocalCopy(Stream& original) : s(original) {} - - Stream& s; - -private: - StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; -}; - -} // namespace internal - -/////////////////////////////////////////////////////////////////////////////// -// SkipWhitespace - -//! Skip the JSON white spaces in a stream. -/*! \param is A input stream for skipping white spaces. - \note This function has SSE2/SSE4.2 specialization. -*/ -template<typename InputStream> -void SkipWhitespace(InputStream& is) { - internal::StreamLocalCopy<InputStream> copy(is); - InputStream& s(copy.s); - - typename InputStream::Ch c; - while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') - s.Take(); -} - -inline const char* SkipWhitespace(const char* p, const char* end) { - while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - return p; -} - -#ifdef RAPIDJSON_SSE42 -//! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - // Fast return for single non-whitespace - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast<const char*>((reinterpret_cast<size_t>(p) + 15) & static_cast<size_t>(~15)); - while (p != nextAligned) - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // The rest of string using SIMD - static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespace[0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast<const __m128i *>(p)); - const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } - } -} - -inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { - // Fast return for single non-whitespace - if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - else - return p; - - // The middle of string using SIMD - static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespace[0])); - - for (; p <= end - 16; p += 16) { - const __m128i s = _mm_loadu_si128(reinterpret_cast<const __m128i *>(p)); - const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } - } - - return SkipWhitespace(p, end); -} - -#elif defined(RAPIDJSON_SSE2) - -//! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - // Fast return for single non-whitespace - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast<const char*>((reinterpret_cast<size_t>(p) + 15) & static_cast<size_t>(~15)); - while (p != nextAligned) - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // The rest of string - #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } - static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; - #undef C16 - - const __m128i w0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[0][0])); - const __m128i w1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[1][0])); - const __m128i w2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[2][0])); - const __m128i w3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[3][0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast<const __m128i *>(p)); - __m128i x = _mm_cmpeq_epi8(s, w0); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = static_cast<unsigned short>(~_mm_movemask_epi8(x)); - if (r != 0) { // some of characters may be non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } - } -} - -inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { - // Fast return for single non-whitespace - if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - else - return p; - - // The rest of string - #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } - static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; - #undef C16 - - const __m128i w0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[0][0])); - const __m128i w1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[1][0])); - const __m128i w2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[2][0])); - const __m128i w3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&whitespaces[3][0])); - - for (; p <= end - 16; p += 16) { - const __m128i s = _mm_loadu_si128(reinterpret_cast<const __m128i *>(p)); - __m128i x = _mm_cmpeq_epi8(s, w0); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = static_cast<unsigned short>(~_mm_movemask_epi8(x)); - if (r != 0) { // some of characters may be non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } - } - - return SkipWhitespace(p, end); -} - -#endif // RAPIDJSON_SSE2 - -#ifdef RAPIDJSON_SIMD -//! Template function specialization for InsituStringStream -template<> inline void SkipWhitespace(InsituStringStream& is) { - is.src_ = const_cast<char*>(SkipWhitespace_SIMD(is.src_)); -} - -//! Template function specialization for StringStream -template<> inline void SkipWhitespace(StringStream& is) { - is.src_ = SkipWhitespace_SIMD(is.src_); -} - -template<> inline void SkipWhitespace(EncodedInputStream<UTF8<>, MemoryStream>& is) { - is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); -} -#endif // RAPIDJSON_SIMD - -/////////////////////////////////////////////////////////////////////////////// -// GenericReader - -//! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. -/*! GenericReader parses JSON text from a stream, and send events synchronously to an - object implementing Handler concept. - - It needs to allocate a stack for storing a single decoded string during - non-destructive parsing. - - For in-situ parsing, the decoded string is directly written to the source - text string, no temporary buffer is required. - - A GenericReader object can be reused for parsing multiple JSON text. - - \tparam SourceEncoding Encoding of the input stream. - \tparam TargetEncoding Encoding of the parse output. - \tparam StackAllocator Allocator type for stack. -*/ -template <typename SourceEncoding, typename TargetEncoding, typename StackAllocator = CrtAllocator> -class GenericReader { -public: - typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type - - //! Constructor. - /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) - \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) - */ - GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} - - //! Parse JSON text. - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept. - \tparam Handler Type of handler, implementing Handler concept. - \param is Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template <unsigned parseFlags, typename InputStream, typename Handler> - ParseResult Parse(InputStream& is, Handler& handler) { - if (parseFlags & kParseIterativeFlag) - return IterativeParse<parseFlags>(is, handler); - - parseResult_.Clear(); - - ClearStackOnExit scope(*this); - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - - if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - } - else { - ParseValue<parseFlags>(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - - if (!(parseFlags & kParseStopWhenDoneFlag)) { - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - - if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - } - } - } - - return parseResult_; - } - - //! Parse JSON text (with \ref kParseDefaultFlags) - /*! \tparam InputStream Type of input stream, implementing Stream concept - \tparam Handler Type of handler, implementing Handler concept. - \param is Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template <typename InputStream, typename Handler> - ParseResult Parse(InputStream& is, Handler& handler) { - return Parse<kParseDefaultFlags>(is, handler); - } - - //! Whether a parse error has occured in the last parsing. - bool HasParseError() const { return parseResult_.IsError(); } - - //! Get the \ref ParseErrorCode of last parsing. - ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } - - //! Get the position of last parsing error in input, 0 otherwise. - size_t GetErrorOffset() const { return parseResult_.Offset(); } - -protected: - void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } - -private: - // Prohibit copy constructor & assignment operator. - GenericReader(const GenericReader&); - GenericReader& operator=(const GenericReader&); - - void ClearStack() { stack_.Clear(); } - - // clear stack on any exit from ParseStream, e.g. due to exception - struct ClearStackOnExit { - explicit ClearStackOnExit(GenericReader& r) : r_(r) {} - ~ClearStackOnExit() { r_.ClearStack(); } - private: - GenericReader& r_; - ClearStackOnExit(const ClearStackOnExit&); - ClearStackOnExit& operator=(const ClearStackOnExit&); - }; - - template<unsigned parseFlags, typename InputStream> - void SkipWhitespaceAndComments(InputStream& is) { - SkipWhitespace(is); - - if (parseFlags & kParseCommentsFlag) { - while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { - if (Consume(is, '*')) { - while (true) { - if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) - RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - else if (Consume(is, '*')) { - if (Consume(is, '/')) - break; - } - else - is.Take(); - } - } - else if (RAPIDJSON_LIKELY(Consume(is, '/'))) - while (is.Peek() != '\0' && is.Take() != '\n'); - else - RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - - SkipWhitespace(is); - } - } - } - - // Parse object: { string : value, ... } - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseObject(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == '{'); - is.Take(); // Skip '{' - - if (RAPIDJSON_UNLIKELY(!handler.StartObject())) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - if (Consume(is, '}')) { - if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } - - for (SizeType memberCount = 0;;) { - if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); - - ParseString<parseFlags>(is, handler, true); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - ParseValue<parseFlags>(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - ++memberCount; - - switch (is.Peek()) { - case ',': - is.Take(); - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - break; - case '}': - is.Take(); - if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - default: - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy - } - - if (parseFlags & kParseTrailingCommasFlag) { - if (is.Peek() == '}') { - if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - is.Take(); - return; - } - } - } - } - - // Parse array: [ value, ... ] - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseArray(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == '['); - is.Take(); // Skip '[' - - if (RAPIDJSON_UNLIKELY(!handler.StartArray())) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - if (Consume(is, ']')) { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } - - for (SizeType elementCount = 0;;) { - ParseValue<parseFlags>(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - ++elementCount; - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - - if (Consume(is, ',')) { - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - } - else if (Consume(is, ']')) { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - - if (parseFlags & kParseTrailingCommasFlag) { - if (is.Peek() == ']') { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - is.Take(); - return; - } - } - } - } - - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseNull(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 'n'); - is.Take(); - - if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { - if (RAPIDJSON_UNLIKELY(!handler.Null())) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); - } - - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseTrue(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 't'); - is.Take(); - - if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { - if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); - } - - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseFalse(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 'f'); - is.Take(); - - if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { - if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); - } - - template<typename InputStream> - RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { - if (RAPIDJSON_LIKELY(is.Peek() == expect)) { - is.Take(); - return true; - } - else - return false; - } - - // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). - template<typename InputStream> - unsigned ParseHex4(InputStream& is, size_t escapeOffset) { - unsigned codepoint = 0; - for (int i = 0; i < 4; i++) { - Ch c = is.Peek(); - codepoint <<= 4; - codepoint += static_cast<unsigned>(c); - if (c >= '0' && c <= '9') - codepoint -= '0'; - else if (c >= 'A' && c <= 'F') - codepoint -= 'A' - 10; - else if (c >= 'a' && c <= 'f') - codepoint -= 'a' - 10; - else { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); - } - is.Take(); - } - return codepoint; - } - - template <typename CharType> - class StackStream { - public: - typedef CharType Ch; - - StackStream(internal::Stack<StackAllocator>& stack) : stack_(stack), length_(0) {} - RAPIDJSON_FORCEINLINE void Put(Ch c) { - *stack_.template Push<Ch>() = c; - ++length_; - } - - RAPIDJSON_FORCEINLINE void* Push(SizeType count) { - length_ += count; - return stack_.template Push<Ch>(count); - } - - size_t Length() const { return length_; } - - Ch* Pop() { - return stack_.template Pop<Ch>(length_); - } - - private: - StackStream(const StackStream&); - StackStream& operator=(const StackStream&); - - internal::Stack<StackAllocator>& stack_; - SizeType length_; - }; - - // Parse string and generate String event. Different code paths for kParseInsituFlag. - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseString(InputStream& is, Handler& handler, bool isKey = false) { - internal::StreamLocalCopy<InputStream> copy(is); - InputStream& s(copy.s); - - RAPIDJSON_ASSERT(s.Peek() == '\"'); - s.Take(); // Skip '\"' - - bool success = false; - if (parseFlags & kParseInsituFlag) { - typename InputStream::Ch *head = s.PutBegin(); - ParseStringToStream<parseFlags, SourceEncoding, SourceEncoding>(s, s); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - size_t length = s.PutEnd(head) - 1; - RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - const typename TargetEncoding::Ch* const str = reinterpret_cast<typename TargetEncoding::Ch*>(head); - success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); - } - else { - StackStream<typename TargetEncoding::Ch> stackStream(stack_); - ParseStringToStream<parseFlags, SourceEncoding, TargetEncoding>(s, stackStream); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SizeType length = static_cast<SizeType>(stackStream.Length()) - 1; - const typename TargetEncoding::Ch* const str = stackStream.Pop(); - success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); - } - if (RAPIDJSON_UNLIKELY(!success)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); - } - - // Parse string to an output is - // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. - template<unsigned parseFlags, typename SEncoding, typename TEncoding, typename InputStream, typename OutputStream> - RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', - Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, - 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, - 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 - }; -#undef Z16 -//!@endcond - - for (;;) { - // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. - if (!(parseFlags & kParseValidateEncodingFlag)) - ScanCopyUnescapedString(is, os); - - Ch c = is.Peek(); - if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape - size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset - is.Take(); - Ch e = is.Peek(); - if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast<unsigned char>(e)])) { - is.Take(); - os.Put(static_cast<typename TEncoding::Ch>(escape[static_cast<unsigned char>(e)])); - } - else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode - is.Take(); - unsigned codepoint = ParseHex4(is, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { - // Handle UTF-16 surrogate pair - if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - unsigned codepoint2 = ParseHex4(is, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; - } - TEncoding::Encode(os, codepoint); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); - } - else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote - is.Take(); - os.Put('\0'); // null-terminate the string - return; - } - else if (RAPIDJSON_UNLIKELY(static_cast<unsigned>(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - if (c == '\0') - RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); - else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell()); - } - else { - size_t offset = is.Tell(); - if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? - !Transcoder<SEncoding, TEncoding>::Validate(is, os) : - !Transcoder<SEncoding, TEncoding>::Transcode(is, os)))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); - } - } - } - - template<typename InputStream, typename OutputStream> - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { - // Do nothing for generic version - } - -#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) - // StringStream -> StackStream<char> - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream<char>& os) { - const char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast<const char*>((reinterpret_cast<size_t>(p) + 15) & static_cast<size_t>(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast<unsigned>(*p) < 0x20)) { - is.src_ = p; - return; - } - else - os.Put(*p++); - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&space[0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast<const __m128i *>(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast<unsigned short>(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - SizeType length; - #ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; - #else - length = static_cast<SizeType>(__builtin_ffs(r) - 1); - #endif - char* q = reinterpret_cast<char*>(os.Push(length)); - for (size_t i = 0; i < length; i++) - q[i] = p[i]; - - p += length; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); - } - - is.src_ = p; - } - - // InsituStringStream -> InsituStringStream - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { - RAPIDJSON_ASSERT(&is == &os); - (void)os; - - if (is.src_ == is.dst_) { - SkipUnescapedString(is); - return; - } - - char* p = is.src_; - char *q = is.dst_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast<const char*>((reinterpret_cast<size_t>(p) + 15) & static_cast<size_t>(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast<unsigned>(*p) < 0x20)) { - is.src_ = p; - is.dst_ = q; - return; - } - else - *q++ = *p++; - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&space[0])); - - for (;; p += 16, q += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast<const __m128i *>(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast<unsigned short>(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - size_t length; -#ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; -#else - length = static_cast<size_t>(__builtin_ffs(r) - 1); -#endif - for (const char* pend = p + length; p != pend; ) - *q++ = *p++; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); - } - - is.src_ = p; - is.dst_ = q; - } - - // When read/write pointers are the same for insitu stream, just skip unescaped characters - static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { - RAPIDJSON_ASSERT(is.src_ == is.dst_); - char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast<const char*>((reinterpret_cast<size_t>(p) + 15) & static_cast<size_t>(~15)); - for (; p != nextAligned; p++) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast<unsigned>(*p) < 0x20)) { - is.src_ = is.dst_ = p; - return; - } - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&space[0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast<const __m128i *>(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast<unsigned short>(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - size_t length; -#ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; -#else - length = static_cast<size_t>(__builtin_ffs(r) - 1); -#endif - p += length; - break; - } - } - - is.src_ = is.dst_ = p; - } -#endif - - template<typename InputStream, bool backup, bool pushOnTake> - class NumberStream; - - template<typename InputStream> - class NumberStream<InputStream, false, false> { - public: - typedef typename InputStream::Ch Ch; - - NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } - ~NumberStream() {} - - RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } - RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } - RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char) {} - - size_t Tell() { return is.Tell(); } - size_t Length() { return 0; } - const char* Pop() { return 0; } - - protected: - NumberStream& operator=(const NumberStream&); - - InputStream& is; - }; - - template<typename InputStream> - class NumberStream<InputStream, true, false> : public NumberStream<InputStream, false, false> { - typedef NumberStream<InputStream, false, false> Base; - public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} - ~NumberStream() {} - - RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put(static_cast<char>(Base::is.Peek())); - return Base::is.Take(); - } - - RAPIDJSON_FORCEINLINE void Push(char c) { - stackStream.Put(c); - } - - size_t Length() { return stackStream.Length(); } - - const char* Pop() { - stackStream.Put('\0'); - return stackStream.Pop(); - } - - private: - StackStream<char> stackStream; - }; - - template<typename InputStream> - class NumberStream<InputStream, true, true> : public NumberStream<InputStream, true, false> { - typedef NumberStream<InputStream, true, false> Base; - public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} - ~NumberStream() {} - - RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } - }; - - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseNumber(InputStream& is, Handler& handler) { - internal::StreamLocalCopy<InputStream> copy(is); - NumberStream<InputStream, - ((parseFlags & kParseNumbersAsStringsFlag) != 0) ? - ((parseFlags & kParseInsituFlag) == 0) : - ((parseFlags & kParseFullPrecisionFlag) != 0), - (parseFlags & kParseNumbersAsStringsFlag) != 0 && - (parseFlags & kParseInsituFlag) == 0> s(*this, copy.s); - - size_t startOffset = s.Tell(); - double d = 0.0; - bool useNanOrInf = false; - - // Parse minus - bool minus = Consume(s, '-'); - - // Parse int: zero / ( digit1-9 *DIGIT ) - unsigned i = 0; - uint64_t i64 = 0; - bool use64bit = false; - int significandDigit = 0; - if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { - i = 0; - s.TakePush(); - } - else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { - i = static_cast<unsigned>(s.TakePush() - '0'); - - if (minus) - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 - if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { - i64 = i; - use64bit = true; - break; - } - } - i = i * 10 + static_cast<unsigned>(s.TakePush() - '0'); - significandDigit++; - } - else - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 - if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { - i64 = i; - use64bit = true; - break; - } - } - i = i * 10 + static_cast<unsigned>(s.TakePush() - '0'); - significandDigit++; - } - } - // Parse NaN or Infinity here - else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { - useNanOrInf = true; - if (RAPIDJSON_LIKELY(Consume(s, 'N') && Consume(s, 'a') && Consume(s, 'N'))) { - d = std::numeric_limits<double>::quiet_NaN(); - } - else if (RAPIDJSON_LIKELY(Consume(s, 'I') && Consume(s, 'n') && Consume(s, 'f'))) { - d = (minus ? -std::numeric_limits<double>::infinity() : std::numeric_limits<double>::infinity()); - if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') - && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); - - // Parse 64bit int - bool useDouble = false; - if (use64bit) { - if (minus) - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 - if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { - d = static_cast<double>(i64); - useDouble = true; - break; - } - i64 = i64 * 10 + static_cast<unsigned>(s.TakePush() - '0'); - significandDigit++; - } - else - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 - if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { - d = static_cast<double>(i64); - useDouble = true; - break; - } - i64 = i64 * 10 + static_cast<unsigned>(s.TakePush() - '0'); - significandDigit++; - } - } - - // Force double for big integer - if (useDouble) { - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); - d = d * 10 + (s.TakePush() - '0'); - } - } - - // Parse frac = decimal-point 1*DIGIT - int expFrac = 0; - size_t decimalPosition; - if (Consume(s, '.')) { - decimalPosition = s.Length(); - - if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); - - if (!useDouble) { -#if RAPIDJSON_64BIT - // Use i64 to store significand in 64-bit architecture - if (!use64bit) - i64 = i; - - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path - break; - else { - i64 = i64 * 10 + static_cast<unsigned>(s.TakePush() - '0'); - --expFrac; - if (i64 != 0) - significandDigit++; - } - } - - d = static_cast<double>(i64); -#else - // Use double to store significand in 32-bit architecture - d = static_cast<double>(use64bit ? i64 : i); -#endif - useDouble = true; - } - - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (significandDigit < 17) { - d = d * 10.0 + (s.TakePush() - '0'); - --expFrac; - if (RAPIDJSON_LIKELY(d > 0.0)) - significandDigit++; - } - else - s.TakePush(); - } - } - else - decimalPosition = s.Length(); // decimal position at the end of integer. - - // Parse exp = e [ minus / plus ] 1*DIGIT - int exp = 0; - if (Consume(s, 'e') || Consume(s, 'E')) { - if (!useDouble) { - d = static_cast<double>(use64bit ? i64 : i); - useDouble = true; - } - - bool expMinus = false; - if (Consume(s, '+')) - ; - else if (Consume(s, '-')) - expMinus = true; - - if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = static_cast<int>(s.Take() - '0'); - if (expMinus) { - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = exp * 10 + static_cast<int>(s.Take() - '0'); - if (exp >= 214748364) { // Issue #313: prevent overflow exponent - while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent - s.Take(); - } - } - } - else { // positive exp - int maxExp = 308 - expFrac; - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = exp * 10 + static_cast<int>(s.Take() - '0'); - if (RAPIDJSON_UNLIKELY(exp > maxExp)) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); - } - } - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); - - if (expMinus) - exp = -exp; - } - - // Finish parsing, call event according to the type of number. - bool cont = true; - - if (parseFlags & kParseNumbersAsStringsFlag) { - if (parseFlags & kParseInsituFlag) { - s.Pop(); // Pop stack no matter if it will be used or not. - typename InputStream::Ch* head = is.PutBegin(); - const size_t length = s.Tell() - startOffset; - RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - // unable to insert the \0 character here, it will erase the comma after this number - const typename TargetEncoding::Ch* const str = reinterpret_cast<typename TargetEncoding::Ch*>(head); - cont = handler.RawNumber(str, SizeType(length), false); - } - else { - SizeType numCharsToCopy = static_cast<SizeType>(s.Length()); - StringStream srcStream(s.Pop()); - StackStream<typename TargetEncoding::Ch> dstStream(stack_); - while (numCharsToCopy--) { - Transcoder<UTF8<>, TargetEncoding>::Transcode(srcStream, dstStream); - } - dstStream.Put('\0'); - const typename TargetEncoding::Ch* str = dstStream.Pop(); - const SizeType length = static_cast<SizeType>(dstStream.Length()) - 1; - cont = handler.RawNumber(str, SizeType(length), true); - } - } - else { - size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - - if (useDouble) { - int p = exp + expFrac; - if (parseFlags & kParseFullPrecisionFlag) - d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); - else - d = internal::StrtodNormalPrecision(d, p); - - cont = handler.Double(minus ? -d : d); - } - else if (useNanOrInf) { - cont = handler.Double(d); - } - else { - if (use64bit) { - if (minus) - cont = handler.Int64(static_cast<int64_t>(~i64 + 1)); - else - cont = handler.Uint64(i64); - } - else { - if (minus) - cont = handler.Int(static_cast<int32_t>(~i + 1)); - else - cont = handler.Uint(i); - } - } - } - if (RAPIDJSON_UNLIKELY(!cont)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); - } - - // Parse any JSON value - template<unsigned parseFlags, typename InputStream, typename Handler> - void ParseValue(InputStream& is, Handler& handler) { - switch (is.Peek()) { - case 'n': ParseNull <parseFlags>(is, handler); break; - case 't': ParseTrue <parseFlags>(is, handler); break; - case 'f': ParseFalse <parseFlags>(is, handler); break; - case '"': ParseString<parseFlags>(is, handler); break; - case '{': ParseObject<parseFlags>(is, handler); break; - case '[': ParseArray <parseFlags>(is, handler); break; - default : - ParseNumber<parseFlags>(is, handler); - break; - - } - } - - // Iterative Parsing - - // States - enum IterativeParsingState { - IterativeParsingStartState = 0, - IterativeParsingFinishState, - IterativeParsingErrorState, - - // Object states - IterativeParsingObjectInitialState, - IterativeParsingMemberKeyState, - IterativeParsingKeyValueDelimiterState, - IterativeParsingMemberValueState, - IterativeParsingMemberDelimiterState, - IterativeParsingObjectFinishState, - - // Array states - IterativeParsingArrayInitialState, - IterativeParsingElementState, - IterativeParsingElementDelimiterState, - IterativeParsingArrayFinishState, - - // Single value state - IterativeParsingValueState - }; - - enum { cIterativeParsingStateCount = IterativeParsingValueState + 1 }; - - // Tokens - enum Token { - LeftBracketToken = 0, - RightBracketToken, - - LeftCurlyBracketToken, - RightCurlyBracketToken, - - CommaToken, - ColonToken, - - StringToken, - FalseToken, - TrueToken, - NullToken, - NumberToken, - - kTokenCount - }; - - RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define N NumberToken -#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N - // Maps from ASCII to Token - static const unsigned char tokenMap[256] = { - N16, // 00~0F - N16, // 10~1F - N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F - N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F - N16, // 40~4F - N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F - N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F - N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F - N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF - }; -#undef N -#undef N16 -//!@endcond - - if (sizeof(Ch) == 1 || static_cast<unsigned>(c) < 256) - return static_cast<Token>(tokenMap[static_cast<unsigned char>(c)]); - else - return NumberToken; - } - - RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { - // current state x one lookahead token -> new state - static const char G[cIterativeParsingStateCount][kTokenCount] = { - // Start - { - IterativeParsingArrayInitialState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingValueState, // String - IterativeParsingValueState, // False - IterativeParsingValueState, // True - IterativeParsingValueState, // Null - IterativeParsingValueState // Number - }, - // Finish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Error(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // ObjectInitial - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // MemberKey - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingKeyValueDelimiterState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // KeyValueDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberValueState, // String - IterativeParsingMemberValueState, // False - IterativeParsingMemberValueState, // True - IterativeParsingMemberValueState, // Null - IterativeParsingMemberValueState // Number - }, - // MemberValue - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingMemberDelimiterState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // MemberDelimiter - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // ObjectFinish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // ArrayInitial - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, - // Element - { - IterativeParsingErrorState, // Left bracket - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingElementDelimiterState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // ElementDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, - // ArrayFinish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Single Value (sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - } - }; // End of G - - return static_cast<IterativeParsingState>(G[state][token]); - } - - // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). - // May return a new state on state pop. - template <unsigned parseFlags, typename InputStream, typename Handler> - RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { - (void)token; - - switch (dst) { - case IterativeParsingErrorState: - return dst; - - case IterativeParsingObjectInitialState: - case IterativeParsingArrayInitialState: - { - // Push the state(Element or MemeberValue) if we are nested in another array or value of member. - // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. - IterativeParsingState n = src; - if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) - n = IterativeParsingElementState; - else if (src == IterativeParsingKeyValueDelimiterState) - n = IterativeParsingMemberValueState; - // Push current state. - *stack_.template Push<SizeType>(1) = n; - // Initialize and push the member/element count. - *stack_.template Push<SizeType>(1) = 0; - // Call handler - bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return dst; - } - } - - case IterativeParsingMemberKeyState: - ParseString<parseFlags>(is, handler, true); - if (HasParseError()) - return IterativeParsingErrorState; - else - return dst; - - case IterativeParsingKeyValueDelimiterState: - RAPIDJSON_ASSERT(token == ColonToken); - is.Take(); - return dst; - - case IterativeParsingMemberValueState: - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue<parseFlags>(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return dst; - - case IterativeParsingElementState: - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue<parseFlags>(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return dst; - - case IterativeParsingMemberDelimiterState: - case IterativeParsingElementDelimiterState: - is.Take(); - // Update member/element count. - *stack_.template Top<SizeType>() = *stack_.template Top<SizeType>() + 1; - return dst; - - case IterativeParsingObjectFinishState: - { - // Transit from delimiter is only allowed when trailing commas are enabled - if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); - return IterativeParsingErrorState; - } - // Get member count. - SizeType c = *stack_.template Pop<SizeType>(1); - // If the object is not empty, count the last member. - if (src == IterativeParsingMemberValueState) - ++c; - // Restore the state. - IterativeParsingState n = static_cast<IterativeParsingState>(*stack_.template Pop<SizeType>(1)); - // Transit to Finish state if this is the topmost scope. - if (n == IterativeParsingStartState) - n = IterativeParsingFinishState; - // Call handler - bool hr = handler.EndObject(c); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return n; - } - } - - case IterativeParsingArrayFinishState: - { - // Transit from delimiter is only allowed when trailing commas are enabled - if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); - return IterativeParsingErrorState; - } - // Get element count. - SizeType c = *stack_.template Pop<SizeType>(1); - // If the array is not empty, count the last element. - if (src == IterativeParsingElementState) - ++c; - // Restore the state. - IterativeParsingState n = static_cast<IterativeParsingState>(*stack_.template Pop<SizeType>(1)); - // Transit to Finish state if this is the topmost scope. - if (n == IterativeParsingStartState) - n = IterativeParsingFinishState; - // Call handler - bool hr = handler.EndArray(c); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return n; - } - } - - default: - // This branch is for IterativeParsingValueState actually. - // Use `default:` rather than - // `case IterativeParsingValueState:` is for code coverage. - - // The IterativeParsingStartState is not enumerated in this switch-case. - // It is impossible for that case. And it can be caught by following assertion. - - // The IterativeParsingFinishState is not enumerated in this switch-case either. - // It is a "derivative" state which cannot triggered from Predict() directly. - // Therefore it cannot happen here. And it can be caught by following assertion. - RAPIDJSON_ASSERT(dst == IterativeParsingValueState); - - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue<parseFlags>(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return IterativeParsingFinishState; - } - } - - template <typename InputStream> - void HandleError(IterativeParsingState src, InputStream& is) { - if (HasParseError()) { - // Error flag has been set. - return; - } - - switch (src) { - case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; - case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; - case IterativeParsingObjectInitialState: - case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; - case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; - case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; - case IterativeParsingKeyValueDelimiterState: - case IterativeParsingArrayInitialState: - case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; - default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; - } - } - - template <unsigned parseFlags, typename InputStream, typename Handler> - ParseResult IterativeParse(InputStream& is, Handler& handler) { - parseResult_.Clear(); - ClearStackOnExit scope(*this); - IterativeParsingState state = IterativeParsingStartState; - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - while (is.Peek() != '\0') { - Token t = Tokenize(is.Peek()); - IterativeParsingState n = Predict(state, t); - IterativeParsingState d = Transit<parseFlags>(state, t, n, is, handler); - - if (d == IterativeParsingErrorState) { - HandleError(state, is); - break; - } - - state = d; - - // Do not further consume streams if a root JSON has been parsed. - if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) - break; - - SkipWhitespaceAndComments<parseFlags>(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - } - - // Handle the end of file. - if (state != IterativeParsingFinishState) - HandleError(state, is); - - return parseResult_; - } - - static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. - internal::Stack<StackAllocator> stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. - ParseResult parseResult_; -}; // class GenericReader - -//! Reader with UTF8 encoding and default allocator. -typedef GenericReader<UTF8<>, UTF8<> > Reader; - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_READER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/schema.h b/ext/librethinkdbxx/src/rapidjson/schema.h deleted file mode 100644 index b182aa27..00000000 --- a/ext/librethinkdbxx/src/rapidjson/schema.h +++ /dev/null @@ -1,2006 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available-> -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License-> You may obtain a copy of the License at -// -// http://opensource->org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied-> See the License for the -// specific language governing permissions and limitations under the License-> - -#ifndef RAPIDJSON_SCHEMA_H_ -#define RAPIDJSON_SCHEMA_H_ - -#include "document.h" -#include "pointer.h" -#include <cmath> // abs, floor - -#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) -#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 -#else -#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 -#endif - -#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) -#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 -#else -#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 -#endif - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX -#include "internal/regex.h" -#elif RAPIDJSON_SCHEMA_USE_STDREGEX -#include <regex> -#endif - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX -#define RAPIDJSON_SCHEMA_HAS_REGEX 1 -#else -#define RAPIDJSON_SCHEMA_HAS_REGEX 0 -#endif - -#ifndef RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_SCHEMA_VERBOSE 0 -#endif - -#if RAPIDJSON_SCHEMA_VERBOSE -#include "stringbuffer.h" -#endif - -RAPIDJSON_DIAG_PUSH - -#if defined(__GNUC__) -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_OFF(weak-vtables) -RAPIDJSON_DIAG_OFF(exit-time-destructors) -RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) -RAPIDJSON_DIAG_OFF(variadic-macros) -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// Verbose Utilities - -#if RAPIDJSON_SCHEMA_VERBOSE - -namespace internal { - -inline void PrintInvalidKeyword(const char* keyword) { - printf("Fail keyword: %s\n", keyword); -} - -inline void PrintInvalidKeyword(const wchar_t* keyword) { - wprintf(L"Fail keyword: %ls\n", keyword); -} - -inline void PrintInvalidDocument(const char* document) { - printf("Fail document: %s\n\n", document); -} - -inline void PrintInvalidDocument(const wchar_t* document) { - wprintf(L"Fail document: %ls\n\n", document); -} - -inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { - printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); -} - -inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { - wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); -} - -} // namespace internal - -#endif // RAPIDJSON_SCHEMA_VERBOSE - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_INVALID_KEYWORD_RETURN - -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) -#else -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) -#endif - -#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ -RAPIDJSON_MULTILINEMACRO_BEGIN\ - context.invalidKeyword = keyword.GetString();\ - RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ - return false;\ -RAPIDJSON_MULTILINEMACRO_END - -/////////////////////////////////////////////////////////////////////////////// -// Forward declarations - -template <typename ValueType, typename Allocator> -class GenericSchemaDocument; - -namespace internal { - -template <typename SchemaDocumentType> -class Schema; - -/////////////////////////////////////////////////////////////////////////////// -// ISchemaValidator - -class ISchemaValidator { -public: - virtual ~ISchemaValidator() {} - virtual bool IsValid() const = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// ISchemaStateFactory - -template <typename SchemaType> -class ISchemaStateFactory { -public: - virtual ~ISchemaStateFactory() {} - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; - virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; - virtual void* CreateHasher() = 0; - virtual uint64_t GetHashCode(void* hasher) = 0; - virtual void DestroryHasher(void* hasher) = 0; - virtual void* MallocState(size_t size) = 0; - virtual void FreeState(void* p) = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// Hasher - -// For comparison of compound value -template<typename Encoding, typename Allocator> -class Hasher { -public: - typedef typename Encoding::Ch Ch; - - Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} - - bool Null() { return WriteType(kNullType); } - bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } - bool Int(int i) { Number n; n.u.i = i; n.d = static_cast<double>(i); return WriteNumber(n); } - bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast<double>(u); return WriteNumber(n); } - bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast<double>(i); return WriteNumber(n); } - bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast<double>(u); return WriteNumber(n); } - bool Double(double d) { - Number n; - if (d < 0) n.u.i = static_cast<int64_t>(d); - else n.u.u = static_cast<uint64_t>(d); - n.d = d; - return WriteNumber(n); - } - - bool RawNumber(const Ch* str, SizeType len, bool) { - WriteBuffer(kNumberType, str, len * sizeof(Ch)); - return true; - } - - bool String(const Ch* str, SizeType len, bool) { - WriteBuffer(kStringType, str, len * sizeof(Ch)); - return true; - } - - bool StartObject() { return true; } - bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } - bool EndObject(SizeType memberCount) { - uint64_t h = Hash(0, kObjectType); - uint64_t* kv = stack_.template Pop<uint64_t>(memberCount * 2); - for (SizeType i = 0; i < memberCount; i++) - h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive - *stack_.template Push<uint64_t>() = h; - return true; - } - - bool StartArray() { return true; } - bool EndArray(SizeType elementCount) { - uint64_t h = Hash(0, kArrayType); - uint64_t* e = stack_.template Pop<uint64_t>(elementCount); - for (SizeType i = 0; i < elementCount; i++) - h = Hash(h, e[i]); // Use hash to achieve element order sensitive - *stack_.template Push<uint64_t>() = h; - return true; - } - - bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } - - uint64_t GetHashCode() const { - RAPIDJSON_ASSERT(IsValid()); - return *stack_.template Top<uint64_t>(); - } - -private: - static const size_t kDefaultSize = 256; - struct Number { - union U { - uint64_t u; - int64_t i; - }u; - double d; - }; - - bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } - - bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } - - bool WriteBuffer(Type type, const void* data, size_t len) { - // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ - uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); - const unsigned char* d = static_cast<const unsigned char*>(data); - for (size_t i = 0; i < len; i++) - h = Hash(h, d[i]); - *stack_.template Push<uint64_t>() = h; - return true; - } - - static uint64_t Hash(uint64_t h, uint64_t d) { - static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); - h ^= d; - h *= kPrime; - return h; - } - - Stack<Allocator> stack_; -}; - -/////////////////////////////////////////////////////////////////////////////// -// SchemaValidationContext - -template <typename SchemaDocumentType> -struct SchemaValidationContext { - typedef Schema<SchemaDocumentType> SchemaType; - typedef ISchemaStateFactory<SchemaType> SchemaValidatorFactoryType; - typedef typename SchemaType::ValueType ValueType; - typedef typename ValueType::Ch Ch; - - enum PatternValidatorType { - kPatternValidatorOnly, - kPatternValidatorWithProperty, - kPatternValidatorWithAdditionalProperty - }; - - SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : - factory(f), - schema(s), - valueSchema(), - invalidKeyword(), - hasher(), - arrayElementHashCodes(), - validators(), - validatorCount(), - patternPropertiesValidators(), - patternPropertiesValidatorCount(), - patternPropertiesSchemas(), - patternPropertiesSchemaCount(), - valuePatternValidatorType(kPatternValidatorOnly), - propertyExist(), - inArray(false), - valueUniqueness(false), - arrayUniqueness(false) - { - } - - ~SchemaValidationContext() { - if (hasher) - factory.DestroryHasher(hasher); - if (validators) { - for (SizeType i = 0; i < validatorCount; i++) - factory.DestroySchemaValidator(validators[i]); - factory.FreeState(validators); - } - if (patternPropertiesValidators) { - for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) - factory.DestroySchemaValidator(patternPropertiesValidators[i]); - factory.FreeState(patternPropertiesValidators); - } - if (patternPropertiesSchemas) - factory.FreeState(patternPropertiesSchemas); - if (propertyExist) - factory.FreeState(propertyExist); - } - - SchemaValidatorFactoryType& factory; - const SchemaType* schema; - const SchemaType* valueSchema; - const Ch* invalidKeyword; - void* hasher; // Only validator access - void* arrayElementHashCodes; // Only validator access this - ISchemaValidator** validators; - SizeType validatorCount; - ISchemaValidator** patternPropertiesValidators; - SizeType patternPropertiesValidatorCount; - const SchemaType** patternPropertiesSchemas; - SizeType patternPropertiesSchemaCount; - PatternValidatorType valuePatternValidatorType; - PatternValidatorType objectPatternValidatorType; - SizeType arrayElementIndex; - bool* propertyExist; - bool inArray; - bool valueUniqueness; - bool arrayUniqueness; -}; - -/////////////////////////////////////////////////////////////////////////////// -// Schema - -template <typename SchemaDocumentType> -class Schema { -public: - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename SchemaDocumentType::AllocatorType AllocatorType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef SchemaValidationContext<SchemaDocumentType> Context; - typedef Schema<SchemaDocumentType> SchemaType; - typedef GenericValue<EncodingType, AllocatorType> SValue; - friend class GenericSchemaDocument<ValueType, AllocatorType>; - - Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : - allocator_(allocator), - enum_(), - enumCount_(), - not_(), - type_((1 << kTotalSchemaType) - 1), // typeless - validatorCount_(), - properties_(), - additionalPropertiesSchema_(), - patternProperties_(), - patternPropertyCount_(), - propertyCount_(), - minProperties_(), - maxProperties_(SizeType(~0)), - additionalProperties_(true), - hasDependencies_(), - hasRequired_(), - hasSchemaDependencies_(), - additionalItemsSchema_(), - itemsList_(), - itemsTuple_(), - itemsTupleCount_(), - minItems_(), - maxItems_(SizeType(~0)), - additionalItems_(true), - uniqueItems_(false), - pattern_(), - minLength_(0), - maxLength_(~SizeType(0)), - exclusiveMinimum_(false), - exclusiveMaximum_(false) - { - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename ValueType::ConstValueIterator ConstValueIterator; - typedef typename ValueType::ConstMemberIterator ConstMemberIterator; - - if (!value.IsObject()) - return; - - if (const ValueType* v = GetMember(value, GetTypeString())) { - type_ = 0; - if (v->IsString()) - AddType(*v); - else if (v->IsArray()) - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) - AddType(*itr); - } - - if (const ValueType* v = GetMember(value, GetEnumString())) - if (v->IsArray() && v->Size() > 0) { - enum_ = static_cast<uint64_t*>(allocator_->Malloc(sizeof(uint64_t) * v->Size())); - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { - typedef Hasher<EncodingType, MemoryPoolAllocator<> > EnumHasherType; - char buffer[256 + 24]; - MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); - EnumHasherType h(&hasherAllocator, 256); - itr->Accept(h); - enum_[enumCount_++] = h.GetHashCode(); - } - } - - if (schemaDocument) { - AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); - AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); - AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); - } - - if (const ValueType* v = GetMember(value, GetNotString())) { - schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); - notValidatorIndex_ = validatorCount_; - validatorCount_++; - } - - // Object - - const ValueType* properties = GetMember(value, GetPropertiesString()); - const ValueType* required = GetMember(value, GetRequiredString()); - const ValueType* dependencies = GetMember(value, GetDependenciesString()); - { - // Gather properties from properties/required/dependencies - SValue allProperties(kArrayType); - - if (properties && properties->IsObject()) - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) - AddUniqueElement(allProperties, itr->name); - - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) - AddUniqueElement(allProperties, *itr); - - if (dependencies && dependencies->IsObject()) - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - AddUniqueElement(allProperties, itr->name); - if (itr->value.IsArray()) - for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) - if (i->IsString()) - AddUniqueElement(allProperties, *i); - } - - if (allProperties.Size() > 0) { - propertyCount_ = allProperties.Size(); - properties_ = static_cast<Property*>(allocator_->Malloc(sizeof(Property) * propertyCount_)); - for (SizeType i = 0; i < propertyCount_; i++) { - new (&properties_[i]) Property(); - properties_[i].name = allProperties[i]; - properties_[i].schema = GetTypeless(); - } - } - } - - if (properties && properties->IsObject()) { - PointerType q = p.Append(GetPropertiesString(), allocator_); - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { - SizeType index; - if (FindPropertyIndex(itr->name, &index)) - schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); - } - } - - if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { - PointerType q = p.Append(GetPatternPropertiesString(), allocator_); - patternProperties_ = static_cast<PatternProperty*>(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); - patternPropertyCount_ = 0; - - for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { - new (&patternProperties_[patternPropertyCount_]) PatternProperty(); - patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); - schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); - patternPropertyCount_++; - } - } - - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) { - SizeType index; - if (FindPropertyIndex(*itr, &index)) { - properties_[index].required = true; - hasRequired_ = true; - } - } - - if (dependencies && dependencies->IsObject()) { - PointerType q = p.Append(GetDependenciesString(), allocator_); - hasDependencies_ = true; - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - SizeType sourceIndex; - if (FindPropertyIndex(itr->name, &sourceIndex)) { - if (itr->value.IsArray()) { - properties_[sourceIndex].dependencies = static_cast<bool*>(allocator_->Malloc(sizeof(bool) * propertyCount_)); - std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); - for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { - SizeType targetIndex; - if (FindPropertyIndex(*targetItr, &targetIndex)) - properties_[sourceIndex].dependencies[targetIndex] = true; - } - } - else if (itr->value.IsObject()) { - hasSchemaDependencies_ = true; - schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); - properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; - validatorCount_++; - } - } - } - } - - if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { - if (v->IsBool()) - additionalProperties_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); - } - - AssignIfExist(minProperties_, value, GetMinPropertiesString()); - AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); - - // Array - if (const ValueType* v = GetMember(value, GetItemsString())) { - PointerType q = p.Append(GetItemsString(), allocator_); - if (v->IsObject()) // List validation - schemaDocument->CreateSchema(&itemsList_, q, *v, document); - else if (v->IsArray()) { // Tuple validation - itemsTuple_ = static_cast<const Schema**>(allocator_->Malloc(sizeof(const Schema*) * v->Size())); - SizeType index = 0; - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) - schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); - } - } - - AssignIfExist(minItems_, value, GetMinItemsString()); - AssignIfExist(maxItems_, value, GetMaxItemsString()); - - if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { - if (v->IsBool()) - additionalItems_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); - } - - AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); - - // String - AssignIfExist(minLength_, value, GetMinLengthString()); - AssignIfExist(maxLength_, value, GetMaxLengthString()); - - if (const ValueType* v = GetMember(value, GetPatternString())) - pattern_ = CreatePattern(*v); - - // Number - if (const ValueType* v = GetMember(value, GetMinimumString())) - if (v->IsNumber()) - minimum_.CopyFrom(*v, *allocator_); - - if (const ValueType* v = GetMember(value, GetMaximumString())) - if (v->IsNumber()) - maximum_.CopyFrom(*v, *allocator_); - - AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); - AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); - - if (const ValueType* v = GetMember(value, GetMultipleOfString())) - if (v->IsNumber() && v->GetDouble() > 0.0) - multipleOf_.CopyFrom(*v, *allocator_); - } - - ~Schema() { - if (allocator_) { - allocator_->Free(enum_); - } - if (properties_) { - for (SizeType i = 0; i < propertyCount_; i++) - properties_[i].~Property(); - AllocatorType::Free(properties_); - } - if (patternProperties_) { - for (SizeType i = 0; i < patternPropertyCount_; i++) - patternProperties_[i].~PatternProperty(); - AllocatorType::Free(patternProperties_); - } - AllocatorType::Free(itemsTuple_); -#if RAPIDJSON_SCHEMA_HAS_REGEX - if (pattern_) { - pattern_->~RegexType(); - allocator_->Free(pattern_); - } -#endif - } - - bool BeginValue(Context& context) const { - if (context.inArray) { - if (uniqueItems_) - context.valueUniqueness = true; - - if (itemsList_) - context.valueSchema = itemsList_; - else if (itemsTuple_) { - if (context.arrayElementIndex < itemsTupleCount_) - context.valueSchema = itemsTuple_[context.arrayElementIndex]; - else if (additionalItemsSchema_) - context.valueSchema = additionalItemsSchema_; - else if (additionalItems_) - context.valueSchema = GetTypeless(); - else - RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); - } - else - context.valueSchema = GetTypeless(); - - context.arrayElementIndex++; - } - return true; - } - - RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { - if (context.patternPropertiesValidatorCount > 0) { - bool otherValid = false; - SizeType count = context.patternPropertiesValidatorCount; - if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) - otherValid = context.patternPropertiesValidators[--count]->IsValid(); - - bool patternValid = true; - for (SizeType i = 0; i < count; i++) - if (!context.patternPropertiesValidators[i]->IsValid()) { - patternValid = false; - break; - } - - if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { - if (!patternValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { - if (!patternValid || !otherValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - - if (enum_) { - const uint64_t h = context.factory.GetHashCode(context.hasher); - for (SizeType i = 0; i < enumCount_; i++) - if (enum_[i] == h) - goto foundEnum; - RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); - foundEnum:; - } - - if (allOf_.schemas) - for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) - if (!context.validators[i]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); - - if (anyOf_.schemas) { - for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) - if (context.validators[i]->IsValid()) - goto foundAny; - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); - foundAny:; - } - - if (oneOf_.schemas) { - bool oneValid = false; - for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) - if (context.validators[i]->IsValid()) { - if (oneValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - else - oneValid = true; - } - if (!oneValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - } - - if (not_ && context.validators[notValidatorIndex_]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); - - return true; - } - - bool Null(Context& context) const { - if (!(type_ & (1 << kNullSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - return CreateParallelValidator(context); - } - - bool Bool(Context& context, bool) const { - if (!(type_ & (1 << kBooleanSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - return CreateParallelValidator(context); - } - - bool Int(Context& context, int i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); - } - - bool Uint(Context& context, unsigned u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); - } - - bool Int64(Context& context, int64_t i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); - } - - bool Uint64(Context& context, uint64_t u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); - } - - bool Double(Context& context, double d) const { - if (!(type_ & (1 << kNumberSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) - return false; - - if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) - return false; - - if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) - return false; - - return CreateParallelValidator(context); - } - - bool String(Context& context, const Ch* str, SizeType length, bool) const { - if (!(type_ & (1 << kStringSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (minLength_ != 0 || maxLength_ != SizeType(~0)) { - SizeType count; - if (internal::CountStringCodePoint<EncodingType>(str, length, &count)) { - if (count < minLength_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); - if (count > maxLength_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); - } - } - - if (pattern_ && !IsPatternMatch(pattern_, str, length)) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); - - return CreateParallelValidator(context); - } - - bool StartObject(Context& context) const { - if (!(type_ & (1 << kObjectSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (hasDependencies_ || hasRequired_) { - context.propertyExist = static_cast<bool*>(context.factory.MallocState(sizeof(bool) * propertyCount_)); - std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); - } - - if (patternProperties_) { // pre-allocate schema array - SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType - context.patternPropertiesSchemas = static_cast<const SchemaType**>(context.factory.MallocState(sizeof(const SchemaType*) * count)); - context.patternPropertiesSchemaCount = 0; - std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); - } - - return CreateParallelValidator(context); - } - - bool Key(Context& context, const Ch* str, SizeType len, bool) const { - if (patternProperties_) { - context.patternPropertiesSchemaCount = 0; - for (SizeType i = 0; i < patternPropertyCount_; i++) - if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; - } - - SizeType index; - if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { - if (context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; - context.valueSchema = GetTypeless(); - context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; - } - else - context.valueSchema = properties_[index].schema; - - if (context.propertyExist) - context.propertyExist[index] = true; - - return true; - } - - if (additionalPropertiesSchema_) { - if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; - context.valueSchema = GetTypeless(); - context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; - } - else - context.valueSchema = additionalPropertiesSchema_; - return true; - } - else if (additionalProperties_) { - context.valueSchema = GetTypeless(); - return true; - } - - if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); - - return true; - } - - bool EndObject(Context& context, SizeType memberCount) const { - if (hasRequired_) - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].required) - if (!context.propertyExist[index]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); - - if (memberCount < minProperties_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); - - if (memberCount > maxProperties_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); - - if (hasDependencies_) { - for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) - if (context.propertyExist[sourceIndex]) { - if (properties_[sourceIndex].dependencies) { - for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) - if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); - } - else if (properties_[sourceIndex].dependenciesSchema) - if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); - } - } - - return true; - } - - bool StartArray(Context& context) const { - if (!(type_ & (1 << kArraySchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - context.arrayElementIndex = 0; - context.inArray = true; - - return CreateParallelValidator(context); - } - - bool EndArray(Context& context, SizeType elementCount) const { - context.inArray = false; - - if (elementCount < minItems_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); - - if (elementCount > maxItems_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); - - return true; - } - - // Generate functions for string literal according to Ch -#define RAPIDJSON_STRING_(name, ...) \ - static const ValueType& Get##name##String() {\ - static const Ch s[] = { __VA_ARGS__, '\0' };\ - static const ValueType v(s, sizeof(s) / sizeof(Ch) - 1);\ - return v;\ - } - - RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') - RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') - RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') - RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') - RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') - RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') - RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') - RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') - RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') - RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') - RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') - RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') - RAPIDJSON_STRING_(Not, 'n', 'o', 't') - RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') - RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') - RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') - RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') - -#undef RAPIDJSON_STRING_ - -private: - enum SchemaValueType { - kNullSchemaType, - kBooleanSchemaType, - kObjectSchemaType, - kArraySchemaType, - kStringSchemaType, - kNumberSchemaType, - kIntegerSchemaType, - kTotalSchemaType - }; - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - typedef internal::GenericRegex<EncodingType> RegexType; -#elif RAPIDJSON_SCHEMA_USE_STDREGEX - typedef std::basic_regex<Ch> RegexType; -#else - typedef char RegexType; -#endif - - struct SchemaArray { - SchemaArray() : schemas(), count() {} - ~SchemaArray() { AllocatorType::Free(schemas); } - const SchemaType** schemas; - SizeType begin; // begin index of context.validators - SizeType count; - }; - - static const SchemaType* GetTypeless() { - static SchemaType typeless(0, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); - return &typeless; - } - - template <typename V1, typename V2> - void AddUniqueElement(V1& a, const V2& v) { - for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) - if (*itr == v) - return; - V1 c(v, *allocator_); - a.PushBack(c, *allocator_); - } - - static const ValueType* GetMember(const ValueType& value, const ValueType& name) { - typename ValueType::ConstMemberIterator itr = value.FindMember(name); - return itr != value.MemberEnd() ? &(itr->value) : 0; - } - - static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsBool()) - out = v->GetBool(); - } - - static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) - out = static_cast<SizeType>(v->GetUint64()); - } - - void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { - if (const ValueType* v = GetMember(value, name)) { - if (v->IsArray() && v->Size() > 0) { - PointerType q = p.Append(name, allocator_); - out.count = v->Size(); - out.schemas = static_cast<const Schema**>(allocator_->Malloc(out.count * sizeof(const Schema*))); - memset(out.schemas, 0, sizeof(Schema*)* out.count); - for (SizeType i = 0; i < out.count; i++) - schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); - out.begin = validatorCount_; - validatorCount_ += out.count; - } - } - } - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - template <typename ValueType> - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) { - RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); - if (!r->IsValid()) { - r->~RegexType(); - AllocatorType::Free(r); - r = 0; - } - return r; - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { - return pattern->Search(str); - } -#elif RAPIDJSON_SCHEMA_USE_STDREGEX - template <typename ValueType> - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) - try { - return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); - } - catch (const std::regex_error&) { - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { - std::match_results<const Ch*> r; - return std::regex_search(str, str + length, r, *pattern); - } -#else - template <typename ValueType> - RegexType* CreatePattern(const ValueType&) { return 0; } - - static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } -#endif // RAPIDJSON_SCHEMA_USE_STDREGEX - - void AddType(const ValueType& type) { - if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; - else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; - else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; - else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; - else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; - else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; - else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); - } - - bool CreateParallelValidator(Context& context) const { - if (enum_ || context.arrayUniqueness) - context.hasher = context.factory.CreateHasher(); - - if (validatorCount_) { - RAPIDJSON_ASSERT(context.validators == 0); - context.validators = static_cast<ISchemaValidator**>(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); - context.validatorCount = validatorCount_; - - if (allOf_.schemas) - CreateSchemaValidators(context, allOf_); - - if (anyOf_.schemas) - CreateSchemaValidators(context, anyOf_); - - if (oneOf_.schemas) - CreateSchemaValidators(context, oneOf_); - - if (not_) - context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); - - if (hasSchemaDependencies_) { - for (SizeType i = 0; i < propertyCount_; i++) - if (properties_[i].dependenciesSchema) - context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); - } - } - - return true; - } - - void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { - for (SizeType i = 0; i < schemas.count; i++) - context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); - } - - // O(n) - bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { - SizeType len = name.GetStringLength(); - const Ch* str = name.GetString(); - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].name.GetStringLength() == len && - (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) - { - *outIndex = index; - return true; - } - return false; - } - - bool CheckInt(Context& context, int64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (!minimum_.IsNull()) { - if (minimum_.IsInt64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - else if (minimum_.IsUint64()) { - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() - } - else if (!CheckDoubleMinimum(context, static_cast<double>(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsInt64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - else if (maximum_.IsUint64()) - /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() - else if (!CheckDoubleMaximum(context, static_cast<double>(i))) - return false; - } - - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (static_cast<uint64_t>(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - else if (!CheckDoubleMultipleOf(context, static_cast<double>(i))) - return false; - } - - return true; - } - - bool CheckUint(Context& context, uint64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (!minimum_.IsNull()) { - if (minimum_.IsUint64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - else if (minimum_.IsInt64()) - /* do nothing */; // i >= 0 > minimum.Getint64() - else if (!CheckDoubleMinimum(context, static_cast<double>(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsUint64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - else if (maximum_.IsInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ - else if (!CheckDoubleMaximum(context, static_cast<double>(i))) - return false; - } - - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (i % multipleOf_.GetUint64() != 0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - else if (!CheckDoubleMultipleOf(context, static_cast<double>(i))) - return false; - } - - return true; - } - - bool CheckDoubleMinimum(Context& context, double d) const { - if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - return true; - } - - bool CheckDoubleMaximum(Context& context, double d) const { - if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - return true; - } - - bool CheckDoubleMultipleOf(Context& context, double d) const { - double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); - double q = std::floor(a / b); - double r = a - q * b; - if (r > 0.0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - return true; - } - - struct Property { - Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} - ~Property() { AllocatorType::Free(dependencies); } - SValue name; - const SchemaType* schema; - const SchemaType* dependenciesSchema; - SizeType dependenciesValidatorIndex; - bool* dependencies; - bool required; - }; - - struct PatternProperty { - PatternProperty() : schema(), pattern() {} - ~PatternProperty() { - if (pattern) { - pattern->~RegexType(); - AllocatorType::Free(pattern); - } - } - const SchemaType* schema; - RegexType* pattern; - }; - - AllocatorType* allocator_; - uint64_t* enum_; - SizeType enumCount_; - SchemaArray allOf_; - SchemaArray anyOf_; - SchemaArray oneOf_; - const SchemaType* not_; - unsigned type_; // bitmask of kSchemaType - SizeType validatorCount_; - SizeType notValidatorIndex_; - - Property* properties_; - const SchemaType* additionalPropertiesSchema_; - PatternProperty* patternProperties_; - SizeType patternPropertyCount_; - SizeType propertyCount_; - SizeType minProperties_; - SizeType maxProperties_; - bool additionalProperties_; - bool hasDependencies_; - bool hasRequired_; - bool hasSchemaDependencies_; - - const SchemaType* additionalItemsSchema_; - const SchemaType* itemsList_; - const SchemaType** itemsTuple_; - SizeType itemsTupleCount_; - SizeType minItems_; - SizeType maxItems_; - bool additionalItems_; - bool uniqueItems_; - - RegexType* pattern_; - SizeType minLength_; - SizeType maxLength_; - - SValue minimum_; - SValue maximum_; - SValue multipleOf_; - bool exclusiveMinimum_; - bool exclusiveMaximum_; -}; - -template<typename Stack, typename Ch> -struct TokenHelper { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - *documentStack.template Push<Ch>() = '/'; - char buffer[21]; - size_t length = static_cast<size_t>((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); - for (size_t i = 0; i < length; i++) - *documentStack.template Push<Ch>() = buffer[i]; - } -}; - -// Partial specialized version for char to prevent buffer copying. -template <typename Stack> -struct TokenHelper<Stack, char> { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - if (sizeof(SizeType) == 4) { - char *buffer = documentStack.template Push<char>(1 + 10); // '/' + uint - *buffer++ = '/'; - const char* end = internal::u32toa(index, buffer); - documentStack.template Pop<char>(static_cast<size_t>(10 - (end - buffer))); - } - else { - char *buffer = documentStack.template Push<char>(1 + 20); // '/' + uint64 - *buffer++ = '/'; - const char* end = internal::u64toa(index, buffer); - documentStack.template Pop<char>(static_cast<size_t>(20 - (end - buffer))); - } - } -}; - -} // namespace internal - -/////////////////////////////////////////////////////////////////////////////// -// IGenericRemoteSchemaDocumentProvider - -template <typename SchemaDocumentType> -class IGenericRemoteSchemaDocumentProvider { -public: - typedef typename SchemaDocumentType::Ch Ch; - - virtual ~IGenericRemoteSchemaDocumentProvider() {} - virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericSchemaDocument - -//! JSON schema document. -/*! - A JSON schema document is a compiled version of a JSON schema. - It is basically a tree of internal::Schema. - - \note This is an immutable class (i.e. its instance cannot be modified after construction). - \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. - \tparam Allocator Allocator type for allocating memory of this document. -*/ -template <typename ValueT, typename Allocator = CrtAllocator> -class GenericSchemaDocument { -public: - typedef ValueT ValueType; - typedef IGenericRemoteSchemaDocumentProvider<GenericSchemaDocument> IRemoteSchemaDocumentProviderType; - typedef Allocator AllocatorType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef internal::Schema<GenericSchemaDocument> SchemaType; - typedef GenericPointer<ValueType, Allocator> PointerType; - friend class internal::Schema<GenericSchemaDocument>; - template <typename, typename, typename> - friend class GenericSchemaValidator; - - //! Constructor. - /*! - Compile a JSON document into schema document. - - \param document A JSON document as source. - \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. - \param allocator An optional allocator instance for allocating memory. Can be null. - */ - explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : - remoteProvider_(remoteProvider), - allocator_(allocator), - ownAllocator_(), - root_(), - schemaMap_(allocator, kInitialSchemaMapSize), - schemaRef_(allocator, kInitialSchemaRefSize) - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - - // Generate root schema, it will call CreateSchema() to create sub-schemas, - // And call AddRefSchema() if there are $ref. - CreateSchemaRecursive(&root_, PointerType(), document, document); - - // Resolve $ref - while (!schemaRef_.Empty()) { - SchemaRefEntry* refEntry = schemaRef_.template Pop<SchemaRefEntry>(1); - if (const SchemaType* s = GetSchema(refEntry->target)) { - if (refEntry->schema) - *refEntry->schema = s; - - // Create entry in map if not exist - if (!GetSchema(refEntry->source)) { - new (schemaMap_.template Push<SchemaEntry>()) SchemaEntry(refEntry->source, const_cast<SchemaType*>(s), false, allocator_); - } - } - refEntry->~SchemaRefEntry(); - } - - RAPIDJSON_ASSERT(root_ != 0); - - schemaRef_.ShrinkToFit(); // Deallocate all memory for ref - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : - remoteProvider_(rhs.remoteProvider_), - allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - root_(rhs.root_), - schemaMap_(std::move(rhs.schemaMap_)), - schemaRef_(std::move(rhs.schemaRef_)) - { - rhs.remoteProvider_ = 0; - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - } -#endif - - //! Destructor - ~GenericSchemaDocument() { - while (!schemaMap_.Empty()) - schemaMap_.template Pop<SchemaEntry>(1)->~SchemaEntry(); - - RAPIDJSON_DELETE(ownAllocator_); - } - - //! Get the root schema. - const SchemaType& GetRoot() const { return *root_; } - -private: - //! Prohibit copying - GenericSchemaDocument(const GenericSchemaDocument&); - //! Prohibit assignment - GenericSchemaDocument& operator=(const GenericSchemaDocument&); - - struct SchemaRefEntry { - SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} - PointerType source; - PointerType target; - const SchemaType** schema; - }; - - struct SchemaEntry { - SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} - ~SchemaEntry() { - if (owned) { - schema->~SchemaType(); - Allocator::Free(schema); - } - } - PointerType pointer; - SchemaType* schema; - bool owned; - }; - - void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { - if (schema) - *schema = SchemaType::GetTypeless(); - - if (v.GetType() == kObjectType) { - const SchemaType* s = GetSchema(pointer); - if (!s) - CreateSchema(schema, pointer, v, document); - - for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) - CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); - } - else if (v.GetType() == kArrayType) - for (SizeType i = 0; i < v.Size(); i++) - CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); - } - - void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { - RAPIDJSON_ASSERT(pointer.IsValid()); - if (v.IsObject()) { - if (!HandleRefSchema(pointer, schema, v, document)) { - SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); - new (schemaMap_.template Push<SchemaEntry>()) SchemaEntry(pointer, s, true, allocator_); - if (schema) - *schema = s; - } - } - } - - bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { - static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; - static const ValueType kRefValue(kRefString, 4); - - typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); - if (itr == v.MemberEnd()) - return false; - - if (itr->value.IsString()) { - SizeType len = itr->value.GetStringLength(); - if (len > 0) { - const Ch* s = itr->value.GetString(); - SizeType i = 0; - while (i < len && s[i] != '#') // Find the first # - i++; - - if (i > 0) { // Remote reference, resolve immediately - if (remoteProvider_) { - if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i - 1)) { - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { - if (schema) - *schema = sc; - return true; - } - } - } - } - } - else if (s[i] == '#') { // Local reference, defer resolution - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const ValueType* nv = pointer.Get(document)) - if (HandleRefSchema(source, schema, *nv, document)) - return true; - - new (schemaRef_.template Push<SchemaRefEntry>()) SchemaRefEntry(source, pointer, schema, allocator_); - return true; - } - } - } - } - return false; - } - - const SchemaType* GetSchema(const PointerType& pointer) const { - for (const SchemaEntry* target = schemaMap_.template Bottom<SchemaEntry>(); target != schemaMap_.template End<SchemaEntry>(); ++target) - if (pointer == target->pointer) - return target->schema; - return 0; - } - - PointerType GetPointer(const SchemaType* schema) const { - for (const SchemaEntry* target = schemaMap_.template Bottom<SchemaEntry>(); target != schemaMap_.template End<SchemaEntry>(); ++target) - if (schema == target->schema) - return target->pointer; - return PointerType(); - } - - static const size_t kInitialSchemaMapSize = 64; - static const size_t kInitialSchemaRefSize = 64; - - IRemoteSchemaDocumentProviderType* remoteProvider_; - Allocator *allocator_; - Allocator *ownAllocator_; - const SchemaType* root_; //!< Root schema. - internal::Stack<Allocator> schemaMap_; // Stores created Pointer -> Schemas - internal::Stack<Allocator> schemaRef_; // Stores Pointer from $ref and schema which holds the $ref -}; - -//! GenericSchemaDocument using Value type. -typedef GenericSchemaDocument<Value> SchemaDocument; -//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. -typedef IGenericRemoteSchemaDocumentProvider<SchemaDocument> IRemoteSchemaDocumentProvider; - -/////////////////////////////////////////////////////////////////////////////// -// GenericSchemaValidator - -//! JSON Schema Validator. -/*! - A SAX style JSON schema validator. - It uses a \c GenericSchemaDocument to validate SAX events. - It delegates the incoming SAX events to an output handler. - The default output handler does nothing. - It can be reused multiple times by calling \c Reset(). - - \tparam SchemaDocumentType Type of schema document. - \tparam OutputHandler Type of output handler. Default handler does nothing. - \tparam StateAllocator Allocator for storing the internal validation states. -*/ -template < - typename SchemaDocumentType, - typename OutputHandler = BaseReaderHandler<typename SchemaDocumentType::SchemaType::EncodingType>, - typename StateAllocator = CrtAllocator> -class GenericSchemaValidator : - public internal::ISchemaStateFactory<typename SchemaDocumentType::SchemaType>, - public internal::ISchemaValidator -{ -public: - typedef typename SchemaDocumentType::SchemaType SchemaType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename SchemaType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - - //! Constructor without output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - outputHandler_(GetNullHandler()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif - { - } - - //! Constructor with output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - OutputHandler& outputHandler, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - outputHandler_(outputHandler), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif - { - } - - //! Destructor. - ~GenericSchemaValidator() { - Reset(); - RAPIDJSON_DELETE(ownStateAllocator_); - } - - //! Reset the internal states. - void Reset() { - while (!schemaStack_.Empty()) - PopSchema(); - documentStack_.Clear(); - valid_ = true; - } - - //! Checks whether the current state is valid. - // Implementation of ISchemaValidator - virtual bool IsValid() const { return valid_; } - - //! Gets the JSON pointer pointed to the invalid schema. - PointerType GetInvalidSchemaPointer() const { - return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); - } - - //! Gets the keyword of invalid schema. - const Ch* GetInvalidSchemaKeyword() const { - return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; - } - - //! Gets the JSON pointer pointed to the invalid value. - PointerType GetInvalidDocumentPointer() const { - return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom<Ch>(), documentStack_.GetSize() / sizeof(Ch)); - } - -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ -RAPIDJSON_MULTILINEMACRO_BEGIN\ - *documentStack_.template Push<Ch>() = '\0';\ - documentStack_.template Pop<Ch>(1);\ - internal::PrintInvalidDocument(documentStack_.template Bottom<Ch>());\ -RAPIDJSON_MULTILINEMACRO_END -#else -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() -#endif - -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ - if (!valid_) return false; \ - if (!BeginValue() || !CurrentSchema().method arg1) {\ - RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ - return valid_ = false;\ - } - -#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ - for (Context* context = schemaStack_.template Bottom<Context>(); context != schemaStack_.template End<Context>(); context++) {\ - if (context->hasher)\ - static_cast<HasherType*>(context->hasher)->method arg2;\ - if (context->validators)\ - for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ - static_cast<GenericSchemaValidator*>(context->validators[i_])->method arg2;\ - if (context->patternPropertiesValidators)\ - for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ - static_cast<GenericSchemaValidator*>(context->patternPropertiesValidators[i_])->method arg2;\ - } - -#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ - return valid_ = EndValue() && outputHandler_.method arg2 - -#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ - RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ - RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) - - bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } - bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } - bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } - bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } - bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } - bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } - bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } - bool RawNumber(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - bool String(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - - bool StartObject() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); - return valid_ = outputHandler_.StartObject(); - } - - bool Key(const Ch* str, SizeType len, bool copy) { - if (!valid_) return false; - AppendToken(str, len); - if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); - return valid_ = outputHandler_.Key(str, len, copy); - } - - bool EndObject(SizeType memberCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); - if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); - } - - bool StartArray() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); - return valid_ = outputHandler_.StartArray(); - } - - bool EndArray(SizeType elementCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); - if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); - } - -#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ -#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ -#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ -#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ - - // Implementation of ISchemaStateFactory<SchemaType> - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { - return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, -#if RAPIDJSON_SCHEMA_VERBOSE - depth_ + 1, -#endif - &GetStateAllocator()); - } - - virtual void DestroySchemaValidator(ISchemaValidator* validator) { - GenericSchemaValidator* v = static_cast<GenericSchemaValidator*>(validator); - v->~GenericSchemaValidator(); - StateAllocator::Free(v); - } - - virtual void* CreateHasher() { - return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); - } - - virtual uint64_t GetHashCode(void* hasher) { - return static_cast<HasherType*>(hasher)->GetHashCode(); - } - - virtual void DestroryHasher(void* hasher) { - HasherType* h = static_cast<HasherType*>(hasher); - h->~HasherType(); - StateAllocator::Free(h); - } - - virtual void* MallocState(size_t size) { - return GetStateAllocator().Malloc(size); - } - - virtual void FreeState(void* p) { - return StateAllocator::Free(p); - } - -private: - typedef typename SchemaType::Context Context; - typedef GenericValue<UTF8<>, StateAllocator> HashCodeArray; - typedef internal::Hasher<EncodingType, StateAllocator> HasherType; - - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - const SchemaType& root, -#if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth, -#endif - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(root), - outputHandler_(GetNullHandler()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(depth) -#endif - { - } - - StateAllocator& GetStateAllocator() { - if (!stateAllocator_) - stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator()); - return *stateAllocator_; - } - - bool BeginValue() { - if (schemaStack_.Empty()) - PushSchema(root_); - else { - if (CurrentContext().inArray) - internal::TokenHelper<internal::Stack<StateAllocator>, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); - - if (!CurrentSchema().BeginValue(CurrentContext())) - return false; - - SizeType count = CurrentContext().patternPropertiesSchemaCount; - const SchemaType** sa = CurrentContext().patternPropertiesSchemas; - typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; - bool valueUniqueness = CurrentContext().valueUniqueness; - if (CurrentContext().valueSchema) - PushSchema(*CurrentContext().valueSchema); - - if (count > 0) { - CurrentContext().objectPatternValidatorType = patternValidatorType; - ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; - SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; - va = static_cast<ISchemaValidator**>(MallocState(sizeof(ISchemaValidator*) * count)); - for (SizeType i = 0; i < count; i++) - va[validatorCount++] = CreateSchemaValidator(*sa[i]); - } - - CurrentContext().arrayUniqueness = valueUniqueness; - } - return true; - } - - bool EndValue() { - if (!CurrentSchema().EndValue(CurrentContext())) - return false; - -#if RAPIDJSON_SCHEMA_VERBOSE - GenericStringBuffer<EncodingType> sb; - schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); - - *documentStack_.template Push<Ch>() = '\0'; - documentStack_.template Pop<Ch>(1); - internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom<Ch>()); -#endif - - uint64_t h = CurrentContext().arrayUniqueness ? static_cast<HasherType*>(CurrentContext().hasher)->GetHashCode() : 0; - - PopSchema(); - - if (!schemaStack_.Empty()) { - Context& context = CurrentContext(); - if (context.valueUniqueness) { - HashCodeArray* a = static_cast<HashCodeArray*>(context.arrayElementHashCodes); - if (!a) - CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); - for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) - if (itr->GetUint64() == h) - RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); - a->PushBack(h, GetStateAllocator()); - } - } - - // Remove the last token of document pointer - while (!documentStack_.Empty() && *documentStack_.template Pop<Ch>(1) != '/') - ; - - return true; - } - - void AppendToken(const Ch* str, SizeType len) { - documentStack_.template Reserve<Ch>(1 + len * 2); // worst case all characters are escaped as two characters - *documentStack_.template PushUnsafe<Ch>() = '/'; - for (SizeType i = 0; i < len; i++) { - if (str[i] == '~') { - *documentStack_.template PushUnsafe<Ch>() = '~'; - *documentStack_.template PushUnsafe<Ch>() = '0'; - } - else if (str[i] == '/') { - *documentStack_.template PushUnsafe<Ch>() = '~'; - *documentStack_.template PushUnsafe<Ch>() = '1'; - } - else - *documentStack_.template PushUnsafe<Ch>() = str[i]; - } - } - - RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push<Context>()) Context(*this, &schema); } - - RAPIDJSON_FORCEINLINE void PopSchema() { - Context* c = schemaStack_.template Pop<Context>(1); - if (HashCodeArray* a = static_cast<HashCodeArray*>(c->arrayElementHashCodes)) { - a->~HashCodeArray(); - StateAllocator::Free(a); - } - c->~Context(); - } - - const SchemaType& CurrentSchema() const { return *schemaStack_.template Top<Context>()->schema; } - Context& CurrentContext() { return *schemaStack_.template Top<Context>(); } - const Context& CurrentContext() const { return *schemaStack_.template Top<Context>(); } - - static OutputHandler& GetNullHandler() { - static OutputHandler nullHandler; - return nullHandler; - } - - static const size_t kDefaultSchemaStackCapacity = 1024; - static const size_t kDefaultDocumentStackCapacity = 256; - const SchemaDocumentType* schemaDocument_; - const SchemaType& root_; - OutputHandler& outputHandler_; - StateAllocator* stateAllocator_; - StateAllocator* ownStateAllocator_; - internal::Stack<StateAllocator> schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) - internal::Stack<StateAllocator> documentStack_; //!< stack to store the current path of validating document (Ch) - bool valid_; -#if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth_; -#endif -}; - -typedef GenericSchemaValidator<SchemaDocument> SchemaValidator; - -/////////////////////////////////////////////////////////////////////////////// -// SchemaValidatingReader - -//! A helper class for parsing with validation. -/*! - This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). - - \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept. - \tparam SourceEncoding Encoding of the input stream. - \tparam SchemaDocumentType Type of schema document. - \tparam StackAllocator Allocator type for stack. -*/ -template < - unsigned parseFlags, - typename InputStream, - typename SourceEncoding, - typename SchemaDocumentType = SchemaDocument, - typename StackAllocator = CrtAllocator> -class SchemaValidatingReader { -public: - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename InputStream::Ch Ch; - - //! Constructor - /*! - \param is Input stream. - \param sd Schema document. - */ - SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} - - template <typename Handler> - bool operator()(Handler& handler) { - GenericReader<SourceEncoding, typename SchemaDocumentType::EncodingType, StackAllocator> reader; - GenericSchemaValidator<SchemaDocumentType, Handler> validator(sd_, handler); - parseResult_ = reader.template Parse<parseFlags>(is_, validator); - - isValid_ = validator.IsValid(); - if (isValid_) { - invalidSchemaPointer_ = PointerType(); - invalidSchemaKeyword_ = 0; - invalidDocumentPointer_ = PointerType(); - } - else { - invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); - invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); - invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); - } - - return parseResult_; - } - - const ParseResult& GetParseResult() const { return parseResult_; } - bool IsValid() const { return isValid_; } - const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } - const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } - const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } - -private: - InputStream& is_; - const SchemaDocumentType& sd_; - - ParseResult parseResult_; - PointerType invalidSchemaPointer_; - const Ch* invalidSchemaKeyword_; - PointerType invalidDocumentPointer_; - bool isValid_; -}; - -RAPIDJSON_NAMESPACE_END -RAPIDJSON_DIAG_POP - -#endif // RAPIDJSON_SCHEMA_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/stream.h b/ext/librethinkdbxx/src/rapidjson/stream.h deleted file mode 100644 index fef82c25..00000000 --- a/ext/librethinkdbxx/src/rapidjson/stream.h +++ /dev/null @@ -1,179 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#include "rapidjson.h" - -#ifndef RAPIDJSON_STREAM_H_ -#define RAPIDJSON_STREAM_H_ - -#include "encodings.h" - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// Stream - -/*! \class rapidjson::Stream - \brief Concept for reading and writing characters. - - For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). - - For write-only stream, only need to implement Put() and Flush(). - -\code -concept Stream { - typename Ch; //!< Character type of the stream. - - //! Read the current character from stream without moving the read cursor. - Ch Peek() const; - - //! Read the current character from stream and moving the read cursor to next character. - Ch Take(); - - //! Get the current read cursor. - //! \return Number of characters read from start. - size_t Tell(); - - //! Begin writing operation at the current read pointer. - //! \return The begin writer pointer. - Ch* PutBegin(); - - //! Write a character. - void Put(Ch c); - - //! Flush the buffer. - void Flush(); - - //! End the writing operation. - //! \param begin The begin write pointer returned by PutBegin(). - //! \return Number of characters written. - size_t PutEnd(Ch* begin); -} -\endcode -*/ - -//! Provides additional information for stream. -/*! - By using traits pattern, this type provides a default configuration for stream. - For custom stream, this type can be specialized for other configuration. - See TEST(Reader, CustomStringStream) in readertest.cpp for example. -*/ -template<typename Stream> -struct StreamTraits { - //! Whether to make local copy of stream for optimization during parsing. - /*! - By default, for safety, streams do not use local copy optimization. - Stream that can be copied fast should specialize this, like StreamTraits<StringStream>. - */ - enum { copyOptimization = 0 }; -}; - -//! Reserve n characters for writing to a stream. -template<typename Stream> -inline void PutReserve(Stream& stream, size_t count) { - (void)stream; - (void)count; -} - -//! Write character to a stream, presuming buffer is reserved. -template<typename Stream> -inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { - stream.Put(c); -} - -//! Put N copies of a character to a stream. -template<typename Stream, typename Ch> -inline void PutN(Stream& stream, Ch c, size_t n) { - PutReserve(stream, n); - for (size_t i = 0; i < n; i++) - PutUnsafe(stream, c); -} - -/////////////////////////////////////////////////////////////////////////////// -// StringStream - -//! Read-only string stream. -/*! \note implements Stream concept -*/ -template <typename Encoding> -struct GenericStringStream { - typedef typename Encoding::Ch Ch; - - GenericStringStream(const Ch *src) : src_(src), head_(src) {} - - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return static_cast<size_t>(src_ - head_); } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. -}; - -template <typename Encoding> -struct StreamTraits<GenericStringStream<Encoding> > { - enum { copyOptimization = 1 }; -}; - -//! String stream with UTF8 encoding. -typedef GenericStringStream<UTF8<> > StringStream; - -/////////////////////////////////////////////////////////////////////////////// -// InsituStringStream - -//! A read-write string stream. -/*! This string stream is particularly designed for in-situ parsing. - \note implements Stream concept -*/ -template <typename Encoding> -struct GenericInsituStringStream { - typedef typename Encoding::Ch Ch; - - GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} - - // Read - Ch Peek() { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() { return static_cast<size_t>(src_ - head_); } - - // Write - void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } - - Ch* PutBegin() { return dst_ = src_; } - size_t PutEnd(Ch* begin) { return static_cast<size_t>(dst_ - begin); } - void Flush() {} - - Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } - void Pop(size_t count) { dst_ -= count; } - - Ch* src_; - Ch* dst_; - Ch* head_; -}; - -template <typename Encoding> -struct StreamTraits<GenericInsituStringStream<Encoding> > { - enum { copyOptimization = 1 }; -}; - -//! Insitu string stream with UTF8 encoding. -typedef GenericInsituStringStream<UTF8<> > InsituStringStream; - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_STREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/stringbuffer.h b/ext/librethinkdbxx/src/rapidjson/stringbuffer.h deleted file mode 100644 index 78f34d20..00000000 --- a/ext/librethinkdbxx/src/rapidjson/stringbuffer.h +++ /dev/null @@ -1,117 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_STRINGBUFFER_H_ -#define RAPIDJSON_STRINGBUFFER_H_ - -#include "stream.h" -#include "internal/stack.h" - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS -#include <utility> // std::move -#endif - -#include "internal/stack.h" - -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Represents an in-memory output stream. -/*! - \tparam Encoding Encoding of the stream. - \tparam Allocator type for allocating memory buffer. - \note implements Stream concept -*/ -template <typename Encoding, typename Allocator = CrtAllocator> -class GenericStringBuffer { -public: - typedef typename Encoding::Ch Ch; - - GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} - GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { - if (&rhs != this) - stack_ = std::move(rhs.stack_); - return *this; - } -#endif - - void Put(Ch c) { *stack_.template Push<Ch>() = c; } - void PutUnsafe(Ch c) { *stack_.template PushUnsafe<Ch>() = c; } - void Flush() {} - - void Clear() { stack_.Clear(); } - void ShrinkToFit() { - // Push and pop a null terminator. This is safe. - *stack_.template Push<Ch>() = '\0'; - stack_.ShrinkToFit(); - stack_.template Pop<Ch>(1); - } - - void Reserve(size_t count) { stack_.template Reserve<Ch>(count); } - Ch* Push(size_t count) { return stack_.template Push<Ch>(count); } - Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe<Ch>(count); } - void Pop(size_t count) { stack_.template Pop<Ch>(count); } - - const Ch* GetString() const { - // Push and pop a null terminator. This is safe. - *stack_.template Push<Ch>() = '\0'; - stack_.template Pop<Ch>(1); - - return stack_.template Bottom<Ch>(); - } - - size_t GetSize() const { return stack_.GetSize(); } - - static const size_t kDefaultCapacity = 256; - mutable internal::Stack<Allocator> stack_; - -private: - // Prohibit copy constructor & assignment operator. - GenericStringBuffer(const GenericStringBuffer&); - GenericStringBuffer& operator=(const GenericStringBuffer&); -}; - -//! String buffer with UTF8 encoding -typedef GenericStringBuffer<UTF8<> > StringBuffer; - -template<typename Encoding, typename Allocator> -inline void PutReserve(GenericStringBuffer<Encoding, Allocator>& stream, size_t count) { - stream.Reserve(count); -} - -template<typename Encoding, typename Allocator> -inline void PutUnsafe(GenericStringBuffer<Encoding, Allocator>& stream, typename Encoding::Ch c) { - stream.PutUnsafe(c); -} - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(GenericStringBuffer<UTF8<> >& stream, char c, size_t n) { - std::memset(stream.stack_.Push<char>(n), c, n * sizeof(c)); -} - -RAPIDJSON_NAMESPACE_END - -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/writer.h b/ext/librethinkdbxx/src/rapidjson/writer.h deleted file mode 100644 index 7d0610eb..00000000 --- a/ext/librethinkdbxx/src/rapidjson/writer.h +++ /dev/null @@ -1,609 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_WRITER_H_ -#define RAPIDJSON_WRITER_H_ - -#include "stream.h" -#include "internal/stack.h" -#include "internal/strfunc.h" -#include "internal/dtoa.h" -#include "internal/itoa.h" -#include "stringbuffer.h" -#include <new> // placement new - -#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) -#include <intrin.h> -#pragma intrinsic(_BitScanForward) -#endif -#ifdef RAPIDJSON_SSE42 -#include <nmmintrin.h> -#elif defined(RAPIDJSON_SSE2) -#include <emmintrin.h> -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(unreachable-code) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// WriteFlag - -/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS - \ingroup RAPIDJSON_CONFIG - \brief User-defined kWriteDefaultFlags definition. - - User can define this as any \c WriteFlag combinations. -*/ -#ifndef RAPIDJSON_WRITE_DEFAULT_FLAGS -#define RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags -#endif - -//! Combination of writeFlags -enum WriteFlag { - kWriteNoFlags = 0, //!< No flags are set. - kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. - kWriteNanAndInfFlag = 2, //!< Allow writing of Inf, -Inf and NaN. - kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS -}; - -//! JSON writer -/*! Writer implements the concept Handler. - It generates JSON text by events to an output os. - - User may programmatically calls the functions of a writer to generate JSON text. - - On the other side, a writer can also be passed to objects that generates events, - - for example Reader::Parse() and Document::Accept(). - - \tparam OutputStream Type of output stream. - \tparam SourceEncoding Encoding of source string. - \tparam TargetEncoding Encoding of output stream. - \tparam StackAllocator Type of allocator for allocating memory of stack. - \note implements Handler concept -*/ -template<typename OutputStream, typename SourceEncoding = UTF8<>, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> -class Writer { -public: - typedef typename SourceEncoding::Ch Ch; - - static const int kDefaultMaxDecimalPlaces = 324; - - //! Constructor - /*! \param os Output stream. - \param stackAllocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of stack. - */ - explicit - Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} - - explicit - Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} - - //! Reset the writer with a new stream. - /*! - This function reset the writer with a new stream and default settings, - in order to make a Writer object reusable for output multiple JSONs. - - \param os New output stream. - \code - Writer<OutputStream> writer(os1); - writer.StartObject(); - // ... - writer.EndObject(); - - writer.Reset(os2); - writer.StartObject(); - // ... - writer.EndObject(); - \endcode - */ - void Reset(OutputStream& os) { - os_ = &os; - hasRoot_ = false; - level_stack_.Clear(); - } - - //! Checks whether the output is a complete JSON. - /*! - A complete JSON has a complete root object or array. - */ - bool IsComplete() const { - return hasRoot_ && level_stack_.Empty(); - } - - int GetMaxDecimalPlaces() const { - return maxDecimalPlaces_; - } - - //! Sets the maximum number of decimal places for double output. - /*! - This setting truncates the output with specified number of decimal places. - - For example, - - \code - writer.SetMaxDecimalPlaces(3); - writer.StartArray(); - writer.Double(0.12345); // "0.123" - writer.Double(0.0001); // "0.0" - writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) - writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) - writer.EndArray(); - \endcode - - The default setting does not truncate any decimal places. You can restore to this setting by calling - \code - writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); - \endcode - */ - void SetMaxDecimalPlaces(int maxDecimalPlaces) { - maxDecimalPlaces_ = maxDecimalPlaces; - } - - /*!@name Implementation of Handler - \see Handler - */ - //@{ - - bool Null() { Prefix(kNullType); return WriteNull(); } - bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } - bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } - bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } - bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } - bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } - - //! Writes the given \c double value to the stream - /*! - \param d The value to be written. - \return Whether it is succeed. - */ - bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } - - bool RawNumber(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - Prefix(kNumberType); - return WriteString(str, length); - } - - bool String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - Prefix(kStringType); - return WriteString(str, length); - } - -#if RAPIDJSON_HAS_STDSTRING - bool String(const std::basic_string<Ch>& str) { - return String(str.data(), SizeType(str.size())); - } -#endif - - bool StartObject() { - Prefix(kObjectType); - new (level_stack_.template Push<Level>()) Level(false); - return WriteStartObject(); - } - - bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(!level_stack_.template Top<Level>()->inArray); - level_stack_.template Pop<Level>(1); - bool ret = WriteEndObject(); - if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text - os_->Flush(); - return ret; - } - - bool StartArray() { - Prefix(kArrayType); - new (level_stack_.template Push<Level>()) Level(true); - return WriteStartArray(); - } - - bool EndArray(SizeType elementCount = 0) { - (void)elementCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(level_stack_.template Top<Level>()->inArray); - level_stack_.template Pop<Level>(1); - bool ret = WriteEndArray(); - if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text - os_->Flush(); - return ret; - } - //@} - - /*! @name Convenience extensions */ - //@{ - - //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - - //@} - - //! Write a raw JSON value. - /*! - For user to write a stringified JSON as a value. - - \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. - \param length Length of the json. - \param type Type of the root of json. - */ - bool RawValue(const Ch* json, size_t length, Type type) { Prefix(type); return WriteRawValue(json, length); } - -protected: - //! Information for each nested level - struct Level { - Level(bool inArray_) : valueCount(0), inArray(inArray_) {} - size_t valueCount; //!< number of values in this level - bool inArray; //!< true if in array, otherwise in object - }; - - static const size_t kDefaultLevelDepth = 32; - - bool WriteNull() { - PutReserve(*os_, 4); - PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; - } - - bool WriteBool(bool b) { - if (b) { - PutReserve(*os_, 4); - PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); - } - else { - PutReserve(*os_, 5); - PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); - } - return true; - } - - bool WriteInt(int i) { - char buffer[11]; - const char* end = internal::i32toa(i, buffer); - PutReserve(*os_, static_cast<size_t>(end - buffer)); - for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast<typename TargetEncoding::Ch>(*p)); - return true; - } - - bool WriteUint(unsigned u) { - char buffer[10]; - const char* end = internal::u32toa(u, buffer); - PutReserve(*os_, static_cast<size_t>(end - buffer)); - for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast<typename TargetEncoding::Ch>(*p)); - return true; - } - - bool WriteInt64(int64_t i64) { - char buffer[21]; - const char* end = internal::i64toa(i64, buffer); - PutReserve(*os_, static_cast<size_t>(end - buffer)); - for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast<typename TargetEncoding::Ch>(*p)); - return true; - } - - bool WriteUint64(uint64_t u64) { - char buffer[20]; - char* end = internal::u64toa(u64, buffer); - PutReserve(*os_, static_cast<size_t>(end - buffer)); - for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast<typename TargetEncoding::Ch>(*p)); - return true; - } - - bool WriteDouble(double d) { - if (internal::Double(d).IsNanOrInf()) { - if (!(writeFlags & kWriteNanAndInfFlag)) - return false; - if (internal::Double(d).IsNan()) { - PutReserve(*os_, 3); - PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); - return true; - } - if (internal::Double(d).Sign()) { - PutReserve(*os_, 9); - PutUnsafe(*os_, '-'); - } - else - PutReserve(*os_, 8); - PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); - PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); - return true; - } - - char buffer[25]; - char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); - PutReserve(*os_, static_cast<size_t>(end - buffer)); - for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast<typename TargetEncoding::Ch>(*p)); - return true; - } - - bool WriteString(const Ch* str, SizeType length) { - static const typename TargetEncoding::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; - static const char escape[256] = { -#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - //0 1 2 3 4 5 6 7 8 9 A B C D E F - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 - 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 - Z16, Z16, // 30~4F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF -#undef Z16 - }; - - if (TargetEncoding::supportUnicode) - PutReserve(*os_, 2 + length * 6); // "\uxxxx..." - else - PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." - - PutUnsafe(*os_, '\"'); - GenericStringStream<SourceEncoding> is(str); - while (ScanWriteUnescapedString(is, length)) { - const Ch c = is.Peek(); - if (!TargetEncoding::supportUnicode && static_cast<unsigned>(c) >= 0x80) { - // Unicode escaping - unsigned codepoint; - if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) - return false; - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, 'u'); - if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { - PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); - } - else { - RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); - // Surrogate pair - unsigned s = codepoint - 0x010000; - unsigned lead = (s >> 10) + 0xD800; - unsigned trail = (s & 0x3FF) + 0xDC00; - PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(lead ) & 15]); - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, 'u'); - PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(trail ) & 15]); - } - } - else if ((sizeof(Ch) == 1 || static_cast<unsigned>(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast<unsigned char>(c)])) { - is.Take(); - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, static_cast<typename TargetEncoding::Ch>(escape[static_cast<unsigned char>(c)])); - if (escape[static_cast<unsigned char>(c)] == 'u') { - PutUnsafe(*os_, '0'); - PutUnsafe(*os_, '0'); - PutUnsafe(*os_, hexDigits[static_cast<unsigned char>(c) >> 4]); - PutUnsafe(*os_, hexDigits[static_cast<unsigned char>(c) & 0xF]); - } - } - else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? - Transcoder<SourceEncoding, TargetEncoding>::Validate(is, *os_) : - Transcoder<SourceEncoding, TargetEncoding>::TranscodeUnsafe(is, *os_)))) - return false; - } - PutUnsafe(*os_, '\"'); - return true; - } - - bool ScanWriteUnescapedString(GenericStringStream<SourceEncoding>& is, size_t length) { - return RAPIDJSON_LIKELY(is.Tell() < length); - } - - bool WriteStartObject() { os_->Put('{'); return true; } - bool WriteEndObject() { os_->Put('}'); return true; } - bool WriteStartArray() { os_->Put('['); return true; } - bool WriteEndArray() { os_->Put(']'); return true; } - - bool WriteRawValue(const Ch* json, size_t length) { - PutReserve(*os_, length); - for (size_t i = 0; i < length; i++) { - RAPIDJSON_ASSERT(json[i] != '\0'); - PutUnsafe(*os_, json[i]); - } - return true; - } - - void Prefix(Type type) { - (void)type; - if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root - Level* level = level_stack_.template Top<Level>(); - if (level->valueCount > 0) { - if (level->inArray) - os_->Put(','); // add comma if it is not the first element in array - else // in object - os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else { - RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. - hasRoot_ = true; - } - } - - OutputStream* os_; - internal::Stack<StackAllocator> level_stack_; - int maxDecimalPlaces_; - bool hasRoot_; - -private: - // Prohibit copy constructor & assignment operator. - Writer(const Writer&); - Writer& operator=(const Writer&); -}; - -// Full specialization for StringStream to prevent memory copying - -template<> -inline bool Writer<StringBuffer>::WriteInt(int i) { - char *buffer = os_->Push(11); - const char* end = internal::i32toa(i, buffer); - os_->Pop(static_cast<size_t>(11 - (end - buffer))); - return true; -} - -template<> -inline bool Writer<StringBuffer>::WriteUint(unsigned u) { - char *buffer = os_->Push(10); - const char* end = internal::u32toa(u, buffer); - os_->Pop(static_cast<size_t>(10 - (end - buffer))); - return true; -} - -template<> -inline bool Writer<StringBuffer>::WriteInt64(int64_t i64) { - char *buffer = os_->Push(21); - const char* end = internal::i64toa(i64, buffer); - os_->Pop(static_cast<size_t>(21 - (end - buffer))); - return true; -} - -template<> -inline bool Writer<StringBuffer>::WriteUint64(uint64_t u) { - char *buffer = os_->Push(20); - const char* end = internal::u64toa(u, buffer); - os_->Pop(static_cast<size_t>(20 - (end - buffer))); - return true; -} - -template<> -inline bool Writer<StringBuffer>::WriteDouble(double d) { - if (internal::Double(d).IsNanOrInf()) { - // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). - if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) - return false; - if (internal::Double(d).IsNan()) { - PutReserve(*os_, 3); - PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); - return true; - } - if (internal::Double(d).Sign()) { - PutReserve(*os_, 9); - PutUnsafe(*os_, '-'); - } - else - PutReserve(*os_, 8); - PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); - PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); - return true; - } - - char *buffer = os_->Push(25); - char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); - os_->Pop(static_cast<size_t>(25 - (end - buffer))); - return true; -} - -#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) -template<> -inline bool Writer<StringBuffer>::ScanWriteUnescapedString(StringStream& is, size_t length) { - if (length < 16) - return RAPIDJSON_LIKELY(is.Tell() < length); - - if (!RAPIDJSON_LIKELY(is.Tell() < length)) - return false; - - const char* p = is.src_; - const char* end = is.head_ + length; - const char* nextAligned = reinterpret_cast<const char*>((reinterpret_cast<size_t>(p) + 15) & static_cast<size_t>(~15)); - const char* endAligned = reinterpret_cast<const char*>(reinterpret_cast<size_t>(end) & static_cast<size_t>(~15)); - if (nextAligned > end) - return true; - - while (p != nextAligned) - if (*p < 0x20 || *p == '\"' || *p == '\\') { - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); - } - else - os_->PutUnsafe(*p++); - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast<const __m128i *>(&space[0])); - - for (; p != endAligned; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast<const __m128i *>(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast<unsigned short>(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - SizeType len; -#ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - len = offset; -#else - len = static_cast<SizeType>(__builtin_ffs(r) - 1); -#endif - char* q = reinterpret_cast<char*>(os_->PushUnsafe(len)); - for (size_t i = 0; i < len; i++) - q[i] = p[i]; - - p += len; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); - } - - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); -} -#endif // defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) - -RAPIDJSON_NAMESPACE_END - -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/term.cc b/ext/librethinkdbxx/src/term.cc deleted file mode 100644 index 711ef27d..00000000 --- a/ext/librethinkdbxx/src/term.cc +++ /dev/null @@ -1,285 +0,0 @@ -#include <cstdlib> -#include <set> - -#include "term.h" -#include "json_p.h" - -namespace RethinkDB { - -using TT = Protocol::Term::TermType; - -struct { - Datum operator() (const Array& array) { - Array copy; - copy.reserve(array.size()); - for (const auto& it : array) { - copy.emplace_back(it.apply<Datum>(*this)); - } - return Datum(Array{TT::MAKE_ARRAY, std::move(copy)}); - } - Datum operator() (const Object& object) { - Object copy; - for (const auto& it : object) { - copy.emplace(it.first, it.second.apply<Datum>(*this)); - } - return std::move(copy); - } - template<class T> - Datum operator() (T&& atomic) { - return Datum(std::forward<T>(atomic)); - } -} datum_to_term; - -Term::Term(Datum&& datum_) : datum(datum_.apply<Datum>(datum_to_term)) { } -Term::Term(const Datum& datum_) : datum(datum_.apply<Datum>(datum_to_term)) { } - -Term::Term(Term&& orig, OptArgs&& new_optargs) : datum(Nil()) { - Datum* cur = orig.datum.get_nth(2); - Object optargs; - free_vars = std::move(orig.free_vars); - if (cur) { - optargs = std::move(cur->extract_object()); - } - for (auto& it : new_optargs) { - optargs.emplace(std::move(it.first), alpha_rename(std::move(it.second))); - } - datum = Array{ std::move(orig.datum.extract_nth(0)), std::move(orig.datum.extract_nth(1)), std::move(optargs) }; -} - -Term nil() { - return Term(Nil()); -} - -Cursor Term::run(Connection& conn, OptArgs&& opts) { - if (!free_vars.empty()) { - throw Error("run: term has free variables"); - } - - return conn.start_query(this, std::move(opts)); -} - -struct { - Datum operator() (Object&& object, const std::map<int, int>& subst, bool) { - Object ret; - for (auto& it : object) { - ret.emplace(std::move(it.first), std::move(it.second).apply<Datum>(*this, subst, false)); - } - return ret; - } - Datum operator() (Array&& array, const std::map<int, int>& subst, bool args) { - if (!args) { - double cmd = array[0].extract_number(); - if (cmd == static_cast<int>(TT::VAR)) { - double var = array[1].extract_nth(0).extract_number(); - auto it = subst.find(static_cast<int>(var)); - if (it != subst.end()) { - return Array{ TT::VAR, { it->second }}; - } - } - if (array.size() == 2) { - return Array{ std::move(array[0]), std::move(array[1]).apply<Datum>(*this, subst, true) }; - } else { - return Array{ - std::move(array[0]), - std::move(array[1]).apply<Datum>(*this, subst, true), - std::move(array[2]).apply<Datum>(*this, subst, false) }; - } - } else { - Array ret; - for (auto& it : array) { - ret.emplace_back(std::move(it).apply<Datum>(*this, subst, false)); - } - return ret; - } - } - template <class T> - Datum operator() (T&& a, const std::map<int, int>&, bool) { - return std::move(a); - } -} alpha_renamer; - -static int new_var_id(const std::map<int, int*>& vars) { - while (true) { - int id = gen_var_id(); - if (vars.find(id) == vars.end()) { - return id; - } - } -} - -Datum Term::alpha_rename(Term&& term) { - if (free_vars.empty()) { - free_vars = std::move(term.free_vars); - return std::move(term.datum); - } - - std::map<int, int> subst; - for (auto it = term.free_vars.begin(); it != term.free_vars.end(); ++it) { - auto var = free_vars.find(it->first); - if (var == free_vars.end()) { - free_vars.emplace(it->first, it->second); - } else if (var->second != it->second) { - int id = new_var_id(free_vars); - subst.emplace(it->first, id); - free_vars.emplace(id, it->second); - } - } - if (subst.empty()) { - return std::move(term.datum); - } else { - return term.datum.apply<Datum>(alpha_renamer, subst, false); - } -} - -int gen_var_id() { - return ::random() % (1<<30); -} - -C0_IMPL(db_list, DB_LIST) -C0_IMPL(table_list, TABLE_LIST) -C0_IMPL(random, RANDOM) -C0_IMPL(now, NOW) -C0_IMPL(range, RANGE) -C0_IMPL(error, ERROR) -C0_IMPL(uuid, UUID) -C0_IMPL(literal, LITERAL) -CO0_IMPL(wait, WAIT) -C0_IMPL(rebalance, REBALANCE) -CO0_IMPL(random, RANDOM) - -Term row(TT::IMPLICIT_VAR, {}); -Term minval(TT::MINVAL, {}); -Term maxval(TT::MAXVAL, {}); - -Term binary(const std::string& data) { - return expr(Binary(data)); -} - -Term binary(std::string&& data) { - return expr(Binary(data)); -} - -Term binary(const char* data) { - return expr(Binary(data)); -} - -struct { - bool operator() (const Object& object) { - for (const auto& it : object) { - if (it.second.apply<bool>(*this)) { - return true; - } - } - return false; - } - bool operator() (const Array& array) { - int type = *array[0].get_number(); - if (type == static_cast<int>(TT::IMPLICIT_VAR)) { - return true; - } - if (type == static_cast<int>(TT::FUNC)) { - return false; - } - for (const auto& it : *array[1].get_array()) { - if (it.apply<bool>(*this)) { - return true; - } - } - if (array.size() == 3) { - return array[2].apply<bool>(*this); - } else { - return false; - } - } - template <class T> - bool operator() (T) { - return false; - } -} needs_func_wrap; - -Term Term::func_wrap(Term&& term) { - if (term.datum.apply<bool>(needs_func_wrap)) { - return Term(TT::FUNC, {expr(Array{new_var_id(term.free_vars)}), std::move(term)}); - } - return term; -} - -Term Term::func_wrap(const Term& term) { - if (term.datum.apply<bool>(needs_func_wrap)) { - // TODO return Term(TT::FUNC, {expr(Array{new_var_id(Term.free_vars)}), Term.copy()}); - return Term(Nil()); - } - return term; -} - -Term Term::make_object(std::vector<Term>&& args) { - if (args.size() % 2 != 0) { - return Term(TT::OBJECT, std::move(args)); - } - std::set<std::string> keys; - for (auto it = args.begin(); it != args.end() && it + 1 != args.end(); it += 2) { - std::string* key = it->datum.get_string(); - if (!key || keys.count(*key)) { - return Term(TT::OBJECT, std::move(args)); - } - keys.insert(*key); - } - Term ret{Nil()}; - Object object; - for (auto it = args.begin(); it != args.end(); it += 2) { - std::string* key = it->datum.get_string(); - object.emplace(std::move(*key), ret.alpha_rename(std::move(*(it + 1)))); - } - ret.datum = std::move(object); - return ret; -} - -Term Term::make_binary(Term&& term) { - std::string* string = term.datum.get_string(); - if (string) { - return expr(Binary(std::move(*string))); - } - return Term(TT::BINARY, std::vector<Term>{term}); -} - -Term::Term(OptArgs&& optargs) : datum(Nil()) { - Object oargs; - for (auto& it : optargs) { - oargs.emplace(it.first, alpha_rename(std::move(it.second))); - } - datum = std::move(oargs); -} - -OptArgs optargs() { - return OptArgs{}; -} - -Term january(TT::JANUARY, {}); -Term february(TT::FEBRUARY, {}); -Term march(TT::MARCH, {}); -Term april(TT::APRIL, {}); -Term may(TT::MAY, {}); -Term june(TT::JUNE, {}); -Term july(TT::JULY, {}); -Term august(TT::AUGUST, {}); -Term september(TT::SEPTEMBER, {}); -Term october(TT::OCTOBER, {}); -Term november(TT::NOVEMBER, {}); -Term december(TT::DECEMBER, {}); -Term monday(TT::MONDAY, {}); -Term tuesday(TT::TUESDAY, {}); -Term wednesday(TT::WEDNESDAY, {}); -Term thursday(TT::THURSDAY, {}); -Term friday(TT::FRIDAY, {}); -Term saturday(TT::SATURDAY, {}); -Term sunday(TT::SUNDAY, {}); - -Term Term::copy() const { - return *this; -} - -Datum Term::get_datum() const { - return datum; -} - -} diff --git a/ext/librethinkdbxx/src/term.h b/ext/librethinkdbxx/src/term.h deleted file mode 100644 index cdee5ec4..00000000 --- a/ext/librethinkdbxx/src/term.h +++ /dev/null @@ -1,592 +0,0 @@ -#pragma once - -#include "datum.h" -#include "connection.h" -#include "protocol_defs.h" -#include "cursor.h" - -namespace RethinkDB { - -using TT = Protocol::Term::TermType; - -class Term; -class Var; - -// An alias for the Term constructor -template <class T> -Term expr(T&&); - -int gen_var_id(); - -// Can be used as the last argument to some ReQL commands that expect named arguments -using OptArgs = std::map<std::string, Term>; - -// Represents a ReQL Term (RethinkDB Query Language) -// Designed to be used with r-value *this -class Term { -public: - Term(const Term& other) = default; - Term(Term&& other) = default; - Term& operator= (const Term& other) = default; - Term& operator= (Term&& other) = default; - - explicit Term(Datum&&); - explicit Term(const Datum&); - explicit Term(OptArgs&&); - - // Create a copy of the Term - Term copy() const; - - Term(std::function<Term()> f) : datum(Nil()) { set_function<std::function<Term()>>(f); } - Term(std::function<Term(Var)> f) : datum(Nil()) { set_function<std::function<Term(Var)>, 0>(f); } - Term(std::function<Term(Var, Var)> f) : datum(Nil()) { set_function<std::function<Term(Var, Var)>, 0, 1>(f); } - Term(std::function<Term(Var, Var, Var)> f) : datum(Nil()) { set_function<std::function<Term(Var, Var, Var)>, 0, 1, 2>(f); } - Term(Protocol::Term::TermType type, std::vector<Term>&& args) : datum(Array()) { - Array dargs; - for (auto& it : args) { - dargs.emplace_back(alpha_rename(std::move(it))); - } - datum = Datum(Array{ type, std::move(dargs) }); - } - - Term(Protocol::Term::TermType type, std::vector<Term>&& args, OptArgs&& optargs) : datum(Array()) { - Array dargs; - for (auto& it : args) { - dargs.emplace_back(alpha_rename(std::move(it))); - } - Object oargs; - for (auto& it : optargs) { - oargs.emplace(it.first, alpha_rename(std::move(it.second))); - } - datum = Array{ type, std::move(dargs), std::move(oargs) }; - } - - // Used internally to support row - static Term func_wrap(Term&&); - static Term func_wrap(const Term&); - - - // These macros are used to define most ReQL commands - // * Cn represents a method with n arguments - // * COn represents a method with n arguments and optional named arguments - // * C_ represents a method with any number of arguments - // Each method is implemented twice, once with r-value *this, and once with const *this - // The third argument, wrap, allows converting arguments into functions if they contain row - -#define C0(name, type) \ - Term name() && { return Term(TT::type, std::vector<Term>{ std::move(*this) }); } \ - Term name() const & { return Term(TT::type, std::vector<Term>{ *this }); } -#define C1(name, type, wrap) \ - template <class T> \ - Term name(T&& a) && { return Term(TT::type, std::vector<Term>{ std::move(*this), wrap(expr(std::forward<T>(a))) }); } \ - template <class T> \ - Term name(T&& a) const & { return Term(TT::type, std::vector<Term>{ *this, wrap(expr(std::forward<T>(a))) }); } -#define C2(name, type) \ - template <class T, class U> Term name(T&& a, U&& b) && { \ - return Term(TT::type, std::vector<Term>{ std::move(*this), \ - expr(std::forward<T>(a)), expr(std::forward<U>(b)) }); } \ - template <class T, class U> Term name(T&& a, U&& b) const & { \ - return Term(TT::type, std::vector<Term>{ *this, \ - expr(std::forward<T>(a)), expr(std::forward<U>(b)) }); } -#define C_(name, type, wrap) \ - template <class ...T> Term name(T&& ...a) && { \ - return Term(TT::type, std::vector<Term>{ std::move(*this), \ - wrap(expr(std::forward<T>(a)))... }); } \ - template <class ...T> Term name(T&& ...a) const & { \ - return Term(TT::type, std::vector<Term>{ *this, \ - wrap(expr(std::forward<T>(a)))... }); } -#define CO0(name, type) \ - Term name(OptArgs&& optarg = {}) && { \ - return Term(TT::type, std::vector<Term>{ std::move(*this) }, std::move(optarg)); } \ - Term name(OptArgs&& optarg = {}) const & { \ - return Term(TT::type, std::vector<Term>{ *this }, std::move(optarg)); } -#define CO1(name, type, wrap) \ - template <class T> Term name(T&& a, OptArgs&& optarg = {}) && { \ - return Term(TT::type, std::vector<Term>{ std::move(*this), \ - wrap(expr(std::forward<T>(a))) }, std::move(optarg)); } \ - template <class T> Term name(T&& a, OptArgs&& optarg = {}) const & { \ - return Term(TT::type, std::vector<Term>{ *this, \ - wrap(expr(std::forward<T>(a))) }, std::move(optarg)); } -#define CO2(name, type, wrap) \ - template <class T, class U> Term name(T&& a, U&& b, OptArgs&& optarg = {}) && { \ - return Term(TT::type, std::vector<Term>{ std::move(*this), \ - wrap(expr(std::forward<T>(a))), wrap(expr(std::forward<U>(b))) }, std::move(optarg)); } \ - template <class T, class U> Term name(T&& a, U&& b, OptArgs&& optarg = {}) const & { \ - return Term(TT::type, std::vector<Term>{ *this, \ - wrap(expr(std::forward<T>(a))), wrap(expr(std::forward<U>(b))) }, std::move(optarg)); } -#define CO3(name, type, wrap) \ - template <class T, class U, class V> Term name(T&& a, U&& b, V&& c, OptArgs&& optarg = {}) && { \ - return Term(TT::type, std::vector<Term>{ std::move(*this), \ - wrap(expr(std::forward<T>(a))), wrap(expr(std::forward<U>(b))), \ - wrap(expr(std::forward<V>(c))) }, std::move(optarg)); } \ - template <class T, class U, class V> Term name(T&& a, U&& b, V&& c, OptArgs&& optarg = {}) const & { \ - return Term(TT::type, std::vector<Term>{ *this, \ - wrap(expr(std::forward<T>(a))), wrap(expr(std::forward<U>(b))), \ - wrap(expr(std::forward<V>(c)))}, std::move(optarg)); } -#define CO4(name, type, wrap) \ - template <class T, class U, class V, class W> Term name(T&& a, U&& b, V&& c, W&& d, OptArgs&& optarg = {}) && { \ - return Term(TT::type, std::vector<Term>{ std::move(*this), \ - wrap(expr(std::forward<T>(a))), wrap(expr(std::forward<U>(b))), \ - wrap(expr(std::forward<V>(c))), wrap(expr(std::forward<W>(d))) }, std::move(optarg)); } \ - template <class T, class U, class V, class W> Term name(T&& a, U&& b, V&& c, W&& d, OptArgs&& optarg = {}) const & { \ - return Term(TT::type, std::vector<Term>{ *this, \ - wrap(expr(std::forward<T>(a))), wrap(expr(std::forward<U>(b))), \ - wrap(expr(std::forward<V>(c))), wrap(expr(std::forward<W>(d))) }, std::move(optarg)); } -#define CO_(name, type, wrap) \ - C_(name, type, wrap) \ - CO0(name, type) \ - CO1(name, type, wrap) \ - CO2(name, type, wrap) \ - CO3(name, type, wrap) \ - CO4(name, type, wrap) -#define no_wrap(x) x - - CO1(table_create, TABLE_CREATE, no_wrap) - C1(table_drop, TABLE_DROP, no_wrap) - C0(table_list, TABLE_LIST) - CO1(index_create, INDEX_CREATE, no_wrap) - CO2(index_create, INDEX_CREATE, func_wrap) - C1(index_drop, INDEX_DROP, no_wrap) - C0(index_list, INDEX_LIST) - CO2(index_rename, INDEX_RENAME, no_wrap) - C_(index_status, INDEX_STATUS, no_wrap) - C_(index_wait, INDEX_WAIT, no_wrap) - CO0(changes, CHANGES) - CO1(insert, INSERT, no_wrap) - CO1(update, UPDATE, func_wrap) - CO1(replace, REPLACE, func_wrap) - CO0(delete_, DELETE) - C0(sync, SYNC) - CO1(table, TABLE, no_wrap) - C1(get, GET, no_wrap) - CO_(get_all, GET_ALL, no_wrap) - CO2(between, BETWEEN, no_wrap) - CO1(filter, FILTER, func_wrap) - C2(inner_join, INNER_JOIN) - C2(outer_join, OUTER_JOIN) - CO2(eq_join, EQ_JOIN, func_wrap) - C0(zip, ZIP) - C_(map, MAP, func_wrap) - C_(with_fields, WITH_FIELDS, no_wrap) - C1(concat_map, CONCAT_MAP, func_wrap) - CO_(order_by, ORDER_BY, func_wrap) - C1(skip, SKIP, no_wrap) - C1(limit, LIMIT, no_wrap) - CO1(slice, SLICE, no_wrap) - CO2(slice, SLICE, no_wrap) - C1(nth, NTH, no_wrap) - C1(offsets_of, OFFSETS_OF, func_wrap) - C0(is_empty, IS_EMPTY) - CO_(union_, UNION, no_wrap) - C1(sample, SAMPLE, no_wrap) - CO_(group, GROUP, func_wrap) - C0(ungroup, UNGROUP) - C1(reduce, REDUCE, no_wrap) - CO2(fold, FOLD, no_wrap) - C0(count, COUNT) - C1(count, COUNT, func_wrap) - C0(sum, SUM) - C1(sum, SUM, func_wrap) - C0(avg, AVG) - C1(avg, AVG, func_wrap) - C1(min, MIN, func_wrap) - CO0(min, MIN) - C1(max, MAX, func_wrap) - CO0(max, MAX) - CO0(distinct, DISTINCT) - C_(contains, CONTAINS, func_wrap) - C_(pluck, PLUCK, no_wrap) - C_(without, WITHOUT, no_wrap) - C_(merge, MERGE, func_wrap) - C1(append, APPEND, no_wrap) - C1(prepend, PREPEND, no_wrap) - C1(difference, DIFFERENCE, no_wrap) - C1(set_insert, SET_INSERT, no_wrap) - C1(set_union, SET_UNION, no_wrap) - C1(set_intersection, SET_INTERSECTION, no_wrap) - C1(set_difference, SET_DIFFERENCE, no_wrap) - C1(operator[], BRACKET, no_wrap) - C1(get_field, GET_FIELD, no_wrap) - C_(has_fields, HAS_FIELDS, no_wrap) - C2(insert_at, INSERT_AT) - C2(splice_at, SPLICE_AT) - C1(delete_at, DELETE_AT, no_wrap) - C2(delete_at, DELETE_AT) - C2(change_at, CHANGE_AT) - C0(keys, KEYS) - C1(match, MATCH, no_wrap) - C0(split, SPLIT) - C1(split, SPLIT, no_wrap) - C2(split, SPLIT) - C0(upcase, UPCASE) - C0(downcase, DOWNCASE) - C_(add, ADD, no_wrap) - C1(operator+, ADD, no_wrap) - C_(sub, SUB, no_wrap) - C1(operator-, SUB, no_wrap) - C_(mul, MUL, no_wrap) - C1(operator*, MUL, no_wrap) - C_(div, DIV, no_wrap) - C1(operator/, DIV, no_wrap) - C1(mod, MOD, no_wrap) - C1(operator%, MOD, no_wrap) - C_(and_, AND, no_wrap) - C1(operator&&, AND, no_wrap) - C_(or_, OR, no_wrap) - C1(operator||, OR, no_wrap) - C1(eq, EQ, no_wrap) - C1(operator==, EQ, no_wrap) - C1(ne, NE, no_wrap) - C1(operator!=, NE, no_wrap) - C1(gt, GT, no_wrap) - C1(operator>, GT, no_wrap) - C1(ge, GE, no_wrap) - C1(operator>=, GE, no_wrap) - C1(lt, LT, no_wrap) - C1(operator<, LT, no_wrap) - C1(le, LE, no_wrap) - C1(operator<=, LE, no_wrap) - C0(not_, NOT) - C0(operator!, NOT) - C1(in_timezone, IN_TIMEZONE, no_wrap) - C0(timezone, TIMEZONE) - CO2(during, DURING, no_wrap) - C0(date, DATE) - C0(time_of_day, TIME_OF_DAY) - C0(year, YEAR) - C0(month, MONTH) - C0(day, DAY) - C0(day_of_week, DAY_OF_WEEK) - C0(day_of_year, DAY_OF_YEAR) - C0(hours, HOURS) - C0(minutes, MINUTES) - C0(seconds, SECONDS) - C0(to_iso8601, TO_ISO8601) - C0(to_epoch_time, TO_EPOCH_TIME) - C1(for_each, FOR_EACH, func_wrap) - C1(default_, DEFAULT, no_wrap) - CO1(js, JAVASCRIPT, no_wrap) - C1(coerce_to, COERCE_TO, no_wrap) - C0(type_of, TYPE_OF) - C0(info, INFO) - C0(to_json, TO_JSON_STRING) - C0(to_json_string, TO_JSON_STRING) - C1(distance, DISTANCE, no_wrap) - C0(fill, FILL) - C0(to_geojson, TO_GEOJSON) - CO1(get_intersecting, GET_INTERSECTING, no_wrap) - CO1(get_nearest, GET_NEAREST, no_wrap) - C1(includes, INCLUDES, no_wrap) - C1(intersects, INTERSECTS, no_wrap) - C1(polygon_sub, POLYGON_SUB, no_wrap) - C0(config, CONFIG) - C0(rebalance, REBALANCE) - CO0(reconfigure, RECONFIGURE) - C0(status, STATUS) - CO0(wait, WAIT) - C0(floor, FLOOR) - C0(ceil, CEIL) - C0(round, ROUND) - C0(values, VALUES) - - // The expansion of this macro fails to compile on some versions of GCC and Clang: - // C_(operator(), FUNCALL, no_wrap) - // The std::enable_if makes the error go away - - // $doc(do) - - template <class T, class ...U> - typename std::enable_if<!std::is_same<T, Var>::value, Term>::type - operator() (T&& a, U&& ...b) && { - return Term(TT::FUNCALL, std::vector<Term>{ - std::move(*this), - expr(std::forward<T>(a)), - expr(std::forward<U>(b))... }); - } - template <class T, class ...U> - typename std::enable_if<!std::is_same<T, Var>::value, Term>::type - operator() (T&& a, U&& ...b) const & { - return Term(TT::FUNCALL, std::vector<Term>{ - *this, - expr(std::forward<T>(a)), - expr(std::forward<U>(b))... }); - } - -#undef C0 -#undef C1 -#undef C2 -#undef C_ -#undef CO0 -#undef CO1 -#undef CO2 - - // Send the term to the server and return the results. - // Errors returned by the server are thrown. - Cursor run(Connection&, OptArgs&& args = {}); - - // $doc(do) - template <class ...T> - Term do_(T&& ...a) && { - auto list = { std::move(*this), Term::func_wrap(expr(std::forward<T>(a)))... }; - std::vector<Term> args; - args.reserve(list.size() + 1); - args.emplace_back(func_wrap(std::move(*(list.end()-1)))); - for (auto it = list.begin(); it + 1 != list.end(); ++it) { - args.emplace_back(std::move(*it)); - } - return Term(TT::FUNCALL, std::move(args)); - } - - // Adds optargs to an already built term - Term opt(OptArgs&& optargs) && { - return Term(std::move(*this), std::move(optargs)); - } - - // Used internally to implement object() - static Term make_object(std::vector<Term>&&); - - // Used internally to implement array() - static Term make_binary(Term&&); - - Datum get_datum() const; - -private: - friend class Var; - friend class Connection; - friend struct Query; - - template <int _> - Var mkvar(std::vector<int>& vars); - - template <class F, int ...N> - void set_function(F); - - Datum alpha_rename(Term&&); - - Term(Term&& orig, OptArgs&& optargs); - - std::map<int, int*> free_vars; - Datum datum; -}; - -// A term representing null -Term nil(); - -template <class T> -Term expr(T&& a) { - return Term(std::forward<T>(a)); -} - -// Represents a ReQL variable. -// This type is passed to functions used in ReQL queries. -class Var { -public: - // Convert to a term - Term operator*() const { - Term term(TT::VAR, std::vector<Term>{expr(*id)}); - term.free_vars = {{*id, id}}; - return term; - } - - Var(int* id_) : id(id_) { } -private: - int* id; -}; - -template <int N> -Var Term::mkvar(std::vector<int>& vars) { - int id = gen_var_id(); - vars.push_back(id); - return Var(&*vars.rbegin()); -} - -template <class F, int ...N> -void Term::set_function(F f) { - std::vector<int> vars; - vars.reserve(sizeof...(N)); - std::vector<Var> args = { mkvar<N>(vars)... }; - Term body = f(args[N] ...); - - int* low = &*vars.begin(); - int* high = &*(vars.end() - 1); - for (auto it = body.free_vars.begin(); it != body.free_vars.end(); ) { - if (it->second >= low && it->second <= high) { - if (it->first != *it->second) { - throw Error("Internal error: variable index mis-match"); - } - ++it; - } else { - free_vars.emplace(*it); - ++it; - } - } - datum = Array{TT::FUNC, Array{Array{TT::MAKE_ARRAY, vars}, body.datum}}; -} - -// These macros are similar to those defined above, but for top-level ReQL operations - -#define C0(name) Term name(); -#define C0_IMPL(name, type) Term name() { return Term(TT::type, std::vector<Term>{}); } -#define CO0(name) Term name(OptArgs&& optargs = {}); -#define CO0_IMPL(name, type) Term name(OptArgs&& optargs) { return Term(TT::type, std::vector<Term>{}, std::move(optargs)); } -#define C1(name, type, wrap) template <class T> Term name(T&& a) { \ - return Term(TT::type, std::vector<Term>{ wrap(expr(std::forward<T>(a))) }); } -#define C2(name, type) template <class T, class U> Term name(T&& a, U&& b) { \ - return Term(TT::type, std::vector<Term>{ expr(std::forward<T>(a)), expr(std::forward<U>(b)) }); } -#define C3(name, type) template <class A, class B, class C> \ - Term name(A&& a, B&& b, C&& c) { return Term(TT::type, std::vector<Term>{ \ - expr(std::forward<A>(a)), expr(std::forward<B>(b)), expr(std::forward<C>(c)) }); } -#define C4(name, type) template <class A, class B, class C, class D> \ - Term name(A&& a, B&& b, C&& c, D&& d) { return Term(TT::type, std::vector<Term>{ \ - expr(std::forward<A>(a)), expr(std::forward<B>(b)), \ - expr(std::forward<C>(c)), expr(std::forward<D>(d))}); } -#define C7(name, type) template <class A, class B, class C, class D, class E, class F, class G> \ - Term name(A&& a, B&& b, C&& c, D&& d, E&& e, F&& f, G&& g) { return Term(TT::type, std::vector<Term>{ \ - expr(std::forward<A>(a)), expr(std::forward<B>(b)), expr(std::forward<C>(c)), \ - expr(std::forward<D>(d)), expr(std::forward<E>(e)), expr(std::forward<F>(f)), \ - expr(std::forward<G>(g))}); } -#define C_(name, type, wrap) template <class ...T> Term name(T&& ...a) { \ - return Term(TT::type, std::vector<Term>{ wrap(expr(std::forward<T>(a)))... }); } -#define CO1(name, type, wrap) template <class T> Term name(T&& a, OptArgs&& optarg = {}) { \ - return Term(TT::type, std::vector<Term>{ wrap(expr(std::forward<T>(a)))}, std::move(optarg)); } -#define CO2(name, type) template <class T, class U> Term name(T&& a, U&& b, OptArgs&& optarg = {}) { \ - return Term(TT::type, std::vector<Term>{ expr(std::forward<T>(a)), expr(std::forward<U>(b))}, std::move(optarg)); } -#define func_wrap Term::func_wrap - -C1(db_create, DB_CREATE, no_wrap) -C1(db_drop, DB_DROP, no_wrap) -C0(db_list) -CO1(table_create, TABLE_CREATE, no_wrap) -C1(table_drop, TABLE_DROP, no_wrap) -C0(table_list) -C1(db, DB, no_wrap) -CO1(table, TABLE, no_wrap) -C_(add, ADD, no_wrap) -C2(sub, SUB) -C_(mul, MUL, no_wrap) -C_(div, DIV, no_wrap) -C2(mod, MOD) -C_(and_, AND, no_wrap) -C_(or_, OR, no_wrap) -C2(eq, EQ) -C2(ne, NE) -C2(gt, GT) -C2(ge, GE) -C2(lt, LT) -C2(le, LE) -C1(not_, NOT, no_wrap) -CO0(random) -CO1(random, RANDOM, no_wrap) -CO2(random, RANDOM) -C0(now) -C4(time, TIME) -C7(time, TIME) -C1(epoch_time, EPOCH_TIME, no_wrap) -CO1(iso8601, ISO8601, no_wrap) -CO1(js, JAVASCRIPT, no_wrap) -C1(args, ARGS, no_wrap) -C_(branch, BRANCH, no_wrap) -C0(range) -C1(range, RANGE, no_wrap) -C2(range, RANGE) -C0(error) -C1(error, ERROR, no_wrap) -C1(json, JSON, no_wrap) -CO1(http, HTTP, func_wrap) -C0(uuid) -C1(uuid, UUID, no_wrap) -CO2(circle, CIRCLE) -C1(geojson, GEOJSON, no_wrap) -C_(line, LINE, no_wrap) -C2(point, POINT) -C_(polygon, POLYGON, no_wrap) -C_(array, MAKE_ARRAY, no_wrap) -C1(desc, DESC, func_wrap) -C1(asc, ASC, func_wrap) -C0(literal) -C1(literal, LITERAL, no_wrap) -C1(type_of, TYPE_OF, no_wrap) -C_(map, MAP, func_wrap) -C1(floor, FLOOR, no_wrap) -C1(ceil, CEIL, no_wrap) -C1(round, ROUND, no_wrap) -C_(union_, UNION, no_wrap) -C_(group, GROUP, func_wrap) -C1(count, COUNT, no_wrap) -C_(count, COUNT, func_wrap) -C1(sum, SUM, no_wrap) -C_(sum, SUM, func_wrap) -C1(avg, AVG, no_wrap) -C_(avg, AVG, func_wrap) -C1(min, MIN, no_wrap) -C_(min, MIN, func_wrap) -C1(max, MAX, no_wrap) -C_(max, MAX, func_wrap) -C1(distinct, DISTINCT, no_wrap) -C1(contains, CONTAINS, no_wrap) -C_(contains, CONTAINS, func_wrap) - -#undef C0 -#undef C1 -#undef C2 -#undef C3 -#undef C4 -#undef C7 -#undef C_ -#undef CO1 -#undef CO2 -#undef func_wrap - -// $doc(do) -template <class R, class ...T> -Term do_(R&& a, T&& ...b) { - return expr(std::forward<R>(a)).do_(std::forward<T>(b)...); -} - -// $doc(object) -template <class ...T> -Term object(T&& ...a) { - return Term::make_object(std::vector<Term>{ expr(std::forward<T>(a))... }); -} - -// $doc(binary) -template <class T> -Term binary(T&& a) { - return Term::make_binary(expr(std::forward<T>(a))); -} - -// Construct an empty optarg -OptArgs optargs(); - -// Construct an optarg made out of pairs of arguments -// For example: optargs("k1", v1, "k2", v2) -template <class V, class ...T> -OptArgs optargs(const char* key, V&& val, T&& ...rest) { - OptArgs opts = optargs(rest...); - opts.emplace(key, expr(std::forward<V>(val))); - return opts; -} - -extern Term row; -extern Term maxval; -extern Term minval; -extern Term january; -extern Term february; -extern Term march; -extern Term april; -extern Term may; -extern Term june; -extern Term july; -extern Term august; -extern Term september; -extern Term october; -extern Term november; -extern Term december; -extern Term monday; -extern Term tuesday; -extern Term wednesday; -extern Term thursday; -extern Term friday; -extern Term saturday; -extern Term sunday; -} diff --git a/ext/librethinkdbxx/src/types.cc b/ext/librethinkdbxx/src/types.cc deleted file mode 100644 index ea9becaf..00000000 --- a/ext/librethinkdbxx/src/types.cc +++ /dev/null @@ -1,47 +0,0 @@ -#include <cstdlib> - -#include "types.h" -#include "error.h" - -namespace RethinkDB { - -bool Time::parse_utc_offset(const std::string& string, double* offset) { - const char *s = string.c_str(); - double sign = 1; - switch (s[0]) { - case '-': - sign = -1; - case '+': - ++s; - break; - case 0: - return false; - } - for (int i = 0; i < 5; ++i) { - if (s[i] == 0) return false; - if (i == 2) continue; - if (s[i] < '0' || s[i] > '9') return false; - } - if (s[2] != ':') return false; - *offset = sign * ((s[0] - '0') * 36000 + (s[1] - '0') * 3600 + (s[3] - '0') * 600 + (s[4] - '0') * 60); - return true; -} - -double Time::parse_utc_offset(const std::string& string) { - double out; - if (!parse_utc_offset(string, &out)) { - throw Error("invalid utc offset `%s'", string.c_str()); - } - return out; -} - -std::string Time::utc_offset_string(double offset) { - char buf[8]; - int hour = offset / 3600; - int minutes = std::abs(static_cast<int>(offset / 60)) % 60; - int n = snprintf(buf, 7, "%+03d:%02d", hour, minutes); - buf[n] = 0; - return std::string(buf); -} - -} diff --git a/ext/librethinkdbxx/src/types.h b/ext/librethinkdbxx/src/types.h deleted file mode 100644 index ac35a871..00000000 --- a/ext/librethinkdbxx/src/types.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include <vector> -#include <map> -#include <ctime> -#include <string> - -namespace RethinkDB { - -class Datum; - -// Represents a null datum -struct Nil { }; - -using Array = std::vector<Datum>; -using Object = std::map<std::string, Datum>; - -// Represents a string of bytes. Plain std::strings are passed on to the server as utf-8 strings -struct Binary { - bool operator== (const Binary& other) const { - return data == other.data; - } - - Binary(const std::string& data_) : data(data_) { } - Binary(std::string&& data_) : data(std::move(data_)) { } - std::string data; -}; - -// Represents a point in time as -// * A floating amount of seconds since the UNIX epoch -// * And a timezone offset represented as seconds relative to UTC -struct Time { - Time(double epoch_time_, double utc_offset_ = 0) : - epoch_time(epoch_time_), utc_offset(utc_offset_) { } - - static Time now() { - return Time(time(NULL)); - } - - static bool parse_utc_offset(const std::string&, double*); - static double parse_utc_offset(const std::string&); - static std::string utc_offset_string(double); - - double epoch_time; - double utc_offset; -}; - -// Not implemented -class Point; -class Line; -class Polygon; - -} diff --git a/ext/librethinkdbxx/src/utils.cc b/ext/librethinkdbxx/src/utils.cc deleted file mode 100644 index 5a2c244d..00000000 --- a/ext/librethinkdbxx/src/utils.cc +++ /dev/null @@ -1,153 +0,0 @@ -#include "utils.h" -#include "error.h" - -namespace RethinkDB { - -size_t utf8_encode(unsigned int code, char* buf) { - if (!(code & ~0x7F)) { - buf[0] = code; - return 1; - } else if (!(code & ~0x7FF)) { - buf[0] = 0xC0 | (code >> 6); - buf[1] = 0x80 | (code & 0x3F); - return 2; - } else if (!(code & ~0xFFFF)) { - buf[0] = 0xE0 | (code >> 12); - buf[1] = 0x80 | ((code >> 6) & 0x3F); - buf[2] = 0x80 | (code & 0x3F); - return 3; - } else if (!(code & ~0x1FFFFF)) { - buf[0] = 0xF0 | (code >> 18); - buf[1] = 0x80 | ((code >> 12) & 0x3F); - buf[2] = 0x80 | ((code >> 6) & 0x3F); - buf[3] = 0x80 | (code & 0x3F); - return 4; - } else if (!(code & ~0x3FFFFFF)) { - buf[0] = 0xF8 | (code >> 24); - buf[1] = 0x80 | ((code >> 18) & 0x3F); - buf[2] = 0x80 | ((code >> 12) & 0x3F); - buf[3] = 0x80 | ((code >> 6) & 0x3F); - buf[4] = 0x80 | (code & 0x3F); - return 5; - } else if (!(code & ~0x7FFFFFFF)) { - buf[0] = 0xFC | (code >> 30); - buf[1] = 0x80 | ((code >> 24) & 0x3F); - buf[2] = 0x80 | ((code >> 18) & 0x3F); - buf[3] = 0x80 | ((code >> 12) & 0x3F); - buf[4] = 0x80 | ((code >> 6) & 0x3F); - buf[5] = 0x80 | (code & 0x3F); - return 6; - } else { - throw Error("Invalid unicode codepoint %ud", code); - } -} - -bool base64_decode(char c, int* out) { - if (c >= 'A' && c <= 'Z') { - *out = c - 'A'; - } else if (c >= 'a' && c <= 'z') { - *out = c - ('a' - 26); - } else if (c >= '0' && c <= '9') { - *out = c - ('0' - 52); - } else if (c == '+') { - *out = 62; - } else if (c == '/') { - *out = 63; - } else { - return false; - } - return true; -} - -bool base64_decode(const std::string& in, std::string& out) { - out.clear(); - out.reserve(in.size() * 3 / 4); - auto read = in.begin(); - while (true) { - int c[4]; - int end = 4; - for (int i = 0; i < 4; i++) { - while (true) { - if (read == in.end()) { - c[i] = 0; - end = i; - i = 3; - break; - } else if (base64_decode(*read, &c[i])) { - ++read; - break; - } else { - ++read; - } - } - } - if (end == 1) return false; - int val = c[0] << 18 | c[1] << 12 | c[2] << 6 | c[3]; - if (end > 1) out.append(1, val >> 16); - if (end > 2) out.append(1, val >> 8 & 0xFF); - if (end > 3) out.append(1, val & 0xFF); - if (end != 4) break; - } - return true; -} - -char base64_encode(unsigned int c) { - if (c < 26) { - return 'A' + c; - } else if (c < 52) { - return 'a' + c - 26; - } else if (c < 62) { - return '0' + c - 52; - } else if (c == 62) { - return '+'; - } else if (c == 63) { - return '/'; - } else { - throw Error("unreachable: base64 encoding %d", c); - } -} - -void base64_encode(unsigned int* c, int n, std::string& out) { - if (n == 0) { - return; - } - out.append(1, base64_encode(c[0] >> 2)); - out.append(1, base64_encode((c[0] & 0x3) << 4 | c[1] >> 4)); - if (n == 1) { - out.append("=="); - return; - } - out.append(1, base64_encode((c[1] & 0xF) << 2 | c[2] >> 6)); - if (n == 2) { - out.append("="); - return; - } - out.append(1, base64_encode(c[2] & 0x3F)); -} - -std::string base64_encode(const std::string& in) { - std::string out; - out.reserve(in.size() * 4 / 3 + in.size() / 48 + 3); - auto read = in.begin(); - while (true) { - for (int group = 0; group < 16; ++group) { - unsigned int c[3]; - int i = 0; - for (; i < 3; ++i) { - if (read == in.end()) { - c[i] = 0; - break; - } else { - c[i] = static_cast<unsigned char>(*read++); - } - } - base64_encode(c, i, out); - if (i != 3) { - return out; - } - } - out.append("\n"); - } -} - -} diff --git a/ext/librethinkdbxx/src/utils.h b/ext/librethinkdbxx/src/utils.h deleted file mode 100644 index 04496e2c..00000000 --- a/ext/librethinkdbxx/src/utils.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include <cstddef> -#include <string> - -namespace RethinkDB { - -// The size of the longest UTF-8 encoded unicode codepoint -const size_t max_utf8_encoded_size = 6; - -// Decode a base64 string. Returns false on failure. -bool base64_decode(const std::string& in, std::string& out); -std::string base64_encode(const std::string&); - -// Encodes a single unicode codepoint into UTF-8. Returns the number of bytes written. -// Does not add a trailing null byte -size_t utf8_encode(unsigned int, char*); - -} diff --git a/ext/miniupnpc/mingw32make.bat b/ext/miniupnpc/mingw32make.bat deleted file mode 100644 index c5d3cc4f..00000000 --- a/ext/miniupnpc/mingw32make.bat +++ /dev/null @@ -1,8 +0,0 @@ -@mingw32-make -f Makefile.mingw %1 -@if errorlevel 1 goto end -@if not exist upnpc-static.exe goto end -@strip upnpc-static.exe -@upx --best upnpc-static.exe -@strip upnpc-shared.exe -@upx --best upnpc-shared.exe -:end diff --git a/ext/miniupnpc/setup.py b/ext/miniupnpc/setup.py deleted file mode 100644 index 97e42bf1..00000000 --- a/ext/miniupnpc/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/python -# vim: tabstop=8 shiftwidth=8 expandtab -# $Id: setup.py,v 1.12 2015/10/26 17:03:17 nanard Exp $ -# the MiniUPnP Project (c) 2007-2014 Thomas Bernard -# http://miniupnp.tuxfamily.org/ or http://miniupnp.free.fr/ -# -# python script to build the miniupnpc module under unix -# -# replace libminiupnpc.a by libminiupnpc.so for shared library usage -try: - from setuptools import setup, Extension -except ImportError: - from distutils.core import setup, Extension -from distutils import sysconfig -sysconfig.get_config_vars()["OPT"] = '' -sysconfig.get_config_vars()["CFLAGS"] = '' -setup(name="miniupnpc", - version=open('VERSION').read().strip(), - author='Thomas BERNARD', - author_email='miniupnp@free.fr', - license=open('LICENSE').read(), - url='http://miniupnp.free.fr/', - description='miniUPnP client', - ext_modules=[ - Extension(name="miniupnpc", sources=["miniupnpcmodule.c"], - extra_objects=["libminiupnpc.a"]) - ]) - diff --git a/ext/miniupnpc/setupmingw32.py b/ext/miniupnpc/setupmingw32.py deleted file mode 100644 index 43dfb465..00000000 --- a/ext/miniupnpc/setupmingw32.py +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/python -# vim: tabstop=8 shiftwidth=8 expandtab -# $Id: setupmingw32.py,v 1.10 2015/10/26 17:03:17 nanard Exp $ -# the MiniUPnP Project (c) 2007-2014 Thomas Bernard -# http://miniupnp.tuxfamily.org/ or http://miniupnp.free.fr/ -# -# python script to build the miniupnpc module under windows (using mingw32) -# -try: - from setuptools import setup, Extension -except ImportError: - from distutils.core import setup, Extension -from distutils import sysconfig -sysconfig.get_config_vars()["OPT"] = '' -sysconfig.get_config_vars()["CFLAGS"] = '' -setup(name="miniupnpc", - version=open('VERSION').read().strip(), - author='Thomas BERNARD', - author_email='miniupnp@free.fr', - license=open('LICENSE').read(), - url='http://miniupnp.free.fr/', - description='miniUPnP client', - ext_modules=[ - Extension(name="miniupnpc", sources=["miniupnpcmodule.c"], - libraries=["ws2_32", "iphlpapi"], - extra_objects=["libminiupnpc.a"]) - ]) - diff --git a/ext/tap-mac/README.txt b/ext/tap-mac/README.txt deleted file mode 100644 index 177b936f..00000000 --- a/ext/tap-mac/README.txt +++ /dev/null @@ -1,19 +0,0 @@ -This is a hack of tuntaposx. It's here for two reasons: - -1) There seem to be issues with large MTUs in the original tuntap code, - so we set up our zt0 tap with the correct ZeroTier MTU as the default. - -2) Lots of other mac products (VPNs, etc.) ship their own tap device - drivers that like to conflict with one another. This gives us no - choice but to play along. But we call our tap device zt0, which means - it won't conflict with everyone else's tap0. - -3) It's nice to call the device zt0, same as Linux, for consistency across - *nix platforms. Mac does not seem to support interface renaming. - -This will be placed in the ZeroTier home as a kext and is auto-loaded by the -ZeroTier One binary if /dev/zt0 is not found. It can also be auto-updated. - -See this page for the original: - -http://tuntaposx.sourceforge.net diff --git a/ext/tap-mac/tuntap/Makefile b/ext/tap-mac/tuntap/Makefile deleted file mode 100644 index 53ab1a9d..00000000 --- a/ext/tap-mac/tuntap/Makefile +++ /dev/null @@ -1,95 +0,0 @@ -# Lets have a version, at last! -TUNTAP_VERSION = 20150118 - -# BASE install directory -BASE= - -all: tap.kext - -keysetup: - -security delete-keychain net.sf.tuntaposx.tmp - security create-keychain -p $$(head -c 32 /dev/urandom | hexdump -e '"%02x"') \ - net.sf.tuntaposx.tmp - security set-keychain-settings -lut 60 net.sf.tuntaposx.tmp - security import identity.p12 -k net.sf.tuntaposx.tmp -f pkcs12 \ - -P $$(read -sp 'identity passphrase: ' pw && echo "$$pw") -A - security find-identity -v net.sf.tuntaposx.tmp | \ - awk -F \" '$$2 ~ /^Developer ID Application:/ { print $$2 }' > .signing_identity - security find-identity -v net.sf.tuntaposx.tmp | \ - awk -F \" '$$2 ~ /^Developer ID Installer:/ { print $$2 }' > .installer_identity - -pkgbuild/%.pkg: %.kext - mkdir -p pkgbuild/$*_root/Library/Extensions - cp -pR $*.kext pkgbuild/$*_root/Library/Extensions - mkdir -p pkgbuild/$*_root/Library/LaunchDaemons - cp pkg/launchd/net.sf.tuntaposx.$*.plist pkgbuild/$*_root/Library/LaunchDaemons - pkgbuild --root pkgbuild/$*_root \ - --component-plist pkg/components/$*.plist \ - --scripts pkg/scripts/$* pkgbuild/$*.pkg - -tuntap_$(TUNTAP_VERSION).pkg: pkgbuild/tap.pkg pkgbuild/tun.pkg - productbuild --distribution pkg/distribution.xml --package-path pkgbuild \ - --resources pkg/res.dummy \ - tuntap_$(TUNTAP_VERSION).pkg ; \ - pkgutil --expand tuntap_$(TUNTAP_VERSION).pkg pkgbuild/tuntap_pkg.d - cp -pR pkg/res/ pkgbuild/tuntap_pkg.d/Resources - pkgutil --flatten pkgbuild/tuntap_pkg.d tuntap_$(TUNTAP_VERSION).pkg - if test -s ".installer_identity"; then \ - productsign --sign "$$(cat .installer_identity)" --keychain net.sf.tuntaposx.tmp \ - tuntap_$(TUNTAP_VERSION).pkg tuntap_$(TUNTAP_VERSION).pkg.signed ; \ - mv tuntap_$(TUNTAP_VERSION).pkg.signed tuntap_$(TUNTAP_VERSION).pkg ; \ - fi - -pkg: tuntap_$(TUNTAP_VERSION).pkg - tar czf tuntap_$(TUNTAP_VERSION).tar.gz \ - README.installer README tuntap_$(TUNTAP_VERSION).pkg - -# Install targets -# They are provided for the gentoo ebuild, but should work just fine for other people as well. -install_%_kext: %.kext - mkdir -p $(BASE)/Library/Extensions - cp -pR $*.kext $(BASE)/Library/Extensions/ - chown -R root:wheel $(BASE)/Library/Extensions/$*.kext - mkdir -p $(BASE)/Library/LaunchDaemons - cp pkg/launchd/net.sf.tuntaposx.$*.plist $(BASE)/Library/LaunchDaemons - chown -R root:wheel $(BASE)/Library/LaunchDaemons/net.sf.tuntaposx.$*.plist - -install: install_tap_kext install_tun_kext - -tarball: clean - touch tuntap_$(TUNTAP_VERSION)_src.tar.gz - tar czf tuntap_$(TUNTAP_VERSION)_src.tar.gz \ - -C .. \ - --exclude "tuntap/identity.p12" \ - --exclude "tuntap/tuntap_$(TUNTAP_VERSION)_src.tar.gz" \ - --exclude "tuntap/tuntap_$(TUNTAP_VERSION).tar.gz" \ - --exclude "tuntap/tuntap_$(TUNTAP_VERSION).pkg" \ - --exclude "*/.*" \ - tuntap - -clean: - cd src/tap && make -f Makefile clean - cd src/tun && make -f Makefile clean - -rm -rf pkgbuild - -rm -rf tuntap_$(TUNTAP_VERSION).pkg - -rm -f tuntap_$(TUNTAP_VERSION).tar.gz - -rm -f tuntap_$(TUNTAP_VERSION)_src.tar.gz - -%.kext: - cd src/$* && make TUNTAP_VERSION=$(TUNTAP_VERSION) -f Makefile all - if test -s ".signing_identity"; then \ - codesign -fv --keychain net.sf.tuntaposx.tmp -s "$$(cat .signing_identity)" \ - $*.kext ; \ - fi - -test: - # configd messes with interface flags, issuing SIOCSIFFLAGS ioctls upon receiving kernel - # events indicating protocols have been attached and detached. Unfortunately, configd does - # this asynchronously, making the SIOCSIFFLAGS changes totally unpredictable when we bring - # our interfaces up and down in rapid succession during our tests. I haven't found a good - # way to suppress or handle this mess other than disabling configd temporarily. - killall -STOP configd - -PYTHONPATH=test python test/tuntap/tuntap_tests.py --tests='$(TESTS)' - killall -CONT configd - -.PHONY: test diff --git a/ext/tap-mac/tuntap/src/lock.cc b/ext/tap-mac/tuntap/src/lock.cc deleted file mode 100644 index 9c78783a..00000000 --- a/ext/tap-mac/tuntap/src/lock.cc +++ /dev/null @@ -1,206 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. - * - * Locking implementation. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "lock.h" - -extern "C" { - -#include <kern/clock.h> - -#include <sys/syslog.h> -#include <sys/proc.h> - -} - -#if 0 -#define dprintf(...) log(LOG_INFO, __VA_ARGS__) -#else -#define dprintf(...) -#endif - -/* class tt_lock */ -lck_grp_t *tt_lock::tt_lck_grp = NULL; - -bool -tt_lock::initialize() -{ - /* init if necessary */ - if (tt_lck_grp == NULL) { - dprintf("initing lock group\n"); - tt_lck_grp = lck_grp_alloc_init("tuntap locks", LCK_GRP_ATTR_NULL); - - if (tt_lck_grp == NULL) { - /* if something fails, the lock won't work */ - log(LOG_ERR, "tuntap: could not allocate locking group\n"); - return false; - } - } - - return true; -} - -void -tt_lock::shutdown() -{ - /* free the locking group */ - if (tt_lck_grp != NULL) { - dprintf("freeing lock group\n"); - lck_grp_free(tt_lck_grp); - tt_lck_grp = NULL; - } -} - -/* tt_mutex */ -tt_mutex::tt_mutex() -{ - /* fail if locking group not initialized */ - if (tt_lck_grp == NULL) - return; - - /* allocate the lock */ - lck = lck_rw_alloc_init(tt_lck_grp, NULL); - - if (lck == NULL) - log(LOG_ERR, "tuntap: could not allocate mutex\n"); -} - -tt_mutex::~tt_mutex() -{ - /* if the lock doesn't exist, this will be a no-op */ - if (lck == NULL) - return; - - /* free the lock */ - lck_rw_free(lck, tt_lck_grp); -} - -void -tt_mutex::lock() -{ - if (lck != NULL) - lck_rw_lock_exclusive(lck); -} - -void -tt_mutex::unlock() -{ - if (lck != NULL) - lck_rw_unlock_exclusive(lck); -} - -void -tt_mutex::sleep(void *cond) -{ - if (lck != NULL) - lck_rw_sleep(lck, LCK_SLEEP_DEFAULT, cond, THREAD_INTERRUPTIBLE); -} - -void -tt_mutex::sleep(void *cond, uint64_t nanoseconds) -{ - if (lck != NULL) { - uint64_t abstime; - nanoseconds_to_absolutetime(nanoseconds, &abstime); - lck_rw_sleep_deadline(lck, LCK_SLEEP_DEFAULT, cond, THREAD_INTERRUPTIBLE, abstime); - } -} - -void -tt_mutex::wakeup(void *cond) -{ - if (lck != NULL) - ::wakeup(cond); -} - -/* tt_gate */ -tt_gate::tt_gate() - : ticket_number(0), - population(0) -{ -} - -void -tt_gate::enter() -{ - /* just try to grab the lock, increase the ticket number and the population */ - auto_lock l(&slock); - ticket_number++; - population++; -} - -void -tt_gate::exit() -{ - auto_lock l(&slock); - ticket_number--; - population--; -} - -bool -tt_gate::is_anyone_in() -{ - return population != 0; -} - -unsigned int -tt_gate::get_ticket_number() -{ - return ticket_number; -} - -void -tt_gate::lock() -{ - slock.lock(); -} - -void -tt_gate::unlock() -{ - slock.unlock(); -} - -void -tt_gate::sleep(void* cond) -{ - slock.sleep(cond); -} - -void -tt_gate::sleep(void* cond, uint64_t nanoseconds) -{ - slock.sleep(cond, nanoseconds); -} - -void -tt_gate::wakeup(void* cond) -{ - slock.wakeup(cond); -} - diff --git a/ext/tap-mac/tuntap/src/lock.h b/ext/tap-mac/tuntap/src/lock.h deleted file mode 100644 index 51d3299a..00000000 --- a/ext/tap-mac/tuntap/src/lock.h +++ /dev/null @@ -1,160 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. - * - * Locking is not as straightforward for Tiger. So declare our own locking class. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __LOCK_H__ -#define __LOCK_H__ - -extern "C" { - -#include <kern/locks.h> -#include <sys/param.h> - -} - -/* our own locking class. declares the common interface of the locking primitives. */ -class tt_lock { - - protected: - /* locking group */ - static lck_grp_t *tt_lck_grp; - - public: - /* be virtual */ - virtual ~tt_lock() { }; - - /* static intialization (inits the locking group) */ - static bool initialize(); - static void shutdown(); - - /* locking */ - virtual void lock() = 0; - virtual void unlock() = 0; - - /* monitor primitives */ - virtual void sleep(void* cond) = 0; - virtual void sleep(void* cond, uint64_t) = 0; - virtual void wakeup(void* cond) = 0; -}; - -/* simple mutex */ -class tt_mutex : public tt_lock { - - private: - /* underlying darwin lock */ - lck_rw_t *lck; - - public: - tt_mutex(); - virtual ~tt_mutex(); - - void lock(); - void unlock(); - - /* monitor primitives */ - void sleep(void* cond); - void sleep(void* cond, uint64_t); - void wakeup(void* cond); -}; - -/* A very special locking class that we use to track threads that enter and leave the character - * device service functions. They call enter() before entering the actual service routinge and - * exit() when done. enter() only permits them to pass when the gate isn't locked. Furthermore, the - * gate assigns ticket numbers to everyone that passes the gate, so you can check whether more - * threads came through. See tuntap_mgr::shutdown() for how we use that stuff. - */ -class tt_gate : public tt_lock { - - private: - /* synchronization lock */ - tt_mutex slock; - /* ticket number */ - unsigned int ticket_number; - /* count of threads that are in */ - unsigned int population; - - public: - /* construct a new gate */ - tt_gate(); - - /* enter - pass the gate */ - void enter(); - /* exit - pass the gate */ - void exit(); - - /* check whether anyone is in */ - bool is_anyone_in(); - /* gets the next ticket number */ - unsigned int get_ticket_number(); - - /* lock the gate */ - void lock(); - /* unlock the gate */ - void unlock(); - - /* monitor primitives */ - void sleep(void* cond); - void sleep(void* cond, uint64_t); - void wakeup(void* cond); -}; - -/* auto_lock and auto_rwlock serve as automatic lock managers: Create an object, passing the - * tt_[rw]lock you want to lock to have it grab the lock. When the object goes out of scope, the - * destructor of the class will release the lock. - */ -class auto_lock { - - protected: - /* the lock we hold */ - tt_lock *l; - - public: - auto_lock(tt_lock *m) - : l(m) - { - lock(); - } - - ~auto_lock() - { - unlock(); - } - - void lock() - { - l->lock(); - } - - void unlock() - { - l->unlock(); - } -}; - -#endif /* __LOCK_H__ */ - diff --git a/ext/tap-mac/tuntap/src/mem.cc b/ext/tap-mac/tuntap/src/mem.cc deleted file mode 100644 index cd3264fa..00000000 --- a/ext/tap-mac/tuntap/src/mem.cc +++ /dev/null @@ -1,76 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. Common functionality of tap_interface and tun_interface. - * - * Memory management implementation. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "mem.h" - -extern "C" { - -#include <libkern/OSMalloc.h> - -} - -#if 0 -#define dprintf(...) log(LOG_INFO, __VA_ARGS__) -#else -#define dprintf(...) -#endif - -static int inited = 0; -static OSMallocTag tag; - -void -mem_initialize(const char* name) { - - if (!inited) { - tag = OSMalloc_Tagalloc(name, OSMT_DEFAULT); - inited = 1; - } -} - -void -mem_shutdown() { - - if (inited) { - OSMalloc_Tagfree(tag); - inited = 0; - } -} - -void * -mem_alloc(uint32_t size) { - - return OSMalloc(size, tag); -} - -void -mem_free(void *addr, uint32_t size) { - - OSFree(addr, size, tag); -} - diff --git a/ext/tap-mac/tuntap/src/mem.h b/ext/tap-mac/tuntap/src/mem.h deleted file mode 100644 index 4d06fd8c..00000000 --- a/ext/tap-mac/tuntap/src/mem.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. Common functionality of tap_interface and tun_interface. - * - * Memory management. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __MEM_H__ -#define __MEM_H__ - -extern "C" { - -#include <stdint.h> - -} - -/* Memory manager initalization and shutdown */ -void mem_initialize(const char *name); -void mem_shutdown(); - -/* Memory allocation functions */ -void *mem_alloc(uint32_t size); -void mem_free(void *addr, uint32_t size); - -#endif /* __MEM_H__ */ - diff --git a/ext/tap-mac/tuntap/src/tap/Info.plist b/ext/tap-mac/tuntap/src/tap/Info.plist deleted file mode 100644 index bb9b03fd..00000000 --- a/ext/tap-mac/tuntap/src/tap/Info.plist +++ /dev/null @@ -1,36 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>CFBundleDevelopmentRegion</key> - <string>@@CFBUNDLEDEVELOPMENTREGION@@</string> - <key>CFBundleExecutable</key> - <string>@@CFBUNDLEEXECUTABLE@@</string> - <key>CFBundleIdentifier</key> - <string>@@CFBUNDLEIDENTIFIER@@</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>@@CFBUNDLEEXECUTABLE@@</string> - <key>CFBundlePackageType</key> - <string>@@CFBUNDLEPACKAGETYPE@@</string> - <key>CFBundleShortVersionString</key> - <string>@@CFBUNDLEVERSION@@</string> - <key>CFBundleSignature</key> - <string>@@CFBUNDLESIGNATURE@@</string> - <key>CFBundleVersion</key> - <string>1.0</string> - <key>OSBundleLibraries</key> - <dict> - <key>com.apple.kpi.mach</key> - <string>8.0</string> - <key>com.apple.kpi.bsd</key> - <string>8.0</string> - <key>com.apple.kpi.libkern</key> - <string>8.0</string> - <key>com.apple.kpi.unsupported</key> - <string>8.0</string> - </dict> -</dict> -</plist> - diff --git a/ext/tap-mac/tuntap/src/tap/Makefile b/ext/tap-mac/tuntap/src/tap/Makefile deleted file mode 100644 index 306a86d7..00000000 --- a/ext/tap-mac/tuntap/src/tap/Makefile +++ /dev/null @@ -1,60 +0,0 @@ -# -# ethertap driver for MacOSX -# -# Makefile -# -# (c) 2004, 2005, 2006, 2007, 2008 Mattias Nissler -# - -OBJS = ../tuntap.o ../tuntap_mgr.o ../lock.o ../mem.o kmod.o tap.o -KMOD_BIN = tap -BUNDLE_DIR = ../.. -BUNDLE_NAME = tap.kext - -TAP_KEXT_VERSION = $(TUNTAP_VERSION) - -BUNDLE_REGION = English -BUNDLE_IDENTIFIER = com.zerotier.tap -BUNDLE_SIGNATURE = ???? -BUNDLE_PACKAGETYPE = KEXT -BUNDLE_VERSION = $(TAP_KEXT_VERSION) - -INCLUDE = -I.. -I/System/Library/Frameworks/Kernel.framework/Headers -CFLAGS = -Wall -Werror -mkernel -force_cpusubtype_ALL \ - -nostdinc -fno-builtin -fno-stack-protector -msoft-float -fno-common \ - -arch x86_64 \ - -DKERNEL -DAPPLE -DKERNEL_PRIVATE -DTUNTAP_VERSION=\"$(TUNTAP_VERSION)\" \ - -DTAP_KEXT_VERSION=\"$(TAP_KEXT_VERSION)\" -CCFLAGS = $(CFLAGS) -LDFLAGS = -Wall -Werror -arch x86_64 -Xlinker -kext -nostdlib -lkmodc++ -lkmod -lcc_kext - -CCP = clang -x c++ -CC = clang -x c -LD = clang - -all: $(KMOD_BIN) bundle - -.c.o: - $(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@ -.cc.o: - $(CCP) $(CCFLAGS) $(INCLUDE) -c $< -o $@ - -$(KMOD_BIN): $(OBJS) - $(LD) $(LDFLAGS) -o $(KMOD_BIN) $(OBJS) - -bundle: $(KMOD_BIN) - rm -rf $(BUNDLE_DIR)/$(BUNDLE_NAME) - mkdir -p $(BUNDLE_DIR)/$(BUNDLE_NAME)/Contents/MacOS - cp $(KMOD_BIN) $(BUNDLE_DIR)/$(BUNDLE_NAME)/Contents/MacOS - sed -e "s/@@CFBUNDLEEXECUTABLE@@/$(KMOD_BIN)/" \ - -e "s/@@CFBUNDLEDEVELOPMENTREGION@@/$(BUNDLE_REGION)/" \ - -e "s/@@CFBUNDLEIDENTIFIER@@/$(BUNDLE_IDENTIFIER)/" \ - -e "s/@@CFBUNDLESIGNATURE@@/$(BUNDLE_SIGNATURE)/" \ - -e "s/@@CFBUNDLEPACKAGETYPE@@/$(BUNDLE_PACKAGETYPE)/" \ - -e "s/@@CFBUNDLEVERSION@@/$(BUNDLE_VERSION)/" \ - Info.plist > $(BUNDLE_DIR)/$(BUNDLE_NAME)/Contents/Info.plist - -clean: - -rm -f $(OBJS) $(KMOD_BIN) - -rm -rf $(BUNDLE_DIR)/$(BUNDLE_NAME) - diff --git a/ext/tap-mac/tuntap/src/tap/kmod.cc b/ext/tap-mac/tuntap/src/tap/kmod.cc deleted file mode 100644 index f9c4a40e..00000000 --- a/ext/tap-mac/tuntap/src/tap/kmod.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - * ethertap device for MacOSX. - * - * Kext definition (it is a mach kmod really...) - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "tap.h" -#include "mem.h" - -extern "C" { - -#include <sys/param.h> - -#include <mach/kmod.h> - -static tap_manager *mgr; - -/* - * start function. called when the kext gets loaded. - */ -static kern_return_t tap_module_start(struct kmod_info *ki, void *data) -{ - mem_initialize(TAP_FAMILY_NAME); - - /* initialize locking */ - if (!tt_lock::initialize()) - return KMOD_RETURN_FAILURE; - - /* create a tap manager that will handle the rest */ - mgr = new tap_manager(); - - if (mgr != NULL) { - if (mgr->initialize(TAP_IF_COUNT, (char *) TAP_FAMILY_NAME)) - return KMOD_RETURN_SUCCESS; - - delete mgr; - mgr = NULL; - /* clean up locking */ - tt_lock::shutdown(); - } - - return KMOD_RETURN_FAILURE; -} - -/* - * stop function. called when the kext should be unloaded. unloading can be prevented by - * returning failure - */ -static kern_return_t tap_module_stop(struct kmod_info *ki, void *data) -{ - if (mgr != NULL) { - if (!mgr->shutdown()) - return KMOD_RETURN_FAILURE; - - delete mgr; - mgr = NULL; - } - - /* clean up locking */ - tt_lock::shutdown(); - - mem_shutdown(); - - return KMOD_RETURN_SUCCESS; -} - -KMOD_DECL(tap, TAP_KEXT_VERSION) - -} - diff --git a/ext/tap-mac/tuntap/src/tap/tap.cc b/ext/tap-mac/tuntap/src/tap/tap.cc deleted file mode 100644 index b348a85e..00000000 --- a/ext/tap-mac/tuntap/src/tap/tap.cc +++ /dev/null @@ -1,533 +0,0 @@ -/* - * ethertap device for macosx. - * - * tap_interface class definition - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "tap.h" - -extern "C" { - -#include <sys/systm.h> -#include <sys/syslog.h> -#include <sys/param.h> -#include <sys/sockio.h> -#include <sys/random.h> -#include <sys/kern_event.h> - -#include <mach/thread_policy.h> - -#include <net/if_types.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> -#include <net/dlil.h> -#include <net/ethernet.h> - -} - -#if 0 -#define dprintf(...) log(LOG_INFO, __VA_ARGS__) -#else -#define dprintf(...) -#endif - -// These declarations are missing in the Kernel.framework headers, put present in userspace :-/ -#pragma pack(4) -struct ifmediareq { - char ifm_name[IFNAMSIZ]; /* if name, e.g. "en0" */ - int ifm_current; /* current media options */ - int ifm_mask; /* don't care mask */ - int ifm_status; /* media status */ - int ifm_active; /* active options */ - int ifm_count; /* # entries in ifm_ulist array */ - int *ifm_ulist; /* media words */ -}; - -struct ifmediareq64 { - char ifm_name[IFNAMSIZ]; /* if name, e.g. "en0" */ - int ifm_current; /* current media options */ - int ifm_mask; /* don't care mask */ - int ifm_status; /* media status */ - int ifm_active; /* active options */ - int ifm_count; /* # entries in ifm_ulist array */ - user64_addr_t ifmu_ulist __attribute__((aligned(8))); -}; - -struct ifmediareq32 { - char ifm_name[IFNAMSIZ]; /* if name, e.g. "en0" */ - int ifm_current; /* current media options */ - int ifm_mask; /* don't care mask */ - int ifm_status; /* media status */ - int ifm_active; /* active options */ - int ifm_count; /* # entries in ifm_ulist array */ - user32_addr_t ifmu_ulist; /* 32-bit pointer */ -}; -#pragma pack() - -#define SIOCGIFMEDIA32 _IOWR('i', 56, struct ifmediareq32) /* get net media */ -#define SIOCGIFMEDIA64 _IOWR('i', 56, struct ifmediareq64) /* get net media (64-bit) */ - -/* thread_policy_set is exported in Mach.kext, but commented in mach/thread_policy.h in the - * Kernel.Framework headers (why?). Add a local declaration to work around that. - */ -extern "C" { -kern_return_t thread_policy_set( - thread_t thread, - thread_policy_flavor_t flavor, - thread_policy_t policy_info, - mach_msg_type_number_t count); -} - -static unsigned char ETHER_BROADCAST_ADDR[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - -/* members */ -tap_interface::tap_interface() { - bzero(attached_protos, sizeof(attached_protos)); - input_thread = THREAD_NULL; -} - -bool -tap_interface::initialize(unsigned short major, unsigned short unit) -{ - this->unit = unit; - this->family_name = TAP_FAMILY_NAME; - this->family = IFNET_FAMILY_ETHERNET; - this->type = IFT_ETHER; - bzero(unique_id, UIDLEN); - snprintf(unique_id, UIDLEN, "%s%d", family_name, unit); - - dprintf("tap: starting interface %s%d\n", TAP_FAMILY_NAME, unit); - - /* register character device */ - if (!tuntap_interface::register_chardev(major)) - return false; - - return true; -} - -void -tap_interface::shutdown() -{ - dprintf("tap: shutting down tap interface %s%d\n", TAP_FAMILY_NAME, unit); - - unregister_chardev(); -} - -int -tap_interface::initialize_interface() -{ - struct sockaddr_dl lladdr; - lladdr.sdl_len = sizeof(lladdr); - lladdr.sdl_family = AF_LINK; - lladdr.sdl_alen = ETHER_ADDR_LEN; - lladdr.sdl_nlen = lladdr.sdl_slen = 0; - - /* generate a random MAC address */ - read_random(LLADDR(&lladdr), ETHER_ADDR_LEN); - - /* clear multicast bit and set local assignment bit (see IEEE 802) */ - (LLADDR(&lladdr))[0] &= 0xfe; - (LLADDR(&lladdr))[0] |= 0x02; - - dprintf("tap: random tap address: %02x:%02x:%02x:%02x:%02x:%02x\n", - (LLADDR(&lladdr))[0] & 0xff, - (LLADDR(&lladdr))[1] & 0xff, - (LLADDR(&lladdr))[2] & 0xff, - (LLADDR(&lladdr))[3] & 0xff, - (LLADDR(&lladdr))[4] & 0xff, - (LLADDR(&lladdr))[5] & 0xff); - - /* register interface */ - if (!tuntap_interface::register_interface(&lladdr, ETHER_BROADCAST_ADDR, ETHER_ADDR_LEN)) - return EIO; - - /* Set link level address. Yes, we need to do that again. Darwin sucks. */ - errno_t err = ifnet_set_lladdr(ifp, LLADDR(&lladdr), ETHER_ADDR_LEN); - if (err) - dprintf("tap: failed to set lladdr on %s%d: %d\n", family_name, unit, err); - - /* set mtu */ - ifnet_set_mtu(ifp, TAP_MTU); - /* set header length */ - ifnet_set_hdrlen(ifp, sizeof(struct ether_header)); - /* add the broadcast flag */ - ifnet_set_flags(ifp, IFF_BROADCAST, IFF_BROADCAST); - - /* we must call bpfattach(). Otherwise we deadlock BPF while unloading. Seems to be a bug in - * the kernel, see bpfdetach() in net/bpf.c, it will return without releasing the lock if - * the interface wasn't attached. I wonder what they were smoking while writing it ;-) - */ - bpfattach(ifp, DLT_EN10MB, ifnet_hdrlen(ifp)); - - /* Inject an empty packet to trigger the input thread calling demux(), which will unblock - * thread_sync_lock. This is part of a hack to avoid a kernel crash on re-attaching - * interfaces, see comment in shutdown_interface for more information. - */ - mbuf_t empty_mbuf; - mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &empty_mbuf); - if (empty_mbuf != NULL) { - mbuf_pkthdr_setrcvif(empty_mbuf, ifp); - mbuf_pkthdr_setlen(empty_mbuf, 0); - mbuf_pkthdr_setheader(empty_mbuf, mbuf_data(empty_mbuf)); - mbuf_set_csum_performed(empty_mbuf, 0, 0); - if (ifnet_input(ifp, empty_mbuf, NULL) == 0) { - auto_lock l(&thread_sync_lock); - for (int i = 0; i < 100 && input_thread == THREAD_NULL; ++i) { - dprintf("input thread not found, waiting...\n"); - thread_sync_lock.sleep(&input_thread, 10000000); - } - } else { - mbuf_freem(empty_mbuf); - } - } - if (input_thread == THREAD_NULL) - dprintf("Failed to determine input thread!\n"); - - return 0; -} - -void -tap_interface::shutdown_interface() -{ - dprintf("tap: shutting down network interface of device %s%d\n", TAP_FAMILY_NAME, unit); - - /* detach all protocols */ - for (unsigned int i = 0; i < MAX_ATTACHED_PROTOS; i++) { - if (attached_protos[i].used) { - errno_t err = ifnet_detach_protocol(ifp, attached_protos[i].proto); - if (err) - log(LOG_WARNING, "tap: could not detach protocol %d from %s%d\n", - attached_protos[i].proto, TAP_FAMILY_NAME, unit); - } - } - - cleanup_interface(); - unregister_interface(); - - /* There's a race condition in the kernel that may cause crashes when quickly re-attaching - * interfaces. The crash happens when the interface gets re-attached before the input thread - * for the interface managed to terminate, in which case an assert on the input_waiting flag - * to be clear triggers in ifnet_attach. The bug is really that there's no synchronization - * for terminating the input thread. To work around this, the following code does add the - * missing synchronization to wait for the input thread to terminate. Of course, threading - * primitives available to kexts are few, and I'm not aware of a way to wait for a thread to - * terminate. Hence, the code calls thread_policy_set (passing bogus parameters) in a loop, - * until it returns KERN_TERMINATED. Since this is all rather fragile, there's an upper - * limit on the loop iteratations we're willing to make, so this terminates eventually even - * if things change on the kernel side eventually. - */ - if (input_thread != THREAD_NULL) { - dprintf("Waiting for input thread...\n"); - kern_return_t result = 0; - for (int i = 0; i < 100; ++i) { - result = thread_policy_set(input_thread, -1, NULL, 0); - dprintf("thread_policy_set result: %d\n", result); - if (result == KERN_TERMINATED) { - dprintf("Input thread terminated.\n"); - thread_deallocate(input_thread); - input_thread = THREAD_NULL; - break; - } - - auto_lock l(&thread_sync_lock); - thread_sync_lock.sleep(&input_thread, 10000000); - } - } -} - -errno_t -tap_interface::if_ioctl(u_int32_t cmd, void *arg) -{ - dprintf("tap: if_ioctl cmd: %d (%x)\n", cmd & 0xff, cmd); - - switch (cmd) { - case SIOCSIFLLADDR: - { - /* set ethernet address */ - struct sockaddr *ea = &(((struct ifreq *) arg)->ifr_addr); - - dprintf("tap: SIOCSIFLLADDR family %d len %d\n", - ea->sa_family, ea->sa_len); - - /* check if it is really an ethernet address */ - if (ea->sa_family != AF_LINK || ea->sa_len != ETHER_ADDR_LEN) - return EINVAL; - - /* ok, copy */ - errno_t err = ifnet_set_lladdr(ifp, ea->sa_data, ETHER_ADDR_LEN); - if (err) { - dprintf("tap: failed to set lladdr on %s%d: %d\n", - family_name, unit, err); - return err; - } - - /* Generate a LINK_ON event. This necessary for configd to re-read - * the interface data and refresh the MAC address. Not doing so - * would result in the DHCP client using a stale MAC address... - */ - generate_link_event(KEV_DL_LINK_ON); - - return 0; - } - - case SIOCGIFMEDIA32: - case SIOCGIFMEDIA64: - { - struct ifmediareq *ifmr = (struct ifmediareq*) arg; - user_addr_t list = USER_ADDR_NULL; - - ifmr->ifm_current = IFM_ETHER; - ifmr->ifm_mask = 0; - ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; - ifmr->ifm_active = IFM_ETHER; - ifmr->ifm_count = 1; - - if (cmd == SIOCGIFMEDIA64) - list = ((struct ifmediareq64*) ifmr)->ifmu_ulist; - else - list = CAST_USER_ADDR_T( - ((struct ifmediareq32*) ifmr)->ifmu_ulist); - - if (list != USER_ADDR_NULL) - return copyout(&ifmr->ifm_current, list, sizeof(int)); - - return 0; - } - - default: - /* let our superclass handle it */ - return tuntap_interface::if_ioctl(cmd, arg); - } - - return EOPNOTSUPP; -} - -errno_t -tap_interface::if_demux(mbuf_t m, char *header, protocol_family_t *proto) -{ - struct ether_header *eh = (struct ether_header *) header; - unsigned char lladdr[ETHER_ADDR_LEN]; - - dprintf("tap: if_demux\n"); - - /* Make note of what input thread this interface is running on. This is part of a hack to - * avoid a crash on re-attaching interfaces, see comment in shutdown_interface for details. - */ - if (input_thread == THREAD_NULL) { - auto_lock l(&thread_sync_lock); - input_thread = current_thread(); - thread_reference(input_thread); - thread_sync_lock.wakeup(&input_thread); - } - - /* size check */ - if (mbuf_len(m) < sizeof(struct ether_header)) - return ENOENT; - - /* catch broadcast and multicast (stolen from bsd/net/ether_if_module.c) */ - if (eh->ether_dhost[0] & 1) { - if (memcmp(ETHER_BROADCAST_ADDR, eh->ether_dhost, ETHER_ADDR_LEN) == 0) { - /* broadcast */ - dprintf("tap: broadcast packet.\n"); - mbuf_setflags_mask(m, MBUF_BCAST, MBUF_BCAST); - } else { - /* multicast */ - dprintf("tap: multicast packet.\n"); - mbuf_setflags_mask(m, MBUF_MCAST, MBUF_MCAST); - } - } else { - /* check wether the packet has our address */ - ifnet_lladdr_copy_bytes(ifp, lladdr, ETHER_ADDR_LEN); - if (memcmp(lladdr, eh->ether_dhost, ETHER_ADDR_LEN) != 0) - mbuf_setflags_mask(m, MBUF_PROMISC, MBUF_PROMISC); - } - - /* find the protocol */ - for (unsigned int i = 0; i < MAX_ATTACHED_PROTOS; i++) { - if (attached_protos[i].used && attached_protos[i].type == eh->ether_type) { - *proto = attached_protos[i].proto; - return 0; - } - } - - dprintf("tap: if_demux() failed to find proto.\n"); - - /* no matching proto found */ - return ENOENT; -} - -errno_t -tap_interface::if_framer(mbuf_t *m, const struct sockaddr *dest, const char *dest_linkaddr, - const char *frame_type) -{ - struct ether_header *eh; - mbuf_t nm = *m; - errno_t err; - - dprintf("tap: if_framer\n"); - - /* prepend the ethernet header */ - err = mbuf_prepend(&nm, sizeof (struct ether_header), MBUF_WAITOK); - if (err) { - dprintf("tap: could not prepend data to mbuf: %d\n", err); - return err; - } - *m = nm; - - /* fill the header */ - eh = (struct ether_header *) mbuf_data(*m); - memcpy(eh->ether_dhost, dest_linkaddr, ETHER_ADDR_LEN); - ifnet_lladdr_copy_bytes(ifp, eh->ether_shost, ETHER_ADDR_LEN); - eh->ether_type = *((u_int16_t *) frame_type); - - return 0; -} - -errno_t -tap_interface::if_add_proto(protocol_family_t proto, const struct ifnet_demux_desc *desc, - u_int32_t ndesc) -{ - errno_t err; - - dprintf("tap: if_add_proto proto %d\n", proto); - - for (unsigned int i = 0; i < ndesc; i++) { - /* try to add the protocol */ - err = add_one_proto(proto, desc[i]); - if (err != 0) { - /* if that fails, remove everything stored so far */ - if_del_proto(proto); - return err; - } - } - - return 0; -} - -errno_t -tap_interface::if_del_proto(protocol_family_t proto) -{ - dprintf("tap: if_del_proto proto %d\n", proto); - - /* delete all matching entries in attached_protos */ - for (unsigned int i = 0; i < MAX_ATTACHED_PROTOS; i++) { - if (attached_protos[i].proto == proto) - attached_protos[i].used = false; - } - - return 0; -} - -errno_t -tap_interface::if_check_multi(const struct sockaddr *maddr) -{ - dprintf("tap: if_check_multi family %d\n", maddr->sa_family); - - /* see whether it is a ethernet address with the multicast bit set */ - if (maddr->sa_family == AF_LINK) { - struct sockaddr_dl *dlmaddr = (struct sockaddr_dl *) maddr; - if (LLADDR(dlmaddr)[0] & 0x01) - return 0; - else - return EADDRNOTAVAIL; - } - - return EOPNOTSUPP; -} - -errno_t -tap_interface::add_one_proto(protocol_family_t proto, const struct ifnet_demux_desc &dd) -{ - int free = -1; - u_int16_t dt; - - /* we only support DLIL_DESC_ETYPE2 */ - if (dd.type != DLIL_DESC_ETYPE2 || dd.datalen != 2) { - log(LOG_WARNING, "tap: tap only supports DLIL_DESC_ETYPE2 protocols.\n"); - return EINVAL; - } - - dt = *((u_int16_t *) (dd.data)); - - /* see if the protocol is already registered */ - for (unsigned int i = 0; i < MAX_ATTACHED_PROTOS; i++) { - if (attached_protos[i].used) { - if (dt == attached_protos[i].type) { - /* already registered */ - if (attached_protos[i].proto == proto) { - /* matches the old entry */ - return 0; - } else - return EEXIST; - } - } else if (free == -1) - free = i; - } - - /* did we find a free entry? */ - if (free == -1) - /* is ENOBUFS correct? */ - return ENOBUFS; - - /* ok, save information */ - attached_protos[free].used = true; - attached_protos[free].type = dt; - attached_protos[free].proto = proto; - - return 0; -} - -/* This code is shamelessly stolen from if_bond.c */ -void -tap_interface::generate_link_event(u_int32_t code) -{ - struct { - struct kern_event_msg header; - u_int32_t unit; - char if_name[IFNAMSIZ]; - } event; - - bzero(&event, sizeof(event)); - event.header.total_size = sizeof(event); - event.header.vendor_code = KEV_VENDOR_APPLE; - event.header.kev_class = KEV_NETWORK_CLASS; - event.header.kev_subclass = KEV_DL_SUBCLASS; - event.header.event_code = code; - event.header.event_data[0] = family; - event.unit = (u_int32_t) unit; - strncpy(event.if_name, ifnet_name(ifp), IFNAMSIZ); - - ifnet_event(ifp, &event.header); -} - -/* tap_manager members */ -tuntap_interface * -tap_manager::create_interface() -{ - return new tap_interface(); -} - diff --git a/ext/tap-mac/tuntap/src/tap/tap.h b/ext/tap-mac/tuntap/src/tap/tap.h deleted file mode 100644 index a5164d4a..00000000 --- a/ext/tap-mac/tuntap/src/tap/tap.h +++ /dev/null @@ -1,111 +0,0 @@ -/* - * ethertap device for MacOSX. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __TAP_H__ -#define __TAP_H__ - -#include "tuntap.h" - -extern "C" { - -#include <kern/thread.h> - -} - -#define TAP_FAMILY_NAME ((char *) "zt") -#define TAP_IF_COUNT 32 /* max number of tap interfaces */ -#define TAP_MTU 2800 -#define TAP_LLADDR tap_lladdr - -/* the mac address of our interfaces. note that the last byte will be replaced by the unit number */ -extern u_char tap_lladdr[]; - -/* tap manager */ -class tap_manager : public tuntap_manager { - - protected: - /* just define the interface creation method */ - virtual tuntap_interface *create_interface(); - -}; - -/* the tap network interface */ -class tap_interface : public tuntap_interface { - public: - tap_interface(); - - protected: - /* maximum number of protocols that can be attached */ - static const unsigned int MAX_ATTACHED_PROTOS = 8; - - /* information about attached protocols for demuxing is stored here */ - struct { - /* whether this entry is used */ - bool used; - /* type in the ethernet header */ - u_int16_t type; - /* protocol passed to add_proto */ - protocol_family_t proto; - } attached_protos[MAX_ATTACHED_PROTOS]; - - /* The input thread for the network interface. */ - thread_t input_thread; - - /* initializes the interface */ - virtual bool initialize(unsigned short major, unsigned short unit); - - /* shuts the interface down */ - virtual void shutdown(); - - /* called when the character device is opened in order to intialize the network - * interface. - */ - virtual int initialize_interface(); - /* called when the character device is closed to shutdown the network interface */ - virtual void shutdown_interface(); - - /* override interface routines */ - virtual errno_t if_ioctl(u_int32_t cmd, void *arg); - virtual errno_t if_demux(mbuf_t m, char *header, protocol_family_t *proto); - virtual errno_t if_framer(mbuf_t *m, const struct sockaddr *dest, - const char *dest_linkaddr, const char *frame_type); - virtual errno_t if_add_proto(protocol_family_t proto, - const struct ifnet_demux_desc *ddesc, u_int32_t ndesc); - virtual errno_t if_del_proto(protocol_family_t proto); - virtual errno_t if_check_multi(const struct sockaddr *maddr); - - /* if_add_proto helper */ - errno_t add_one_proto(protocol_family_t proto, const struct ifnet_demux_desc &dd); - - /* generates a kernel event */ - void generate_link_event(u_int32_t code); - - friend class tap_manager; -}; - -#endif /* __TAP_H__ */ - diff --git a/ext/tap-mac/tuntap/src/tuntap.cc b/ext/tap-mac/tuntap/src/tuntap.cc deleted file mode 100644 index d0f89018..00000000 --- a/ext/tap-mac/tuntap/src/tuntap.cc +++ /dev/null @@ -1,963 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. Common functionality of tap_interface and tun_interface. - * - * tuntap_interface class definition - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "tuntap.h" - -#if 0 -#define dprintf(...) log(LOG_INFO, __VA_ARGS__) -#else -#define dprintf(...) -#endif - -extern "C" { - -#include <sys/conf.h> -#include <sys/syslog.h> -#include <sys/param.h> -#include <sys/filio.h> -#include <sys/sockio.h> -#include <sys/fcntl.h> -#include <sys/kpi_socket.h> - -#include <vm/vm_kern.h> - -#include <net/if_types.h> -#include <net/if_var.h> -#include <net/if_dl.h> -#include <net/if_arp.h> - -#include <miscfs/devfs/devfs.h> - -} - -extern "C" { - -/* interface service functions that delegate to the appropriate tuntap_interface instance */ -errno_t -tuntap_if_output(ifnet_t ifp, mbuf_t m) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_output(m); - } - - if (m != NULL) - mbuf_freem_list(m); - - return ENODEV; -} - -errno_t -tuntap_if_ioctl(ifnet_t ifp, long unsigned int cmd, void *arg) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_ioctl(cmd, arg); - } - - return ENODEV; -} - -errno_t -tuntap_if_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, int (*cb)(ifnet_t, mbuf_t)) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_set_bpf_tap(mode, cb); - } - - return ENODEV; -} - -errno_t -tuntap_if_demux(ifnet_t ifp, mbuf_t m, char *header, protocol_family_t *proto) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_demux(m, header, proto); - } - - return ENODEV; -} - -errno_t -tuntap_if_framer(ifnet_t ifp, mbuf_t *m, const struct sockaddr *dest, const char *dest_linkaddr, - const char *frame_type) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_framer(m, dest, dest_linkaddr, frame_type); - } - - return ENODEV; -} - -errno_t -tuntap_if_add_proto(ifnet_t ifp, protocol_family_t proto, const struct ifnet_demux_desc *ddesc, - u_int32_t ndesc) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_add_proto(proto, ddesc, ndesc); - } - - return ENODEV; -} - -errno_t -tuntap_if_del_proto(ifnet_t ifp, protocol_family_t proto) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_del_proto(proto); - } - - return ENODEV; -} - -errno_t -tuntap_if_check_multi(ifnet_t ifp, const struct sockaddr* maddr) -{ - if (ifp != NULL) - { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - return ttif->if_check_multi(maddr); - } - - return ENODEV; -} - -void -tuntap_if_detached(ifnet_t ifp) -{ - if (ifp != NULL) { - tuntap_interface *ttif = (tuntap_interface *) ifnet_softc(ifp); - if (ttif != NULL) - ttif->if_detached(); - } -} - -errno_t -tuntap_if_noop_output(ifnet_t, mbuf_t) -{ - return ENODEV; -} - -errno_t -tuntap_if_noop_demux(ifnet_t, mbuf_t, char*, protocol_family_t*) -{ - return ENODEV; -} - -errno_t -tuntap_if_noop_add_proto(ifnet_t, protocol_family_t, const struct ifnet_demux_desc*, u_int32_t) -{ - return ENODEV; -} - -errno_t -tuntap_if_noop_del_proto(ifnet_t, protocol_family_t) -{ - return ENODEV; -} - -} /* extern "C" */ - -/* tuntap_mbuf_queue */ -tuntap_mbuf_queue::tuntap_mbuf_queue() -{ - head = tail = NULL; - size = 0; -} - -tuntap_mbuf_queue::~tuntap_mbuf_queue() -{ - clear(); -} - -bool -tuntap_mbuf_queue::enqueue(mbuf_t mb) -{ - if (size == QUEUE_SIZE) - return false; - - mbuf_setnextpkt(mb, NULL); - - if (head == NULL) - head = tail = mb; - else { - mbuf_setnextpkt(tail, mb); - tail = mb; - } - size++; - - return true; -} - -mbuf_t -tuntap_mbuf_queue::dequeue() -{ - mbuf_t ret; - - /* check wether there is a packet in the queue */ - if (head == NULL) - return NULL; - - /* fetch it */ - ret = head; - head = mbuf_nextpkt(head); - mbuf_setnextpkt(ret, NULL); - size--; - - return ret; -} - -void -tuntap_mbuf_queue::clear() -{ - /* free mbufs that are in the queue */ - if (head != NULL) - mbuf_freem_list(head); - - head = NULL; - tail = NULL; - size = 0; -} - -/* tuntap_interface members */ -tuntap_interface::tuntap_interface() -{ - /* initialize the members */ - ifp = NULL; - open = false; - block_io = true; - dev_handle = NULL; - pid = 0; - selthreadclear(&rsel); - bpf_mode = BPF_MODE_DISABLED; - bpf_callback = NULL; - bzero(unique_id, UIDLEN); - in_ioctl = false; -} - -tuntap_interface::~tuntap_interface() -{ -} - -bool -tuntap_interface::register_chardev(unsigned short major) -{ - /* register character device */ - dev_handle = devfs_make_node(makedev(major, unit), DEVFS_CHAR, 0, 0, 0660, "%s%d", - family_name, (int) unit); - - if (dev_handle == NULL) { - log(LOG_ERR, "tuntap: could not make /dev/%s%d\n", family_name, (int) unit); - return false; - } - - return true; -} - -void -tuntap_interface::unregister_chardev() -{ - dprintf("unregistering character device\n"); - - /* unregister character device */ - if (dev_handle != NULL) - devfs_remove(dev_handle); - dev_handle = NULL; -} - -bool -tuntap_interface::register_interface(const struct sockaddr_dl* lladdr, void *bcaddr, - u_int32_t bcaddrlen) -{ - struct ifnet_init_params ip; - errno_t err; - - dprintf("register_interface\n"); - - /* initialize an initialization info struct */ - ip.uniqueid_len = UIDLEN; - ip.uniqueid = unique_id; - ip.name = family_name; - ip.unit = unit; - ip.family = family; - ip.type = type; - ip.output = tuntap_if_output; - ip.demux = tuntap_if_demux; - ip.add_proto = tuntap_if_add_proto; - ip.del_proto = tuntap_if_del_proto; - ip.check_multi = tuntap_if_check_multi; - ip.framer = tuntap_if_framer; - ip.softc = this; - ip.ioctl = tuntap_if_ioctl; - ip.set_bpf_tap = tuntap_if_set_bpf_tap; - ip.detach = tuntap_if_detached; - ip.event = NULL; - ip.broadcast_addr = bcaddr; - ip.broadcast_len = bcaddrlen; - - dprintf("tuntap: tuntap_if_check_multi is at 0x%08x\n", (void*) tuntap_if_check_multi); - - /* allocate the interface */ - err = ifnet_allocate(&ip, &ifp); - if (err) { - log(LOG_ERR, "tuntap: could not allocate interface for %s%d: %d\n", family_name, - (int) unit, err); - ifp = NULL; - return false; - } - - /* activate the interface */ - err = ifnet_attach(ifp, lladdr); - if (err) { - log(LOG_ERR, "tuntap: could not attach interface %s%d: %d\n", family_name, - (int) unit, err); - ifnet_release(ifp); - ifp = NULL; - return false; - } - - dprintf("setting interface flags\n"); - - /* set interface flags */ - ifnet_set_flags(ifp, IFF_RUNNING | IFF_MULTICAST | IFF_SIMPLEX, (u_int16_t) ~0UL); - - dprintf("flags: %x\n", ifnet_flags(ifp)); - - return true; -} - -void -tuntap_interface::unregister_interface() -{ - errno_t err; - - dprintf("unregistering network interface\n"); - - if (ifp != NULL) { - interface_detached = false; - - /* detach interface */ - err = ifnet_detach(ifp); - if (err) - log(LOG_ERR, "tuntap: error detaching interface %s%d: %d\n", - family_name, unit, err); - - dprintf("interface detaching\n"); - - /* Wait until the interface has completely been detached. */ - thread_sync_lock.lock(); - while (!interface_detached) - thread_sync_lock.sleep(&interface_detached); - thread_sync_lock.unlock(); - - dprintf("interface detached\n"); - - /* release the interface */ - ifnet_release(ifp); - - ifp = NULL; - } - - dprintf("network interface unregistered\n"); -} - -void -tuntap_interface::cleanup_interface() -{ - errno_t err; - ifaddr_t *addrs; - ifaddr_t *a; - struct ifreq ifr; - - /* mark the interface down */ - ifnet_set_flags(ifp, 0, IFF_UP | IFF_RUNNING); - - /* Unregister all interface addresses. This works around a deficiency in the Darwin kernel. - * If we don't remove all IP addresses that are attached to the interface it can happen that - * the IP code fails to clean them up itself. When the interface is recycled, the IP code - * might then think some addresses are still attached to the interface... - */ - - err = ifnet_get_address_list(ifp, &addrs); - if (!err) { - - /* Execute a SIOCDIFADDR ioctl for each address. For technical reasons, we can only - * do that with a socket of the appropriate family. So try to create a dummy socket. - * I know this is a little expensive, but better than crashing... - * - * This really sucks. - */ - for (a = addrs; *a != NULL; a++) { - /* initialize the request parameters */ - snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d", - ifnet_name(ifp), ifnet_unit(ifp)); - ifaddr_address(*a, &(ifr.ifr_addr), sizeof(ifr.ifr_addr)); - if (ifr.ifr_addr.sa_family != AF_INET) - continue; - - dprintf("trying to delete address of family %d\n", ifr.ifr_addr.sa_family); - - do_sock_ioctl(ifr.ifr_addr.sa_family, SIOCDIFADDR, &ifr); - } - - /* release the address list */ - ifnet_free_address_list(addrs); - } -} - -bool -tuntap_interface::idle() -{ - return !(open); -} - -void -tuntap_interface::notify_bpf(mbuf_t mb, bool out) -{ - auto_lock l(&bpf_lock); - - if ((out && bpf_mode == BPF_MODE_OUTPUT) - || (!out && bpf_mode == BPF_MODE_INPUT) - || (bpf_mode == BPF_MODE_INPUT_OUTPUT)) - (*bpf_callback)(ifp, mb); -} - -void -tuntap_interface::do_sock_ioctl(sa_family_t af, unsigned long cmd, void* arg) { - if (in_ioctl) { - log(LOG_ERR, "tuntap: ioctl recursion detected, aborting.\n"); - return; - } - - socket_t sock; - errno_t err = sock_socket(af, SOCK_RAW, 0, NULL, NULL, &sock); - if (err) { - log(LOG_ERR, "tuntap: failed to create socket: %d\n", err); - return; - } - - in_ioctl = true; - - /* issue the ioctl */ - err = sock_ioctl(sock, cmd, arg); - if (err) - log(LOG_ERR, "tuntap: socket ioctl %d failed: %d\n", cmd, err); - - in_ioctl = false; - - /* get rid of the socket */ - sock_close(sock); -} - -/* character device service methods */ -int -tuntap_interface::cdev_open(int flags, int devtype, proc_t p) -{ - dprintf("tuntap: cdev_open()\n"); - - /* grab the lock so that there can only be one thread inside */ - auto_lock l(&lock); - - /* check wether it is already open */ - if (open) - return EBUSY; - - /* bring the network interface up */ - int error = initialize_interface(); - if (error) - return error; - - open = true; - pid = proc_pid(p); - - return 0; -} - -int -tuntap_interface::cdev_close(int flags, int devtype, proc_t p) -{ - dprintf("tuntap: cdev_close()\n"); - - auto_lock l(&lock); - - if (open) { - open = false; - - /* shut down the network interface */ - shutdown_interface(); - - /* clear the queue */ - send_queue.clear(); - - /* wakeup the cdev thread and notify selects */ - wakeup(this); - selwakeup(&rsel); - - return 0; - } - - return EBADF; -} - -int -tuntap_interface::cdev_read(uio_t uio, int ioflag) -{ - auto_lock l(&lock); - - unsigned int nb = 0; - int error; - - dprintf("tuntap: cdev read\n"); - - if (!open || ifp == NULL || !(ifnet_flags(ifp) & IFF_UP)) - return EIO; - - /* fetch a new mbuf from the queue if necessary */ - mbuf_t cur_mbuf = NULL; - while (cur_mbuf == NULL) { - dprintf("tuntap: fetching new mbuf\n"); - - cur_mbuf = send_queue.dequeue(); - if (cur_mbuf == NULL) { - /* nothing in queue, block or return */ - if (!block_io) { - dprintf("tuntap: aborting (nbio)\n"); - return EWOULDBLOCK; - } else { - /* block */ - dprintf("tuntap: waiting\n"); - /* release the lock while waiting */ - l.unlock(); - error = msleep(this, NULL, PZERO | PCATCH, "tuntap", NULL); - - l.lock(); - - if (error) - return error; - - /* see whether the device was closed in the meantime */ - if (!open || ifp == NULL || !(ifnet_flags(ifp) & IFF_UP)) - return EIO; - - } - } - } - - /* notify bpf */ - notify_bpf(cur_mbuf, true); - - /* output what we have */ - do { - dprintf("tuntap: got new mbuf: %p uio_resid: %d\n", cur_mbuf, uio_resid(uio)); - - /* now we have an mbuf */ - int chunk_len = min(mbuf_len(cur_mbuf), uio_resid(uio)); - error = uiomove((char *) mbuf_data(cur_mbuf), chunk_len, uio); - if (error) { - mbuf_freem(cur_mbuf); - return error; - } - nb += chunk_len; - - dprintf("tuntap: moved %d bytes to userspace uio_resid: %d\n", chunk_len, - uio_resid(uio)); - - /* update cur_mbuf */ - cur_mbuf = mbuf_free(cur_mbuf); - - } while (uio_resid(uio) > 0 && cur_mbuf != NULL); - - /* update statistics */ - ifnet_stat_increment_out(ifp, 1, nb, 0); - - /* still data left? forget about that ;-) */ - if (cur_mbuf != NULL) - mbuf_freem(cur_mbuf); - - dprintf("tuntap: read done\n"); - - return 0; -} - -int -tuntap_interface::cdev_write(uio_t uio, int ioflag) -{ - auto_lock l(&lock); - - if (!open || ifp == NULL || !(ifnet_flags(ifp) & IFF_UP)) - return EIO; - - dprintf("tuntap: cdev write. uio_resid: %d\n", uio_resid(uio)); - - /* pack the data into an mbuf chain */ - mbuf_t first, mb; - - /* first we need an mbuf having a header */ - mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &first); - if (first == NULL) { - log(LOG_ERR, "tuntap: could not get mbuf.\n"); - return ENOMEM; - } - mbuf_setlen(first, 0); - - unsigned int mlen = mbuf_maxlen(first); - unsigned int chunk_len; - unsigned int copied = 0; - unsigned int max_data_len = ifnet_mtu(ifp) + ifnet_hdrlen(ifp); - int error; - - /* stuff the data into the mbuf(s) */ - mb = first; - while (uio_resid(uio) > 0) { - /* copy a chunk. enforce mtu (don't know if this is correct behaviour) */ - chunk_len = min(max_data_len - copied, min(uio_resid(uio), mlen)); - error = uiomove((caddr_t) mbuf_data(mb), chunk_len, uio); - if (error) { - log(LOG_ERR, "tuntap: could not copy data from userspace: %d\n", error); - mbuf_freem(first); - return error; - } - - dprintf("tuntap: copied %d bytes, uio_resid %d\n", chunk_len, - uio_resid(uio)); - - mlen -= chunk_len; - mbuf_setlen(mb, mbuf_len(mb) + chunk_len); - copied += chunk_len; - - /* if done, break the loop */ - if (uio_resid(uio) <= 0 || copied >= max_data_len) - break; - - /* allocate a new mbuf if the current is filled */ - if (mlen == 0) { - mbuf_t next; - mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &next); - if (next == NULL) { - log(LOG_ERR, "tuntap: could not get mbuf.\n"); - mbuf_freem(first); - return ENOMEM; - } - mbuf_setnext(mb, next); - mb = next; - mbuf_setlen(mb, 0); - mlen = mbuf_maxlen(mb); - } - } - - /* fill in header info */ - mbuf_pkthdr_setrcvif(first, ifp); - mbuf_pkthdr_setlen(first, copied); - mbuf_pkthdr_setheader(first, mbuf_data(first)); - mbuf_set_csum_performed(first, 0, 0); - - /* update statistics */ - ifnet_stat_increment_in(ifp, 1, copied, 0); - - dprintf("tuntap: mbuf chain constructed. first: %p mb: %p len: %d data: %p\n", - first, mb, mbuf_len(first), mbuf_data(first)); - - /* notify bpf */ - notify_bpf(first, false); - - /* need to adjust the data pointer to point directly behind the linklevel header. The header - * itself is later accessed via m_pkthdr.header. Well, if something is ugly, here is it. - */ - mbuf_adj(first, ifnet_hdrlen(ifp)); - - /* pass the packet over to the network stack */ - error = ifnet_input(ifp, first, NULL); - - if (error) { - log(LOG_ERR, "tuntap: could not input packet into network stack.\n"); - mbuf_freem(first); - return error; - } - - return 0; -} - -int -tuntap_interface::cdev_ioctl(u_long cmd, caddr_t data, int fflag, proc_t p) -{ - auto_lock l(&lock); - - dprintf("tuntap: cdev ioctl: %d\n", (int) (cmd & 0xff)); - - switch (cmd) { - case FIONBIO: - /* set i/o mode */ - block_io = *((int *) data) ? false : true; - return 0; - case FIOASYNC: - /* don't allow switching it on */ - if (*((int *) data)) - return ENOTTY; - return 0; - } - - return ENOTTY; -} - -int -tuntap_interface::cdev_select(int which, void *wql, proc_t p) -{ - auto_lock l(&lock); - - int ret = 0; - - dprintf("tuntap: select. which: %d\n", which); - - switch (which) { - case FREAD: - /* check wether data is available */ - { - if (!send_queue.empty()) - ret = 1; - else { - dprintf("tuntap: select: waiting\n"); - selrecord(p, &rsel, wql); - } - } - break; - case FWRITE: - /* we are always writeable */ - ret = 1; - } - - return ret; -} - -/* interface service methods */ -errno_t -tuntap_interface::if_output(mbuf_t m) -{ - mbuf_t pkt; - - dprintf("tuntap: if output\n"); - - /* just to be sure */ - if (m == NULL) - return 0; - - if (!open || ifp == NULL || !(ifnet_flags(ifp) & IFF_UP)) { - mbuf_freem_list(m); - return EHOSTDOWN; - } - - /* check whether packet has a header */ - if ((mbuf_flags(m) & MBUF_PKTHDR) == 0) { - log(LOG_ERR, "tuntap: packet to be output has no mbuf header.\n"); - mbuf_freem_list(m); - return EINVAL; - } - - /* put the packet(s) into the output queue */ - while (m != NULL) { - /* keep pointer, iterate */ - pkt = m; - m = mbuf_nextpkt(m); - mbuf_setnextpkt(pkt, NULL); - - auto_lock l(&lock); - - if (!send_queue.enqueue(pkt)) { - mbuf_freem(pkt); - mbuf_freem_list(m); - return ENOBUFS; - } - } - - /* protect the wakeup calls with the lock, not sure they are safe. */ - { - auto_lock l(&lock); - - /* wakeup the cdev thread and notify selects */ - wakeup(this); - selwakeup(&rsel); - } - - return 0; -} - -errno_t -tuntap_interface::if_ioctl(u_int32_t cmd, void *arg) -{ - dprintf("tuntap: if ioctl: %d\n", (int) (cmd & 0xff)); - - switch (cmd) { - case SIOCSIFADDR: - { - dprintf("tuntap: if_ioctl: SIOCSIFADDR\n"); - - /* Unfortunately, ifconfig sets the address family field of an INET - * netmask to zero, which makes early mDNSresponder versions ignore - * the interface. Fix that here. This one is of the category "ugly - * workaround". Dumb Darwin... - * - * Meanwhile, Apple has fixed mDNSResponder, and recent versions of - * Leopard don't need this hack anymore. However, Tiger still has a - * broken version so we leave the hack in for now. - * - * TODO: Revisit when dropping Tiger support. - * - * Btw. If you configure other network interfaces using ifconfig, - * you run into the same problem. I still don't know how to make the - * tap devices show up in the network configuration panel... - */ - ifaddr_t ifa = (ifaddr_t) arg; - if (ifa == NULL) - return 0; - - sa_family_t af = ifaddr_address_family(ifa); - if (af != AF_INET) - return 0; - - struct ifaliasreq ifra; - int sa_size = sizeof(struct sockaddr); - if (ifaddr_address(ifa, &ifra.ifra_addr, sa_size) - || ifaddr_dstaddress(ifa, &ifra.ifra_broadaddr, sa_size) - || ifaddr_netmask(ifa, &ifra.ifra_mask, sa_size)) { - log(LOG_WARNING, - "tuntap: failed to parse interface address.\n"); - return 0; - } - - // Check that the address family fields match. If not, issue another - // SIOCAIFADDR to fix the entry. - if (ifra.ifra_addr.sa_family != af - || ifra.ifra_broadaddr.sa_family != af - || ifra.ifra_mask.sa_family != af) { - log(LOG_INFO, "tuntap: Fixing address family for %s%d\n", - family_name, unit); - - snprintf(ifra.ifra_name, sizeof(ifra.ifra_name), "%s%d", - family_name, unit); - ifra.ifra_addr.sa_family = af; - ifra.ifra_broadaddr.sa_family = af; - ifra.ifra_mask.sa_family = af; - - do_sock_ioctl(af, SIOCAIFADDR, &ifra); - } - - return 0; - } - - case SIOCSIFFLAGS: - return 0; - - case SIOCGIFSTATUS: - { - struct ifstat *stat = (struct ifstat *) arg; - int len; - char *p; - - if (stat == NULL) - return EINVAL; - - /* print status */ - len = strlen(stat->ascii); - p = stat->ascii + len; - if (open) { - snprintf(p, IFSTATMAX - len, "\topen (pid %u)\n", pid); - } else { - snprintf(p, IFSTATMAX - len, "\tclosed\n"); - } - - return 0; - } - - case SIOCSIFMTU: - { - struct ifreq *ifr = (struct ifreq *) arg; - - if (ifr == NULL) - return EINVAL; - - ifnet_set_mtu(ifp, ifr->ifr_mtu); - - return 0; - } - - case SIOCDIFADDR: - return 0; - - } - - return EOPNOTSUPP; -} - -errno_t -tuntap_interface::if_set_bpf_tap(bpf_tap_mode mode, int (*cb)(ifnet_t, mbuf_t)) -{ - dprintf("tuntap: mode %d\n", mode); - - auto_lock l(&bpf_lock); - - bpf_callback = cb; - bpf_mode = mode; - - return 0; -} - -errno_t -tuntap_interface::if_check_multi(const struct sockaddr *maddr) -{ - dprintf("tuntap: if_check_multi\n"); - - return EOPNOTSUPP; -} - -void -tuntap_interface::if_detached() -{ - dprintf("tuntap: if_detached\n"); - - /* wake unregister_interface() */ - thread_sync_lock.lock(); - interface_detached = true; - thread_sync_lock.wakeup(&interface_detached); - thread_sync_lock.unlock(); - - dprintf("if_detached done\n"); -} - diff --git a/ext/tap-mac/tuntap/src/tuntap.h b/ext/tap-mac/tuntap/src/tuntap.h deleted file mode 100644 index d5f398d0..00000000 --- a/ext/tap-mac/tuntap/src/tuntap.h +++ /dev/null @@ -1,301 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. - * - * The class tuntaptap_interface contains the common functionality of tuntap_interface and - * tap_interface. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __TUNTAP_H__ -#define __TUNTAP_H__ - -#include "util.h" -#include "lock.h" - -extern "C" { - -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/select.h> -#include <sys/systm.h> -#include <sys/kpi_mbuf.h> - -#include <kern/locks.h> - -#include <net/if.h> -#include <net/bpf.h> -#include <net/kpi_interface.h> - -} - -extern "C" { - -errno_t tuntap_if_output(ifnet_t ifp, mbuf_t m); -errno_t tuntap_if_ioctl(ifnet_t ifp, long unsigned int cmd, void *arg); -errno_t tuntap_if_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, int (*cb)(ifnet_t, mbuf_t)); -errno_t tuntap_if_demux(ifnet_t ifp, mbuf_t m, char *header, protocol_family_t *proto); -errno_t tuntap_if_framer(ifnet_t ifp, mbuf_t *m, const struct sockaddr *dest, - const char *dest_linkaddr, const char *frame_type); -errno_t tuntap_if_add_proto(ifnet_t ifp, protocol_family_t proto, - const struct ifnet_demux_desc *ddesc, u_int32_t ndesc); -errno_t tuntap_if_del_proto(ifnet_t ifp, protocol_family_t proto); -errno_t tuntap_if_check_multi(ifnet_t ifp, const struct sockaddr *maddr); -void tuntap_if_detached(ifnet_t ifp); - -} - -/* forward declaration */ -class tuntap_interface; - -/* both interface families have their manager object that will create, initialize, shutdown and - * delete interfaces. This is (mostly) generic so it can be used both for tun and tap. The only - * exception is the interface creation, therefore this class is abstract. tun and tap have their own - * versions that simply fill in create_interface(). - */ -class tuntap_manager { - - protected: - /* manager cdev gate */ - tt_gate cdev_gate; - /* interface count */ - unsigned int count; - /* an array holding all the interface instances */ - tuntap_interface **tuntaps; - /* the major device number */ - int dev_major; - /* family name */ - char *family; - - /* wether static members are initialized */ - static bool statics_initialized; - - /* major-to-manager-map */ - static const int MAX_CDEV = 256; - static tuntap_manager *mgr_map[MAX_CDEV]; - - /* initializes static members */ - void initialize_statics(); - - public: - /* sets major device number, allocates the interface table. */ - bool initialize(unsigned int count, char *family); - - /* tries to shutdown the family. returns true if successful. the manager object may - * not be deleted if this wasn't called successfully. - */ - bool shutdown(); - - /* the destructor deletes allocated memory and unregisters the character device - * switch */ - virtual ~tuntap_manager(); - - /* here are the cdev routines for the class. They will figure out the manager object - * and call the service methods declared below. - */ - static int cdev_open(dev_t dev, int flags, int devtype, proc_t p); - static int cdev_close(dev_t dev, int flags, int devtype, proc_t p); - static int cdev_read(dev_t dev, uio_t uio, int ioflag); - static int cdev_write(dev_t dev, uio_t uio, int ioflag); - static int cdev_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, - proc_t p); - static int cdev_select(dev_t dev, int which, void *wql, proc_t p); - - protected: - /* Here are the actual service routines that will do the required things (creating - * interfaces and such) and forward to the interface's implementation. - */ - int do_cdev_open(dev_t dev, int flags, int devtype, proc_t p); - int do_cdev_close(dev_t dev, int flags, int devtype, proc_t p); - int do_cdev_read(dev_t dev, uio_t uio, int ioflag); - int do_cdev_write(dev_t dev, uio_t uio, int ioflag); - int do_cdev_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, proc_t p); - int do_cdev_select(dev_t dev, int which, void *wql, proc_t p); - - /* abstract method that will create an interface. Implemented by tun and tap */ - virtual tuntap_interface *create_interface() = 0; - - /* makes sure there is one idle interface available (if nothing fails */ - void ensure_idle_device(); - -}; - -/* a class implementing a mbuf packet queue. On Darwin 7 we had struct ifqueue, but that is now - * internal to the kernel for Darwin 8. So lets have our own. - */ -class tuntap_mbuf_queue { - - private: - /* output end of the queue. dequeueing takes mbufs from here */ - mbuf_t head; - /* input end. new mbufs are appended here. */ - mbuf_t tail; - - /* size */ - unsigned int size; - - /* maximum queue size */ - static const unsigned int QUEUE_SIZE = 128; - - public: - /* initialize new empty queue */ - tuntap_mbuf_queue(); - ~tuntap_mbuf_queue(); - - /* is the queue full? */ - bool full() { return size == QUEUE_SIZE; } - /* is it emtpy? */ - bool empty() { return size == 0; } - - /* enqueue an mbuf. returns true if there was space left, so the mbuf could be - * queued, false otherwise */ - bool enqueue(mbuf_t mb); - - /* tries to dequeue the next mbuf. If the queue is empty, NULL is returned */ - mbuf_t dequeue(); - - /* makes the queue empty, discarding any queue packets */ - void clear(); -}; - -class tuntap_interface { - - protected: - /* interface number */ - unsigned int unit; - /* family name */ - char *family_name; - /* family identifier */ - ifnet_family_t family; - /* interface type */ - u_int32_t type; - /* id string */ - static const unsigned int UIDLEN = 20; - char unique_id[UIDLEN]; - - /* synchronization */ - tt_mutex lock; - tt_mutex bpf_lock; - tt_mutex thread_sync_lock; - - /* the interface structure registered */ - ifnet_t ifp; - /* whether the device has been opened */ - bool open; - /* whether we are doing blocking i/o */ - bool block_io; - /* whether the interface has properly been detached */ - bool interface_detached; - /* handle to the devfs node for the character device */ - void *dev_handle; - /* the pid of the process that opened the cdev, if any */ - pid_t pid; - /* read select info */ - struct selinfo rsel; - /* bpf mode, wether filtering is on or off */ - bpf_tap_mode bpf_mode; - /* bpf callback. called when packet arrives/leaves */ - int (*bpf_callback)(ifnet_t, mbuf_t); - /* pending packets queue (for output), must be accessed with the lock held */ - tuntap_mbuf_queue send_queue; - /* whether an ioctl that we issued is currently being processed */ - bool in_ioctl; - - /* protected constructor. initializes most of the members */ - tuntap_interface(); - virtual ~tuntap_interface(); - - /* initialize the device */ - virtual bool initialize(unsigned short major, unsigned short unit) = 0; - - /* character device management */ - virtual bool register_chardev(unsigned short major); - virtual void unregister_chardev(); - - /* network interface management */ - virtual bool register_interface(const struct sockaddr_dl *lladdr, - void *bcaddr, u_int32_t bcaddrlen); - virtual void unregister_interface(); - virtual void cleanup_interface(); - - /* called when the character device is opened in order to intialize the network - * interface. - */ - virtual int initialize_interface() = 0; - /* called when the character device is closed to shutdown the network interface */ - virtual void shutdown_interface() = 0; - - /* check wether the interface is idle (so it can be brought down) */ - virtual bool idle(); - - /* shut it down */ - virtual void shutdown() = 0; - - /* notifies BPF of a packet coming through */ - virtual void notify_bpf(mbuf_t mb, bool out); - - /* executes a socket ioctl through a temporary socket */ - virtual void do_sock_ioctl(sa_family_t af, unsigned long cmd, void* arg); - - /* character device service methods. Called by the manager */ - virtual int cdev_open(int flags, int devtype, proc_t p); - virtual int cdev_close(int flags, int devtype, proc_t p); - virtual int cdev_read(uio_t uio, int ioflag); - virtual int cdev_write(uio_t uio, int ioflag); - virtual int cdev_ioctl(u_long cmd, caddr_t data, int fflag, proc_t p); - virtual int cdev_select(int which, void *wql, proc_t p); - - /* interface functions. friends and implementation methods */ - friend errno_t tuntap_if_output(ifnet_t ifp, mbuf_t m); - friend errno_t tuntap_if_ioctl(ifnet_t ifp, long unsigned int cmd, void *arg); - friend errno_t tuntap_if_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, - int (*cb)(ifnet_t, mbuf_t)); - friend errno_t tuntap_if_demux(ifnet_t ifp, mbuf_t m, char *header, - protocol_family_t *proto); - friend errno_t tuntap_if_framer(ifnet_t ifp, mbuf_t *m, const struct sockaddr *dest, - const char *dest_linkaddr, const char *frame_type); - friend errno_t tuntap_if_add_proto(ifnet_t ifp, protocol_family_t proto, - const struct ifnet_demux_desc *ddesc, u_int32_t ndesc); - friend errno_t tuntap_if_del_proto(ifnet_t ifp, protocol_family_t proto); - friend errno_t tuntap_if_check_multi(ifnet_t ifp, const struct sockaddr *maddr); - friend void tuntap_if_detached(ifnet_t ifp); - - virtual errno_t if_output(mbuf_t m); - virtual errno_t if_ioctl(u_int32_t cmd, void *arg); - virtual errno_t if_set_bpf_tap(bpf_tap_mode mode, int (*cb)(ifnet_t, mbuf_t)); - virtual errno_t if_demux(mbuf_t m, char *header, protocol_family_t *proto) = 0; - virtual errno_t if_framer(mbuf_t *m, const struct sockaddr *dest, - const char *dest_linkaddr, const char *frame_type) = 0; - virtual errno_t if_add_proto(protocol_family_t proto, - const struct ifnet_demux_desc *ddesc, u_int32_t ndesc) = 0; - virtual errno_t if_del_proto(protocol_family_t proto) = 0; - virtual errno_t if_check_multi(const struct sockaddr *maddr); - virtual void if_detached(); - - /* tuntap_manager feeds us with cdev input, so it is our friend */ - friend class tuntap_manager; -}; - -#endif /* __TUNTAP_H__ */ - diff --git a/ext/tap-mac/tuntap/src/tuntap_mgr.cc b/ext/tap-mac/tuntap/src/tuntap_mgr.cc deleted file mode 100644 index f41394e9..00000000 --- a/ext/tap-mac/tuntap/src/tuntap_mgr.cc +++ /dev/null @@ -1,372 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. - * - * tuntap_manager definition. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "tuntap.h" -#include "mem.h" - -extern "C" { - -#include <sys/conf.h> -#include <sys/param.h> -#include <sys/syslog.h> -#include <sys/systm.h> - -#include <vm/vm_kern.h> - -#include <miscfs/devfs/devfs.h> - -} - -#if 0 -#define dprintf(...) log(LOG_INFO, __VA_ARGS__) -#else -#define dprintf(...) -#endif - -/* cdevsw for tuntap_manager */ -static struct cdevsw mgr_cdevsw = -{ - tuntap_manager::cdev_open, - tuntap_manager::cdev_close, - tuntap_manager::cdev_read, - tuntap_manager::cdev_write, - tuntap_manager::cdev_ioctl, - eno_stop, - eno_reset, - NULL, - tuntap_manager::cdev_select, - eno_mmap, - eno_strat, - eno_getc, - eno_putc, - 0 -}; - -/* tuntap_manager members */ -tuntap_manager *tuntap_manager::mgr_map[MAX_CDEV]; - -bool tuntap_manager::statics_initialized = false; - -/* static initializer */ -void -tuntap_manager::initialize_statics() -{ - dprintf("initializing mgr_map\n"); - - /* initialize the major-to-manager map */ - for (int i = 0; i < MAX_CDEV; i++) - mgr_map[i] = NULL; - - statics_initialized = true; -} - -bool -tuntap_manager::initialize(unsigned int count, char *family) -{ - this->count = count; - this->family = family; - this->tuntaps = NULL; - - if (!statics_initialized) - initialize_statics(); - - /* make sure noone can access the character devices until we are done */ - auto_lock l(&cdev_gate); - - /* register the switch for the tap character devices */ - dev_major = cdevsw_add(-1, &mgr_cdevsw); - if (dev_major == -1) { - log(LOG_ERR, "%s: could not register character device switch.\n", family); - return false; - } - - /* allocate memory for the interface instance table */ - tuntaps = (tuntap_interface **) mem_alloc(count * sizeof(tuntap_interface *)); - if (tuntaps == NULL) - { - log(LOG_ERR, "%s: no memory!\n", family); - return false; - } - - bzero(tuntaps, count * sizeof(tuntap_interface *)); - - /* Create the interfaces. This will only add the character devices. The network devices will - * be created upon open()ing the corresponding character devices. - */ - for (int i = 0; i < (int) count; i++) - { - tuntaps[i] = create_interface(); - - if (tuntaps[i] != NULL) - { - if (tuntaps[i]->initialize(dev_major, i)) - { - continue; - } - - /* error here. current interface needs to be shut down */ - i++; - } - - /* something went wrong. clean up. */ - while (--i >= 0) - { - tuntaps[i]->shutdown(); - delete tuntaps[i]; - } - - return false; - } - - /* register the new family in the mgr switch */ - mgr_map[dev_major] = this; - - log(LOG_INFO, "%s kernel extension version %s <mattias.nissler@gmx.de>\n", - family, TUNTAP_VERSION); - - return true; -} - -bool -tuntap_manager::shutdown() -{ - bool ok = true; - - /* we halt the whole thing while we check whether we can shutdown */ - auto_lock l(&cdev_gate); - - /* anyone in? */ - if (cdev_gate.is_anyone_in()) { - dprintf("tuntap_mgr: won't shutdown, threads still behind the gate."); - ok = false; - } else { - /* query the interfaces to see if shutting down is ok */ - if (tuntaps != NULL) { - for (unsigned int i = 0; i < count; i++) { - if (tuntaps[i] != NULL) - ok &= tuntaps[i]->idle(); - } - - /* if yes, do it now */ - if (ok) { - for (unsigned int i = 0; i < count; i++) { - if (tuntaps[i] != NULL) { - tuntaps[i]->shutdown(); - delete tuntaps[i]; - tuntaps[i] = NULL; - } - } - } - } - } - - /* unregister the character device switch */ - if (ok) { - if (dev_major != -1 && cdevsw_remove(dev_major, &mgr_cdevsw) == -1) { - log(LOG_WARNING, - "%s: character device switch got lost. strange.\n", family); - } - mgr_map[dev_major] = NULL; - dev_major = -1; - - /* at this point there is still a chance that some thread hangs at the cdev_gate in - * one of the cdev service functions. I can't imagine any way that would aviod this. - * So lets unblock the gate such that they fail. - */ - unsigned int old_number; - do { - old_number = cdev_gate.get_ticket_number(); - - dprintf("tuntap_manager: waiting for other threads to give up.\n"); - - /* wait one second */ - cdev_gate.sleep(&cdev_gate, 1000000); - - } while (cdev_gate.get_ticket_number() != old_number); - - /* I hope it is safe to unload now. */ - - } else { - log(LOG_WARNING, "%s: won't unload, at least one interface is busy.\n", family); - } - - dprintf("tuntap manager: shutdown %s\n", ok ? "ok" : "failed"); - - return ok; -} - -tuntap_manager::~tuntap_manager() -{ - dprintf("freeing interface table\n"); - - /* free memory */ - if (tuntaps != NULL) - mem_free(tuntaps, count * sizeof(tuntap_interface *)); -} - -/* service method dispatchers */ -int -tuntap_manager::cdev_open(dev_t dev, int flags, int devtype, proc_t p) -{ - return (mgr_map[major(dev)] == NULL ? ENOENT - : mgr_map[major(dev)]->do_cdev_open(dev, flags, devtype, p)); -} - -int -tuntap_manager::cdev_close(dev_t dev, int flags, int devtype, proc_t p) -{ - return (mgr_map[major(dev)] == NULL ? EBADF - : mgr_map[major(dev)]->do_cdev_close(dev, flags, devtype, p)); -} - -int -tuntap_manager::cdev_read(dev_t dev, uio_t uio, int ioflag) -{ - return (mgr_map[major(dev)] == NULL ? EBADF - : mgr_map[major(dev)]->do_cdev_read(dev, uio, ioflag)); -} - -int -tuntap_manager::cdev_write(dev_t dev, uio_t uio, int ioflag) -{ - return (mgr_map[major(dev)] == NULL ? EBADF - : mgr_map[major(dev)]->do_cdev_write(dev, uio, ioflag)); -} - -int -tuntap_manager::cdev_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, proc_t p) -{ - return (mgr_map[major(dev)] == NULL ? EBADF - : mgr_map[major(dev)]->do_cdev_ioctl(dev, cmd, data, fflag, p)); -} - -int -tuntap_manager::cdev_select(dev_t dev, int which, void *wql, proc_t p) -{ - return (mgr_map[major(dev)] == NULL ? EBADF - : mgr_map[major(dev)]->do_cdev_select(dev, which, wql, p)); -} - -/* character device service methods */ -int -tuntap_manager::do_cdev_open(dev_t dev, int flags, int devtype, proc_t p) -{ - int dmin = minor(dev); - int error = ENOENT; - - cdev_gate.enter(); - - if (dmin < (int) count && dmin >= 0 && tuntaps[dmin] != NULL) - error = tuntaps[dmin]->cdev_open(flags, devtype, p); - - cdev_gate.exit(); - - return error; -} - -int -tuntap_manager::do_cdev_close(dev_t dev, int flags, int devtype, proc_t p) -{ - int dmin = minor(dev); - int error = EBADF; - - cdev_gate.enter(); - - if (dmin < (int) count && dmin >= 0 && tuntaps[dmin] != NULL) - error = tuntaps[dmin]->cdev_close(flags, devtype, p); - - cdev_gate.exit(); - - return error; -} - -int -tuntap_manager::do_cdev_read(dev_t dev, uio_t uio, int ioflag) -{ - int dmin = minor(dev); - int error = EBADF; - - cdev_gate.enter(); - - if (dmin < (int) count && dmin >= 0 && tuntaps[dmin] != NULL) - error = tuntaps[dmin]->cdev_read(uio, ioflag); - - cdev_gate.exit(); - - return error; -} - -int -tuntap_manager::do_cdev_write(dev_t dev, uio_t uio, int ioflag) -{ - int dmin = minor(dev); - int error = EBADF; - - cdev_gate.enter(); - - if (dmin < (int) count && dmin >= 0 && tuntaps[dmin] != NULL) - error = tuntaps[dmin]->cdev_write(uio, ioflag); - - cdev_gate.exit(); - - return error; -} - -int -tuntap_manager::do_cdev_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, proc_t p) -{ - int dmin = minor(dev); - int error = EBADF; - - cdev_gate.enter(); - - if (dmin < (int) count && dmin >= 0 && tuntaps[dmin] != NULL) - error = tuntaps[dmin]->cdev_ioctl(cmd, data, fflag, p); - - cdev_gate.exit(); - - return error; -} - -int -tuntap_manager::do_cdev_select(dev_t dev, int which, void *wql, proc_t p) -{ - int dmin = minor(dev); - int error = EBADF; - - cdev_gate.enter(); - - if (dmin < (int) count && dmin >= 0 && tuntaps[dmin] != NULL) - error = tuntaps[dmin]->cdev_select(which, wql, p); - - cdev_gate.exit(); - - return error; -} - diff --git a/ext/tap-mac/tuntap/src/util.h b/ext/tap-mac/tuntap/src/util.h deleted file mode 100644 index 0f6955e8..00000000 --- a/ext/tap-mac/tuntap/src/util.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * ip tunnel/ethertap device for MacOSX. - * - * Some utilities and misc stuff. - */ -/* - * Copyright (c) 2011 Mattias Nissler <mattias.nissler@gmx.de> - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials provided - * with the distribution. - * 3. The name of the author may not be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __UTIL_H__ -#define __UTIL_H__ - -extern "C" { - -/* In Darwin 8 (OS X Tiger) there is a problem with struct selinfo. It was made `private' to the - * kernel, so its definition is not available from the headers in Kernel.framework. However, we need - * to declare something :-( - */ -struct selinfo { - char data[128]; /* should be enough... */ -}; - -} /* extern "C" */ - -#endif /* __UTIL_H__ */ - diff --git a/include/ZeroTierDebug.h b/include/ZeroTierDebug.h index 8e5366f0..561b5a63 100644 --- a/include/ZeroTierDebug.h +++ b/include/ZeroTierDebug.h @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -76,6 +76,9 @@ #ifdef __APPLE__ #define ZT_THREAD_ID (long)0 // (long)gettid() #endif +#ifdef __FreeBSD__ + #define ZT_THREAD_ID (long)0 // (long)gettid() +#endif #ifdef _WIN32 #define ZT_THREAD_ID (long)0 // #endif @@ -86,7 +89,7 @@ #include <android/log.h> #define ZT_LOG_TAG "ZTSDK" #endif -#if defined(ZT_TRACE) +#if defined(ZT_DEBUG_TRACE) #if ZT_MSG_INFO == true #if defined(__ANDROID__) #define DEBUG_INFO(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \ @@ -96,7 +99,7 @@ #define DEBUG_INFO(fmt, ...) fprintf(stderr, ZT_GRN "INFO [%ld]: %17s:%5d:%25s: " fmt "\n" \ ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__) #endif - #if defined(__linux__) or defined(__APPLE__) + #if defined(__linux__) or defined(__APPLE__) or defined(__FreeBSD__) #define DEBUG_INFO(fmt, args ...) fprintf(stderr, ZT_GRN "INFO [%ld]: %17s:%5d:%25s: " fmt "\n" \ ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args) #endif diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 6da53a73..74600989 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -102,10 +102,11 @@ extern "C" { /** * Default UDP payload size (physical path MTU) not including UDP and IP overhead * - * This is 1500 - IPv6 UDP overhead - PPPoE overhead and is safe for 99.9% of - * all Internet links. + * This is small enough for PPPoE and for Google Cloud's bizarrely tiny MTUs. + * A 2800 byte payload still fits into two packets, so this should not impact + * real world throughput at all vs the previous default of 1444. */ -#define ZT_DEFAULT_PHYSMTU 1444 +#define ZT_DEFAULT_PHYSMTU 1432 /** * Maximum physical UDP payload @@ -411,7 +412,7 @@ enum ZT_ResultCode ZT_RESULT_ERROR_UNSUPPORTED_OPERATION = 1001, /** - * The requestion operation was given a bad parameter or was called in an invalid state + * The requested operation was given a bad parameter or was called in an invalid state */ ZT_RESULT_ERROR_BAD_PARAMETER = 1002 }; @@ -423,6 +424,35 @@ enum ZT_ResultCode #define ZT_ResultCode_isFatal(x) ((((int)(x)) >= 100)&&(((int)(x)) < 1000)) /** + * The multipath algorithm in use by this node. + */ +enum ZT_MultipathMode +{ + /** + * No active multipath. + * + * Traffic is merely sent over the strongest path. That being + * said, this mode will automatically failover in the event that a link goes down. + */ + ZT_MULTIPATH_NONE = 0, + + /** + * Traffic is randomly distributed among all active paths. + * + * Will cease sending traffic over links that appear to be stale. + */ + ZT_MULTIPATH_RANDOM = 1, + + /** + * Traffic is allocated across all active paths in proportion to their strength and + * reliability. + * + * Will cease sending traffic over links that appear to be stale. + */ + ZT_MULTIPATH_PROPORTIONALLY_BALANCED = 2, +}; + +/** * Status codes sent to status update callback when things happen */ enum ZT_Event @@ -621,6 +651,24 @@ typedef struct } ZT_NodeStatus; /** + * Internal node statistics + * + * This structure is subject to change between versions. + */ +typedef struct +{ + /** + * Number of each protocol verb (possible verbs 0..31) received + */ + uint64_t inVerbCounts[32]; + + /** + * Number of bytes for each protocol verb received + */ + uint64_t inVerbBytes[32]; +} ZT_NodeStatistics; + +/** * Virtual network status codes */ enum ZT_VirtualNetworkStatus @@ -716,6 +764,11 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_ACTION_BREAK = 5, /** + * Place a matching frame in the specified QoS bucket + */ + ZT_NETWORK_RULE_ACTION_PRIORITY = 6, + + /** * Maximum ID for an ACTION, anything higher is a MATCH */ ZT_NETWORK_RULE_ACTION__MAX_ID = 15, @@ -905,6 +958,11 @@ typedef struct uint32_t flags; uint16_t length; } fwd; + + /** + * Quality of Service (QoS) bucket we want a frame to be placed in + */ + uint8_t qosBucket; } v; } ZT_VirtualNetworkRule; @@ -1038,7 +1096,8 @@ enum ZT_Architecture ZT_ARCHITECTURE_SPARC64 = 12, ZT_ARCHITECTURE_DOTNET_CLR = 13, ZT_ARCHITECTURE_JAVA_JVM = 14, - ZT_ARCHITECTURE_WEB = 15 + ZT_ARCHITECTURE_WEB = 15, + ZT_ARCHITECTURE_S390X = 16 }; /** @@ -1186,6 +1245,56 @@ typedef struct uint64_t trustedPathId; /** + * One-way latency + */ + float latency; + + /** + * How much latency varies over time + */ + float packetDelayVariance; + + /** + * How much observed throughput varies over time + */ + float throughputDisturbCoeff; + + /** + * Packet Error Ratio (PER) + */ + float packetErrorRatio; + + /** + * Packet Loss Ratio (PLR) + */ + float packetLossRatio; + + /** + * Stability of the path + */ + float stability; + + /** + * Current throughput (moving average) + */ + uint64_t throughput; + + /** + * Maximum observed throughput for this path + */ + uint64_t maxThroughput; + + /** + * Percentage of traffic allocated to this path + */ + float allocation; + + /** + * Name of physical interface (for monitoring) + */ + char *ifname; + + /** * Is path expired? */ int expired; @@ -1237,6 +1346,11 @@ typedef struct unsigned int pathCount; /** + * Whether this peer was ever reachable via an aggregate link + */ + bool hadAggregateLink; + + /** * Known network paths to peer */ ZT_PeerPhysicalPath paths[ZT_MAX_PEER_NETWORK_PATHS]; @@ -1462,7 +1576,7 @@ typedef int (*ZT_WirePacketSendFunction)( /** * Function to check whether a path should be used for ZeroTier traffic * - * Paramters: + * Parameters: * (1) Node * (2) User pointer * (3) ZeroTier address or 0 for none/any @@ -1495,7 +1609,7 @@ typedef int (*ZT_PathCheckFunction)( * (1) Node * (2) User pointer * (3) ZeroTier address (least significant 40 bits) - * (4) Desried address family or -1 for any + * (4) Desired address family or -1 for any * (5) Buffer to fill with result * * If provided this function will be occasionally called to get physical @@ -1660,7 +1774,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void * Join a network * * This may generate calls to the port config callback before it returns, - * or these may be deffered if a netconf is not available yet. + * or these may be differed if a netconf is not available yet. * * If we are already a member of the network, nothing is done and OK is * returned. diff --git a/java/jni/Android.mk b/java/jni/Android.mk index 8cda2474..f018950d 100644 --- a/java/jni/Android.mk +++ b/java/jni/Android.mk @@ -6,11 +6,9 @@ LOCAL_MODULE := ZeroTierOneJNI LOCAL_C_INCLUDES := \ $(ZT1)/include \ $(ZT1)/node \ - $(ZT1)/osdep \ - $(ZT1)/ext/miniupnpc \ - $(ZT1)/ext/libnatpmp + $(ZT1)/osdep -LOCAL_LDLIBS := -llog -latomic +LOCAL_LDLIBS := -llog # LOCAL_CFLAGS := -g LOCAL_CFLAGS := -DZT_USE_MINIUPNPC @@ -43,26 +41,7 @@ LOCAL_SRC_FILES := \ $(ZT1)/node/Topology.cpp \ $(ZT1)/node/Trace.cpp \ $(ZT1)/node/Utils.cpp \ - $(ZT1)/osdep/OSUtils.cpp \ - $(ZT1)/osdep/PortMapper.cpp - -# libminiupnpc and libnatpmp files -LOCAL_SRC_FILES += \ - $(ZT1)/ext/miniupnpc/connecthostport.c \ - $(ZT1)/ext/miniupnpc/igd_desc_parse.c \ - $(ZT1)/ext/miniupnpc/minisoap.c \ - $(ZT1)/ext/miniupnpc/minissdpc.c \ - $(ZT1)/ext/miniupnpc/miniupnpc.c \ - $(ZT1)/ext/miniupnpc/miniwget.c \ - $(ZT1)/ext/miniupnpc/minixml.c \ - $(ZT1)/ext/miniupnpc/portlistingparse.c \ - $(ZT1)/ext/miniupnpc/receivedata.c \ - $(ZT1)/ext/miniupnpc/upnpcommands.c \ - $(ZT1)/ext/miniupnpc/upnpdev.c \ - $(ZT1)/ext/miniupnpc/upnperrors.c \ - $(ZT1)/ext/miniupnpc/upnpreplyparse.c \ - $(ZT1)/ext/libnatpmp/natpmp.c \ - $(ZT1)/ext/libnatpmp/getgateway.c + $(ZT1)/osdep/OSUtils.cpp # JNI Files LOCAL_SRC_FILES += \ diff --git a/java/jni/Application.mk b/java/jni/Application.mk index f5a87ac7..4fc50f73 100644 --- a/java/jni/Application.mk +++ b/java/jni/Application.mk @@ -1,5 +1,5 @@ # NDK_TOOLCHAIN_VERSION := clang3.5 APP_STL := c++_static -APP_CPPFLAGS := -Wall -fstack-protector -fexceptions -fno-strict-aliasing -Wno-deprecated-register -DZT_NO_TYPE_PUNNING=1 +APP_CPPFLAGS := -Wall -fstack-protector -fexceptions -fno-strict-aliasing -frtti -Wno-deprecated-register -DZT_NO_TYPE_PUNNING=1 APP_PLATFORM := android-14 APP_ABI := all diff --git a/java/jni/com_zerotierone_sdk_Node.cpp b/java/jni/com_zerotierone_sdk_Node.cpp index 9a36e99b..6ddc3556 100644 --- a/java/jni/com_zerotierone_sdk_Node.cpp +++ b/java/jni/com_zerotierone_sdk_Node.cpp @@ -32,8 +32,6 @@ #include <ZeroTierOne.h> #include "Mutex.hpp" -#include "PortMapper.hpp" - #include <map> #include <string> #include <assert.h> @@ -60,7 +58,6 @@ namespace { , configListener(NULL) , pathChecker(NULL) , callbacks(NULL) - , portMapper(NULL) { callbacks = (ZT_Node_Callbacks*)malloc(sizeof(ZT_Node_Callbacks)); memset(callbacks, 0, sizeof(ZT_Node_Callbacks)); @@ -81,9 +78,6 @@ namespace { free(callbacks); callbacks = NULL; - - delete portMapper; - portMapper = NULL; } int64_t id; @@ -101,8 +95,6 @@ namespace { jobject pathChecker; ZT_Node_Callbacks *callbacks; - - ZeroTier::PortMapper *portMapper; }; @@ -880,13 +872,6 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_node_1init( return resultObject; } - uint64_t nodeId = ZT_Node_address(node); - if (nodeId != 0) { - char uniqueName[64]; - snprintf(uniqueName, sizeof(uniqueName), "ZeroTier Android/%.10llx@%u", (unsigned long long)nodeId, 9993); - ref->portMapper = new ZeroTier::PortMapper(9993, uniqueName); - } - ZeroTier::Mutex::Lock lock(nodeMapMutex); ref->node = node; nodeMap.insert(std::make_pair(ref->id, ref)); diff --git a/make-bsd.mk b/make-bsd.mk index 713cafc0..8c7a6ad2 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -123,11 +123,12 @@ endif ifeq ($(ZT_USE_ARM32_NEON_ASM_SALSA2012),1) override DEFS+=-DZT_USE_ARM32_NEON_ASM_SALSA2012 override CORE_OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o + override ASFLAGS+=-meabi=5 endif override DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" -CXXFLAGS+=$(CFLAGS) -fno-rtti -std=c++11 #-D_GLIBCXX_USE_C99 -D_GLIBCXX_USE_C99_MATH -D_GLIBCXX_USE_C99_MATH_TR1 +CXXFLAGS+=$(CFLAGS) -std=c++11 #-D_GLIBCXX_USE_C99 -D_GLIBCXX_USE_C99_MATH -D_GLIBCXX_USE_C99_MATH_TR1 all: one @@ -159,7 +160,7 @@ clean: rm -rf *.a *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli $(ONE_OBJS) $(CORE_OBJS) debug: FORCE - gmake -j 4 ZT_DEBUG=1 + $(MAKE) -j ZT_DEBUG=1 install: one rm -f /usr/local/sbin/zerotier-one diff --git a/make-linux.mk b/make-linux.mk index a9414d0c..d18126c9 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -17,6 +17,13 @@ DESTDIR?= include objects.mk ONE_OBJS+=osdep/LinuxEthernetTap.o +ONE_OBJS+=osdep/LinuxNetLink.o + +NLTEST_OBJS+=osdep/LinuxNetLink.o node/InetAddress.o node/Utils.o node/Salsa20.o +NLTEST_OBJS+=nltest.o + +# for central controller builds +TIMESTAMP=$(shell date +"%Y%m%d%H%M") # Auto-detect miniupnpc and nat-pmp as well and use system libs if present, # otherwise build into binary as done on Mac and Windows. @@ -42,8 +49,15 @@ endif # Trying to use dynamically linked libhttp-parser causes tons of compatibility problems. ONE_OBJS+=ext/http-parser/http_parser.o +# Build with address sanitization library for advanced debugging (clang) +ifeq ($(ZT_SANITIZE),1) + DEFS+=-fsanitize=address -DASAN_OPTIONS=symbolize=1 +endif +ifeq ($(ZT_DEBUG_TRACE),1) + DEFS+=-DZT_DEBUG_TRACE +endif ifeq ($(ZT_TRACE),1) - override DEFS+=-DZT_TRACE + DEFS+=-DZT_TRACE endif ifeq ($(ZT_RULES_ENGINE_DEBUGGING),1) @@ -82,6 +96,10 @@ ifeq ($(ZT_SYNOLOGY), 1) override DEFS+=-D__SYNOLOGY__ endif +ifeq ($(ZT_DISABLE_COMPRESSION), 1) + override DEFS+=-DZT_DISABLE_COMPRESSION +endif + ifeq ($(ZT_TRACE),1) override DEFS+=-DZT_TRACE endif @@ -90,6 +108,11 @@ ifeq ($(ZT_USE_TEST_TAP),1) override DEFS+=-DZT_USE_TEST_TAP endif +ifeq ($(ZT_VAULT_SUPPORT),1) + override DEFS+=-DZT_VAULT_SUPPORT=1 + override LDLIBS+=-lcurl +endif + # Uncomment for gprof profile build #CFLAGS=-Wall -g -pg -pthread $(INCLUDES) $(DEFS) #CXXFLAGS=-Wall -g -pg -pthread $(INCLUDES) $(DEFS) @@ -176,10 +199,10 @@ ifeq ($(CC_MACH),armv7l) override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_CRYPTO=1 endif -ifeq ($(CC_MACH),armv7l) - ZT_ARCHITECTURE=3 +ifeq ($(CC_MACH),armv7hl) + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING - ZT_USE_ARM32_NEON_ASM_SALSA2012=1 + ZT_USE_ARM32_NEON_ASM_CRYPTO=1 endif ifeq ($(CC_MACH),arm64) ZT_ARCHITECTURE=4 @@ -205,6 +228,9 @@ ifeq ($(CC_MACH),mips64el) ZT_ARCHITECTURE=6 override DEFS+=-DZT_NO_TYPE_PUNNING endif +ifeq ($(CC_MACH),s390x) + ZT_ARCHITECTURE=16 +endif # Fail if system architecture could not be determined ifeq ($(ZT_ARCHITECTURE),999) @@ -302,13 +328,18 @@ official: FORCE make -j4 ZT_OFFICIAL=1 all central-controller: FORCE - cd ext/librethinkdbxx ; make - make -j4 LDLIBS="ext/librethinkdbxx/build/librethinkdb++.a" DEFS="-DZT_CONTROLLER_USE_RETHINKDB" ZT_OFFICIAL=1 ZT_USE_X64_ASM_ED25519=1 one + make -j4 LDLIBS="-L/usr/pgsql-10/lib/ -lpq -Lext/librabbitmq/centos_x64/lib/ -lrabbitmq" CXXFLAGS="-I/usr/pgsql-10/include -I./ext/librabbitmq/centos_x64/include -fPIC" DEFS="-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER" ZT_OFFICIAL=1 ZT_USE_X64_ASM_ED25519=1 one + +central-controller-docker: central-controller + docker build -t docker.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP} -f docker/Dockerfile . debug: FORCE make ZT_DEBUG=1 one make ZT_DEBUG=1 selftest +nltest: $(NLTEST_OBJS) + $(CXX) $(CXXFLAGS) $(LDFLAGS) -o nltest $(NLTEST_OBJS) $(LDLIBS) + # Note: keep the symlinks in /var/lib/zerotier-one to the binaries since these # provide backward compatibility with old releases where the binaries actually # lived here. Folks got scripts. diff --git a/make-mac.mk b/make-mac.mk index 60aa465a..0735a47f 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -19,7 +19,12 @@ ZT_VERSION_BUILD=$(shell cat version.h | grep -F VERSION_BUILD | cut -d ' ' -f 3 DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE) include objects.mk -ONE_OBJS+=osdep/OSXEthernetTap.o ext/http-parser/http_parser.o +ONE_OBJS+=osdep/MacEthernetTap.o ext/http-parser/http_parser.o + +ifeq ($(ZT_CONTROLLER),1) + LIBS+=-lpq -lrabbitmq + DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER +endif # Official releases are signed with our Apple cert and apply software updates by default ifeq ($(ZT_OFFICIAL_RELEASE),1) @@ -43,19 +48,22 @@ ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connec # Build with address sanitization library for advanced debugging (clang) ifeq ($(ZT_SANITIZE),1) - SANFLAGS+=-fsanitize=address -DASAN_OPTIONS=symbolize=1 + DEFS+=-fsanitize=address -DASAN_OPTIONS=symbolize=1 +endif +ifeq ($(ZT_DEBUG_TRACE),1) + DEFS+=-DZT_DEBUG_TRACE endif # Debug mode -- dump trace output, build binary with -g ifeq ($(ZT_DEBUG),1) ZT_TRACE=1 - CFLAGS+=-Wall -Werror -g $(INCLUDES) $(DEFS) + CFLAGS+=-Wall -g $(INCLUDES) $(DEFS) STRIP=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g $(INCLUDES) $(DEFS) else CFLAGS?=-Ofast -fstack-protector-strong - CFLAGS+=$(ARCH_FLAGS) -Wall -Werror -flto -fPIE -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) + CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip endif @@ -63,6 +71,11 @@ ifeq ($(ZT_TRACE),1) DEFS+=-DZT_TRACE endif +ifeq ($(ZT_VAULT_SUPPORT),1) + DEFS+=-DZT_VAULT_SUPPORT=1 + LIBS+=-lcurl +endif + CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ all: one macui @@ -70,7 +83,11 @@ all: one macui ext/x64-salsa2012-asm/salsa2012.o: $(CC) $(CFLAGS) -c ext/x64-salsa2012-asm/salsa2012.s -o ext/x64-salsa2012-asm/salsa2012.o -one: $(CORE_OBJS) $(ONE_OBJS) one.o +mac-agent: FORCE + $(CC) -Ofast -o MacEthernetTapAgent osdep/MacEthernetTapAgent.c + $(CODESIGN) -f -s $(CODESIGN_APP_CERT) MacEthernetTapAgent + +one: $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent $(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) $(STRIP) zerotier-one ln -sf zerotier-one zerotier-idtool @@ -79,6 +96,9 @@ one: $(CORE_OBJS) $(ONE_OBJS) one.o zerotier-one: one +central-controller: + make ZT_CONTROLLER=1 one + zerotier-idtool: one zerotier-cli: one @@ -120,17 +140,10 @@ official: FORCE make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg clean: - rm -rf *.dSYM build-* *.a *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* + rm -rf MacEthernetTapAgent *.dSYM build-* *.a *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* distclean: clean realclean: clean -# For those building from source -- installs signed binary tap driver in system ZT home -install-mac-tap: FORCE - mkdir -p /Library/Application\ Support/ZeroTier/One - rm -rf /Library/Application\ Support/ZeroTier/One/tap.kext - cp -R ext/bin/tap-mac/tap.kext /Library/Application\ Support/ZeroTier/One - chown -R root:wheel /Library/Application\ Support/ZeroTier/One/tap.kext - FORCE: diff --git a/node/Address.hpp b/node/Address.hpp index 41977af2..ae0566fe 100644 --- a/node/Address.hpp +++ b/node/Address.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -54,28 +54,16 @@ public: * @param bits Raw address -- 5 bytes, big-endian byte order * @param len Length of array */ - Address(const void *bits,unsigned int len) - { - setTo(bits,len); - } + Address(const void *bits,unsigned int len) { setTo(bits,len); } - inline Address &operator=(const Address &a) - { - _a = a._a; - return *this; - } - - inline Address &operator=(const uint64_t a) - { - _a = (a & 0xffffffffffULL); - return *this; - } + inline Address &operator=(const Address &a) { _a = a._a; return *this; } + inline Address &operator=(const uint64_t a) { _a = (a & 0xffffffffffULL); return *this; } /** * @param bits Raw address -- 5 bytes, big-endian byte order * @param len Length of array */ - inline void setTo(const void *bits,unsigned int len) + inline void setTo(const void *bits,const unsigned int len) { if (len < ZT_ADDRESS_LENGTH) { _a = 0; @@ -94,7 +82,7 @@ public: * @param bits Buffer to hold 5-byte address in big-endian byte order * @param len Length of array */ - inline void copyTo(void *bits,unsigned int len) const + inline void copyTo(void *const bits,const unsigned int len) const { if (len < ZT_ADDRESS_LENGTH) return; @@ -125,26 +113,17 @@ public: /** * @return Integer containing address (0 to 2^40) */ - inline uint64_t toInt() const - { - return _a; - } + inline uint64_t toInt() const { return _a; } /** * @return Hash code for use with Hashtable */ - inline unsigned long hashCode() const - { - return (unsigned long)_a; - } + inline unsigned long hashCode() const { return (unsigned long)_a; } /** * @return Hexadecimal string */ - inline char *toString(char buf[11]) const - { - return Utils::hex10(_a,buf); - } + inline char *toString(char buf[11]) const { return Utils::hex10(_a,buf); } /** * @return True if this address is not zero @@ -152,11 +131,6 @@ public: inline operator bool() const { return (_a != 0); } /** - * Set to null/zero - */ - inline void zero() { _a = 0; } - - /** * Check if this address is reserved * * The all-zero null address and any address beginning with 0xff are @@ -165,16 +139,15 @@ public: * * @return True if address is reserved and may not be used */ - inline bool isReserved() const - { - return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX)); - } + inline bool isReserved() const { return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX)); } /** * @param i Value from 0 to 4 (inclusive) * @return Byte at said position (address interpreted in big-endian order) */ - inline unsigned char operator[](unsigned int i) const { return (unsigned char)((_a >> (32 - (i * 8))) & 0xff); } + inline uint8_t operator[](unsigned int i) const { return (uint8_t)(_a >> (32 - (i * 8))); } + + inline void zero() { _a = 0; } inline bool operator==(const uint64_t &a) const { return (_a == (a & 0xffffffffffULL)); } inline bool operator!=(const uint64_t &a) const { return (_a != (a & 0xffffffffffULL)); } diff --git a/node/AtomicCounter.hpp b/node/AtomicCounter.hpp index a42a18d4..b18ba540 100644 --- a/node/AtomicCounter.hpp +++ b/node/AtomicCounter.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Buffer.hpp b/node/Buffer.hpp index bbf4ee37..fe63eb2e 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -116,9 +116,9 @@ public: if (unlikely(b._l > C)) throw ZT_EXCEPTION_OUT_OF_BOUNDS; if (C2 == C) { - ZT_FAST_MEMCPY(this,&b,sizeof(Buffer<C>)); + memcpy(this,&b,sizeof(Buffer<C>)); } else { - ZT_FAST_MEMCPY(_b,b._b,_l = b._l); + memcpy(_b,b._b,_l = b._l); } return *this; } @@ -127,7 +127,7 @@ public: { if (unlikely(l > C)) throw ZT_EXCEPTION_OUT_OF_BOUNDS; - ZT_FAST_MEMCPY(_b,b,l); + memcpy(_b,b,l); _l = l; } @@ -281,7 +281,7 @@ public: { if (unlikely((_l + l) > C)) throw ZT_EXCEPTION_OUT_OF_BOUNDS; - ZT_FAST_MEMCPY(_b + _l,b,l); + memcpy(_b + _l,b,l); _l += l; } diff --git a/node/C25519.cpp b/node/C25519.cpp index 77084bd8..4384f8fd 100644 --- a/node/C25519.cpp +++ b/node/C25519.cpp @@ -15,6 +15,8 @@ Derived from public domain code by D. J. Bernstein. #include "C25519.hpp" #include "SHA512.hpp" #include "Buffer.hpp" +#include "Hashtable.hpp" +#include "Mutex.hpp" #ifdef __WINDOWS__ #pragma warning(disable: 4146) @@ -22,15 +24,722 @@ Derived from public domain code by D. J. Bernstein. namespace { -////////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - #define crypto_int32 int32_t #define crypto_uint32 uint32_t #define crypto_int64 int64_t #define crypto_uint64 uint64_t #define crypto_hash_sha512_BYTES 64 +////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// + +typedef uint8_t u8; +typedef int32_t s32; +typedef int64_t limb; + +static inline void fsum(limb *output, const limb *in) { + unsigned i; + for (i = 0; i < 10; i += 2) { + output[0+i] = output[0+i] + in[0+i]; + output[1+i] = output[1+i] + in[1+i]; + } +} + +static inline void fdifference(limb *output, const limb *in) { + unsigned i; + for (i = 0; i < 10; ++i) { + output[i] = in[i] - output[i]; + } +} + +static inline void fscalar_product(limb *output, const limb *in, const limb scalar) { + unsigned i; + for (i = 0; i < 10; ++i) { + output[i] = in[i] * scalar; + } +} + +static inline void fproduct(limb *output, const limb *in2, const limb *in) { + output[0] = ((limb) ((s32) in2[0])) * ((s32) in[0]); + output[1] = ((limb) ((s32) in2[0])) * ((s32) in[1]) + + ((limb) ((s32) in2[1])) * ((s32) in[0]); + output[2] = 2 * ((limb) ((s32) in2[1])) * ((s32) in[1]) + + ((limb) ((s32) in2[0])) * ((s32) in[2]) + + ((limb) ((s32) in2[2])) * ((s32) in[0]); + output[3] = ((limb) ((s32) in2[1])) * ((s32) in[2]) + + ((limb) ((s32) in2[2])) * ((s32) in[1]) + + ((limb) ((s32) in2[0])) * ((s32) in[3]) + + ((limb) ((s32) in2[3])) * ((s32) in[0]); + output[4] = ((limb) ((s32) in2[2])) * ((s32) in[2]) + + 2 * (((limb) ((s32) in2[1])) * ((s32) in[3]) + + ((limb) ((s32) in2[3])) * ((s32) in[1])) + + ((limb) ((s32) in2[0])) * ((s32) in[4]) + + ((limb) ((s32) in2[4])) * ((s32) in[0]); + output[5] = ((limb) ((s32) in2[2])) * ((s32) in[3]) + + ((limb) ((s32) in2[3])) * ((s32) in[2]) + + ((limb) ((s32) in2[1])) * ((s32) in[4]) + + ((limb) ((s32) in2[4])) * ((s32) in[1]) + + ((limb) ((s32) in2[0])) * ((s32) in[5]) + + ((limb) ((s32) in2[5])) * ((s32) in[0]); + output[6] = 2 * (((limb) ((s32) in2[3])) * ((s32) in[3]) + + ((limb) ((s32) in2[1])) * ((s32) in[5]) + + ((limb) ((s32) in2[5])) * ((s32) in[1])) + + ((limb) ((s32) in2[2])) * ((s32) in[4]) + + ((limb) ((s32) in2[4])) * ((s32) in[2]) + + ((limb) ((s32) in2[0])) * ((s32) in[6]) + + ((limb) ((s32) in2[6])) * ((s32) in[0]); + output[7] = ((limb) ((s32) in2[3])) * ((s32) in[4]) + + ((limb) ((s32) in2[4])) * ((s32) in[3]) + + ((limb) ((s32) in2[2])) * ((s32) in[5]) + + ((limb) ((s32) in2[5])) * ((s32) in[2]) + + ((limb) ((s32) in2[1])) * ((s32) in[6]) + + ((limb) ((s32) in2[6])) * ((s32) in[1]) + + ((limb) ((s32) in2[0])) * ((s32) in[7]) + + ((limb) ((s32) in2[7])) * ((s32) in[0]); + output[8] = ((limb) ((s32) in2[4])) * ((s32) in[4]) + + 2 * (((limb) ((s32) in2[3])) * ((s32) in[5]) + + ((limb) ((s32) in2[5])) * ((s32) in[3]) + + ((limb) ((s32) in2[1])) * ((s32) in[7]) + + ((limb) ((s32) in2[7])) * ((s32) in[1])) + + ((limb) ((s32) in2[2])) * ((s32) in[6]) + + ((limb) ((s32) in2[6])) * ((s32) in[2]) + + ((limb) ((s32) in2[0])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[0]); + output[9] = ((limb) ((s32) in2[4])) * ((s32) in[5]) + + ((limb) ((s32) in2[5])) * ((s32) in[4]) + + ((limb) ((s32) in2[3])) * ((s32) in[6]) + + ((limb) ((s32) in2[6])) * ((s32) in[3]) + + ((limb) ((s32) in2[2])) * ((s32) in[7]) + + ((limb) ((s32) in2[7])) * ((s32) in[2]) + + ((limb) ((s32) in2[1])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[1]) + + ((limb) ((s32) in2[0])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[0]); + output[10] = 2 * (((limb) ((s32) in2[5])) * ((s32) in[5]) + + ((limb) ((s32) in2[3])) * ((s32) in[7]) + + ((limb) ((s32) in2[7])) * ((s32) in[3]) + + ((limb) ((s32) in2[1])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[1])) + + ((limb) ((s32) in2[4])) * ((s32) in[6]) + + ((limb) ((s32) in2[6])) * ((s32) in[4]) + + ((limb) ((s32) in2[2])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[2]); + output[11] = ((limb) ((s32) in2[5])) * ((s32) in[6]) + + ((limb) ((s32) in2[6])) * ((s32) in[5]) + + ((limb) ((s32) in2[4])) * ((s32) in[7]) + + ((limb) ((s32) in2[7])) * ((s32) in[4]) + + ((limb) ((s32) in2[3])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[3]) + + ((limb) ((s32) in2[2])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[2]); + output[12] = ((limb) ((s32) in2[6])) * ((s32) in[6]) + + 2 * (((limb) ((s32) in2[5])) * ((s32) in[7]) + + ((limb) ((s32) in2[7])) * ((s32) in[5]) + + ((limb) ((s32) in2[3])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[3])) + + ((limb) ((s32) in2[4])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[4]); + output[13] = ((limb) ((s32) in2[6])) * ((s32) in[7]) + + ((limb) ((s32) in2[7])) * ((s32) in[6]) + + ((limb) ((s32) in2[5])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[5]) + + ((limb) ((s32) in2[4])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[4]); + output[14] = 2 * (((limb) ((s32) in2[7])) * ((s32) in[7]) + + ((limb) ((s32) in2[5])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[5])) + + ((limb) ((s32) in2[6])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[6]); + output[15] = ((limb) ((s32) in2[7])) * ((s32) in[8]) + + ((limb) ((s32) in2[8])) * ((s32) in[7]) + + ((limb) ((s32) in2[6])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[6]); + output[16] = ((limb) ((s32) in2[8])) * ((s32) in[8]) + + 2 * (((limb) ((s32) in2[7])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[7])); + output[17] = ((limb) ((s32) in2[8])) * ((s32) in[9]) + + ((limb) ((s32) in2[9])) * ((s32) in[8]); + output[18] = 2 * ((limb) ((s32) in2[9])) * ((s32) in[9]); +} + +static inline void freduce_degree(limb *output) { + output[8] += output[18] << 4; + output[8] += output[18] << 1; + output[8] += output[18]; + output[7] += output[17] << 4; + output[7] += output[17] << 1; + output[7] += output[17]; + output[6] += output[16] << 4; + output[6] += output[16] << 1; + output[6] += output[16]; + output[5] += output[15] << 4; + output[5] += output[15] << 1; + output[5] += output[15]; + output[4] += output[14] << 4; + output[4] += output[14] << 1; + output[4] += output[14]; + output[3] += output[13] << 4; + output[3] += output[13] << 1; + output[3] += output[13]; + output[2] += output[12] << 4; + output[2] += output[12] << 1; + output[2] += output[12]; + output[1] += output[11] << 4; + output[1] += output[11] << 1; + output[1] += output[11]; + output[0] += output[10] << 4; + output[0] += output[10] << 1; + output[0] += output[10]; +} + +#if (-1 & 3) != 3 +#error "This code only works on a two's complement system" +#endif + +static inline limb div_by_2_26(const limb v) +{ + /* High word of v; no shift needed. */ + const uint32_t highword = (uint32_t) (((uint64_t) v) >> 32); + /* Set to all 1s if v was negative; else set to 0s. */ + const int32_t sign = ((int32_t) highword) >> 31; + /* Set to 0x3ffffff if v was negative; else set to 0. */ + const int32_t roundoff = ((uint32_t) sign) >> 6; + /* Should return v / (1<<26) */ + return (v + roundoff) >> 26; +} + +static inline limb div_by_2_25(const limb v) +{ + /* High word of v; no shift needed*/ + const uint32_t highword = (uint32_t) (((uint64_t) v) >> 32); + /* Set to all 1s if v was negative; else set to 0s. */ + const int32_t sign = ((int32_t) highword) >> 31; + /* Set to 0x1ffffff if v was negative; else set to 0. */ + const int32_t roundoff = ((uint32_t) sign) >> 7; + /* Should return v / (1<<25) */ + return (v + roundoff) >> 25; +} + +static inline void freduce_coefficients(limb *output) { + unsigned i; + + output[10] = 0; + + for (i = 0; i < 10; i += 2) { + limb over = div_by_2_26(output[i]); + /* The entry condition (that |output[i]| < 280*2^54) means that over is, at + * most, 280*2^28 in the first iteration of this loop. This is added to the + * next limb and we can approximate the resulting bound of that limb by + * 281*2^54. */ + output[i] -= over << 26; + output[i+1] += over; + + /* For the first iteration, |output[i+1]| < 281*2^54, thus |over| < + * 281*2^29. When this is added to the next limb, the resulting bound can + * be approximated as 281*2^54. + * + * For subsequent iterations of the loop, 281*2^54 remains a conservative + * bound and no overflow occurs. */ + over = div_by_2_25(output[i+1]); + output[i+1] -= over << 25; + output[i+2] += over; + } + /* Now |output[10]| < 281*2^29 and all other coefficients are reduced. */ + output[0] += output[10] << 4; + output[0] += output[10] << 1; + output[0] += output[10]; + + output[10] = 0; + + /* Now output[1..9] are reduced, and |output[0]| < 2^26 + 19*281*2^29 + * So |over| will be no more than 2^16. */ + { + limb over = div_by_2_26(output[0]); + output[0] -= over << 26; + output[1] += over; + } + + /* Now output[0,2..9] are reduced, and |output[1]| < 2^25 + 2^16 < 2^26. The + * bound on |output[1]| is sufficient to meet our needs. */ +} + +static inline void fmul(limb *output, const limb *in, const limb *in2) { + limb t[19]; + fproduct(t, in, in2); + /* |t[i]| < 14*2^54 */ + freduce_degree(t); + freduce_coefficients(t); + /* |t[i]| < 2^26 */ + memcpy(output, t, sizeof(limb) * 10); +} + +static inline void fsquare_inner(limb *output, const limb *in) { + output[0] = ((limb) ((s32) in[0])) * ((s32) in[0]); + output[1] = 2 * ((limb) ((s32) in[0])) * ((s32) in[1]); + output[2] = 2 * (((limb) ((s32) in[1])) * ((s32) in[1]) + + ((limb) ((s32) in[0])) * ((s32) in[2])); + output[3] = 2 * (((limb) ((s32) in[1])) * ((s32) in[2]) + + ((limb) ((s32) in[0])) * ((s32) in[3])); + output[4] = ((limb) ((s32) in[2])) * ((s32) in[2]) + + 4 * ((limb) ((s32) in[1])) * ((s32) in[3]) + + 2 * ((limb) ((s32) in[0])) * ((s32) in[4]); + output[5] = 2 * (((limb) ((s32) in[2])) * ((s32) in[3]) + + ((limb) ((s32) in[1])) * ((s32) in[4]) + + ((limb) ((s32) in[0])) * ((s32) in[5])); + output[6] = 2 * (((limb) ((s32) in[3])) * ((s32) in[3]) + + ((limb) ((s32) in[2])) * ((s32) in[4]) + + ((limb) ((s32) in[0])) * ((s32) in[6]) + + 2 * ((limb) ((s32) in[1])) * ((s32) in[5])); + output[7] = 2 * (((limb) ((s32) in[3])) * ((s32) in[4]) + + ((limb) ((s32) in[2])) * ((s32) in[5]) + + ((limb) ((s32) in[1])) * ((s32) in[6]) + + ((limb) ((s32) in[0])) * ((s32) in[7])); + output[8] = ((limb) ((s32) in[4])) * ((s32) in[4]) + + 2 * (((limb) ((s32) in[2])) * ((s32) in[6]) + + ((limb) ((s32) in[0])) * ((s32) in[8]) + + 2 * (((limb) ((s32) in[1])) * ((s32) in[7]) + + ((limb) ((s32) in[3])) * ((s32) in[5]))); + output[9] = 2 * (((limb) ((s32) in[4])) * ((s32) in[5]) + + ((limb) ((s32) in[3])) * ((s32) in[6]) + + ((limb) ((s32) in[2])) * ((s32) in[7]) + + ((limb) ((s32) in[1])) * ((s32) in[8]) + + ((limb) ((s32) in[0])) * ((s32) in[9])); + output[10] = 2 * (((limb) ((s32) in[5])) * ((s32) in[5]) + + ((limb) ((s32) in[4])) * ((s32) in[6]) + + ((limb) ((s32) in[2])) * ((s32) in[8]) + + 2 * (((limb) ((s32) in[3])) * ((s32) in[7]) + + ((limb) ((s32) in[1])) * ((s32) in[9]))); + output[11] = 2 * (((limb) ((s32) in[5])) * ((s32) in[6]) + + ((limb) ((s32) in[4])) * ((s32) in[7]) + + ((limb) ((s32) in[3])) * ((s32) in[8]) + + ((limb) ((s32) in[2])) * ((s32) in[9])); + output[12] = ((limb) ((s32) in[6])) * ((s32) in[6]) + + 2 * (((limb) ((s32) in[4])) * ((s32) in[8]) + + 2 * (((limb) ((s32) in[5])) * ((s32) in[7]) + + ((limb) ((s32) in[3])) * ((s32) in[9]))); + output[13] = 2 * (((limb) ((s32) in[6])) * ((s32) in[7]) + + ((limb) ((s32) in[5])) * ((s32) in[8]) + + ((limb) ((s32) in[4])) * ((s32) in[9])); + output[14] = 2 * (((limb) ((s32) in[7])) * ((s32) in[7]) + + ((limb) ((s32) in[6])) * ((s32) in[8]) + + 2 * ((limb) ((s32) in[5])) * ((s32) in[9])); + output[15] = 2 * (((limb) ((s32) in[7])) * ((s32) in[8]) + + ((limb) ((s32) in[6])) * ((s32) in[9])); + output[16] = ((limb) ((s32) in[8])) * ((s32) in[8]) + + 4 * ((limb) ((s32) in[7])) * ((s32) in[9]); + output[17] = 2 * ((limb) ((s32) in[8])) * ((s32) in[9]); + output[18] = 2 * ((limb) ((s32) in[9])) * ((s32) in[9]); +} + +static void fsquare(limb *output, const limb *in) { + limb t[19]; + fsquare_inner(t, in); + /* |t[i]| < 14*2^54 because the largest product of two limbs will be < + * 2^(27+27) and fsquare_inner adds together, at most, 14 of those + * products. */ + freduce_degree(t); + freduce_coefficients(t); + /* |t[i]| < 2^26 */ + memcpy(output, t, sizeof(limb) * 10); +} + +static inline void fexpand(limb *output, const u8 *input) { +#define F(n,start,shift,mask) \ + output[n] = ((((limb) input[start + 0]) | \ + ((limb) input[start + 1]) << 8 | \ + ((limb) input[start + 2]) << 16 | \ + ((limb) input[start + 3]) << 24) >> shift) & mask; + F(0, 0, 0, 0x3ffffff); + F(1, 3, 2, 0x1ffffff); + F(2, 6, 3, 0x3ffffff); + F(3, 9, 5, 0x1ffffff); + F(4, 12, 6, 0x3ffffff); + F(5, 16, 0, 0x1ffffff); + F(6, 19, 1, 0x3ffffff); + F(7, 22, 3, 0x1ffffff); + F(8, 25, 4, 0x3ffffff); + F(9, 28, 6, 0x1ffffff); +#undef F +} + +#if (-32 >> 1) != -16 +#error "This code only works when >> does sign-extension on negative numbers" +#endif + +static inline s32 s32_eq(s32 a, s32 b) { + a = ~(a ^ b); + a &= a << 16; + a &= a << 8; + a &= a << 4; + a &= a << 2; + a &= a << 1; + return a >> 31; +} + +static inline s32 s32_gte(s32 a, s32 b) { + a -= b; + /* a >= 0 iff a >= b. */ + return ~(a >> 31); +} + +static inline void fcontract(u8 *output, limb *input_limbs) { + int i; + int j; + s32 input[10]; + s32 mask; + + /* |input_limbs[i]| < 2^26, so it's valid to convert to an s32. */ + for (i = 0; i < 10; i++) { + input[i] = input_limbs[i]; + } + + for (j = 0; j < 2; ++j) { + for (i = 0; i < 9; ++i) { + if ((i & 1) == 1) { + /* This calculation is a time-invariant way to make input[i] + * non-negative by borrowing from the next-larger limb. */ + const s32 mask = input[i] >> 31; + const s32 carry = -((input[i] & mask) >> 25); + input[i] = input[i] + (carry << 25); + input[i+1] = input[i+1] - carry; + } else { + const s32 mask = input[i] >> 31; + const s32 carry = -((input[i] & mask) >> 26); + input[i] = input[i] + (carry << 26); + input[i+1] = input[i+1] - carry; + } + } + + /* There's no greater limb for input[9] to borrow from, but we can multiply + * by 19 and borrow from input[0], which is valid mod 2^255-19. */ + { + const s32 mask = input[9] >> 31; + const s32 carry = -((input[9] & mask) >> 25); + input[9] = input[9] + (carry << 25); + input[0] = input[0] - (carry * 19); + } + + /* After the first iteration, input[1..9] are non-negative and fit within + * 25 or 26 bits, depending on position. However, input[0] may be + * negative. */ + } + + /* The first borrow-propagation pass above ended with every limb + except (possibly) input[0] non-negative. + + If input[0] was negative after the first pass, then it was because of a + carry from input[9]. On entry, input[9] < 2^26 so the carry was, at most, + one, since (2**26-1) >> 25 = 1. Thus input[0] >= -19. + + In the second pass, each limb is decreased by at most one. Thus the second + borrow-propagation pass could only have wrapped around to decrease + input[0] again if the first pass left input[0] negative *and* input[1] + through input[9] were all zero. In that case, input[1] is now 2^25 - 1, + and this last borrow-propagation step will leave input[1] non-negative. */ + { + const s32 mask = input[0] >> 31; + const s32 carry = -((input[0] & mask) >> 26); + input[0] = input[0] + (carry << 26); + input[1] = input[1] - carry; + } + + /* All input[i] are now non-negative. However, there might be values between + * 2^25 and 2^26 in a limb which is, nominally, 25 bits wide. */ + for (j = 0; j < 2; j++) { + for (i = 0; i < 9; i++) { + if ((i & 1) == 1) { + const s32 carry = input[i] >> 25; + input[i] &= 0x1ffffff; + input[i+1] += carry; + } else { + const s32 carry = input[i] >> 26; + input[i] &= 0x3ffffff; + input[i+1] += carry; + } + } + + { + const s32 carry = input[9] >> 25; + input[9] &= 0x1ffffff; + input[0] += 19*carry; + } + } + + /* If the first carry-chain pass, just above, ended up with a carry from + * input[9], and that caused input[0] to be out-of-bounds, then input[0] was + * < 2^26 + 2*19, because the carry was, at most, two. + * + * If the second pass carried from input[9] again then input[0] is < 2*19 and + * the input[9] -> input[0] carry didn't push input[0] out of bounds. */ + + /* It still remains the case that input might be between 2^255-19 and 2^255. + * In this case, input[1..9] must take their maximum value and input[0] must + * be >= (2^255-19) & 0x3ffffff, which is 0x3ffffed. */ + mask = s32_gte(input[0], 0x3ffffed); + for (i = 1; i < 10; i++) { + if ((i & 1) == 1) { + mask &= s32_eq(input[i], 0x1ffffff); + } else { + mask &= s32_eq(input[i], 0x3ffffff); + } + } + + /* mask is either 0xffffffff (if input >= 2^255-19) and zero otherwise. Thus + * this conditionally subtracts 2^255-19. */ + input[0] -= mask & 0x3ffffed; + + for (i = 1; i < 10; i++) { + if ((i & 1) == 1) { + input[i] -= mask & 0x1ffffff; + } else { + input[i] -= mask & 0x3ffffff; + } + } + + input[1] <<= 2; + input[2] <<= 3; + input[3] <<= 5; + input[4] <<= 6; + input[6] <<= 1; + input[7] <<= 3; + input[8] <<= 4; + input[9] <<= 6; +#define F(i, s) \ + output[s+0] |= input[i] & 0xff; \ + output[s+1] = (input[i] >> 8) & 0xff; \ + output[s+2] = (input[i] >> 16) & 0xff; \ + output[s+3] = (input[i] >> 24) & 0xff; + output[0] = 0; + output[16] = 0; + F(0,0); + F(1,3); + F(2,6); + F(3,9); + F(4,12); + F(5,16); + F(6,19); + F(7,22); + F(8,25); + F(9,28); +#undef F +} + +static inline void fmonty(limb *x2, limb *z2, /* output 2Q */ + limb *x3, limb *z3, /* output Q + Q' */ + limb *x, limb *z, /* input Q */ + limb *xprime, limb *zprime, /* input Q' */ + const limb *qmqp /* input Q - Q' */) { + limb origx[10], origxprime[10], zzz[19], xx[19], zz[19], xxprime[19], + zzprime[19], zzzprime[19], xxxprime[19]; + + memcpy(origx, x, 10 * sizeof(limb)); + fsum(x, z); + /* |x[i]| < 2^27 */ + fdifference(z, origx); /* does x - z */ + /* |z[i]| < 2^27 */ + + memcpy(origxprime, xprime, sizeof(limb) * 10); + fsum(xprime, zprime); + /* |xprime[i]| < 2^27 */ + fdifference(zprime, origxprime); + /* |zprime[i]| < 2^27 */ + fproduct(xxprime, xprime, z); + /* |xxprime[i]| < 14*2^54: the largest product of two limbs will be < + * 2^(27+27) and fproduct adds together, at most, 14 of those products. + * (Approximating that to 2^58 doesn't work out.) */ + fproduct(zzprime, x, zprime); + /* |zzprime[i]| < 14*2^54 */ + freduce_degree(xxprime); + freduce_coefficients(xxprime); + /* |xxprime[i]| < 2^26 */ + freduce_degree(zzprime); + freduce_coefficients(zzprime); + /* |zzprime[i]| < 2^26 */ + memcpy(origxprime, xxprime, sizeof(limb) * 10); + fsum(xxprime, zzprime); + /* |xxprime[i]| < 2^27 */ + fdifference(zzprime, origxprime); + /* |zzprime[i]| < 2^27 */ + fsquare(xxxprime, xxprime); + /* |xxxprime[i]| < 2^26 */ + fsquare(zzzprime, zzprime); + /* |zzzprime[i]| < 2^26 */ + fproduct(zzprime, zzzprime, qmqp); + /* |zzprime[i]| < 14*2^52 */ + freduce_degree(zzprime); + freduce_coefficients(zzprime); + /* |zzprime[i]| < 2^26 */ + memcpy(x3, xxxprime, sizeof(limb) * 10); + memcpy(z3, zzprime, sizeof(limb) * 10); + + fsquare(xx, x); + /* |xx[i]| < 2^26 */ + fsquare(zz, z); + /* |zz[i]| < 2^26 */ + fproduct(x2, xx, zz); + /* |x2[i]| < 14*2^52 */ + freduce_degree(x2); + freduce_coefficients(x2); + /* |x2[i]| < 2^26 */ + fdifference(zz, xx); // does zz = xx - zz + /* |zz[i]| < 2^27 */ + memset(zzz + 10, 0, sizeof(limb) * 9); + fscalar_product(zzz, zz, 121665); + /* |zzz[i]| < 2^(27+17) */ + /* No need to call freduce_degree here: + fscalar_product doesn't increase the degree of its input. */ + freduce_coefficients(zzz); + /* |zzz[i]| < 2^26 */ + fsum(zzz, xx); + /* |zzz[i]| < 2^27 */ + fproduct(z2, zz, zzz); + /* |z2[i]| < 14*2^(26+27) */ + freduce_degree(z2); + freduce_coefficients(z2); + /* |z2|i| < 2^26 */ +} + +static inline void swap_conditional(limb a[19], limb b[19], limb iswap) { + unsigned i; + const s32 swap = (s32) -iswap; + + for (i = 0; i < 10; ++i) { + const s32 x = swap & ( ((s32)a[i]) ^ ((s32)b[i]) ); + a[i] = ((s32)a[i]) ^ x; + b[i] = ((s32)b[i]) ^ x; + } +} + +static inline void cmult(limb *resultx, limb *resultz, const u8 *n, const limb *q) { + limb a[19] = {0}, b[19] = {1}, c[19] = {1}, d[19] = {0}; + limb *nqpqx = a, *nqpqz = b, *nqx = c, *nqz = d, *t; + limb e[19] = {0}, f[19] = {1}, g[19] = {0}, h[19] = {1}; + limb *nqpqx2 = e, *nqpqz2 = f, *nqx2 = g, *nqz2 = h; + + unsigned i, j; + + memcpy(nqpqx, q, sizeof(limb) * 10); + + for (i = 0; i < 32; ++i) { + u8 byte = n[31 - i]; + for (j = 0; j < 8; ++j) { + const limb bit = byte >> 7; + + swap_conditional(nqx, nqpqx, bit); + swap_conditional(nqz, nqpqz, bit); + fmonty(nqx2, nqz2, + nqpqx2, nqpqz2, + nqx, nqz, + nqpqx, nqpqz, + q); + swap_conditional(nqx2, nqpqx2, bit); + swap_conditional(nqz2, nqpqz2, bit); + + t = nqx; + nqx = nqx2; + nqx2 = t; + t = nqz; + nqz = nqz2; + nqz2 = t; + t = nqpqx; + nqpqx = nqpqx2; + nqpqx2 = t; + t = nqpqz; + nqpqz = nqpqz2; + nqpqz2 = t; + + byte <<= 1; + } + } + + memcpy(resultx, nqx, sizeof(limb) * 10); + memcpy(resultz, nqz, sizeof(limb) * 10); +} + +static inline void crecip(limb *out, const limb *z) { + limb z2[10]; + limb z9[10]; + limb z11[10]; + limb z2_5_0[10]; + limb z2_10_0[10]; + limb z2_20_0[10]; + limb z2_50_0[10]; + limb z2_100_0[10]; + limb t0[10]; + limb t1[10]; + int i; + + /* 2 */ fsquare(z2,z); + /* 4 */ fsquare(t1,z2); + /* 8 */ fsquare(t0,t1); + /* 9 */ fmul(z9,t0,z); + /* 11 */ fmul(z11,z9,z2); + /* 22 */ fsquare(t0,z11); + /* 2^5 - 2^0 = 31 */ fmul(z2_5_0,t0,z9); + + /* 2^6 - 2^1 */ fsquare(t0,z2_5_0); + /* 2^7 - 2^2 */ fsquare(t1,t0); + /* 2^8 - 2^3 */ fsquare(t0,t1); + /* 2^9 - 2^4 */ fsquare(t1,t0); + /* 2^10 - 2^5 */ fsquare(t0,t1); + /* 2^10 - 2^0 */ fmul(z2_10_0,t0,z2_5_0); + + /* 2^11 - 2^1 */ fsquare(t0,z2_10_0); + /* 2^12 - 2^2 */ fsquare(t1,t0); + /* 2^20 - 2^10 */ for (i = 2;i < 10;i += 2) { fsquare(t0,t1); fsquare(t1,t0); } + /* 2^20 - 2^0 */ fmul(z2_20_0,t1,z2_10_0); + + /* 2^21 - 2^1 */ fsquare(t0,z2_20_0); + /* 2^22 - 2^2 */ fsquare(t1,t0); + /* 2^40 - 2^20 */ for (i = 2;i < 20;i += 2) { fsquare(t0,t1); fsquare(t1,t0); } + /* 2^40 - 2^0 */ fmul(t0,t1,z2_20_0); + + /* 2^41 - 2^1 */ fsquare(t1,t0); + /* 2^42 - 2^2 */ fsquare(t0,t1); + /* 2^50 - 2^10 */ for (i = 2;i < 10;i += 2) { fsquare(t1,t0); fsquare(t0,t1); } + /* 2^50 - 2^0 */ fmul(z2_50_0,t0,z2_10_0); + + /* 2^51 - 2^1 */ fsquare(t0,z2_50_0); + /* 2^52 - 2^2 */ fsquare(t1,t0); + /* 2^100 - 2^50 */ for (i = 2;i < 50;i += 2) { fsquare(t0,t1); fsquare(t1,t0); } + /* 2^100 - 2^0 */ fmul(z2_100_0,t1,z2_50_0); + + /* 2^101 - 2^1 */ fsquare(t1,z2_100_0); + /* 2^102 - 2^2 */ fsquare(t0,t1); + /* 2^200 - 2^100 */ for (i = 2;i < 100;i += 2) { fsquare(t1,t0); fsquare(t0,t1); } + /* 2^200 - 2^0 */ fmul(t1,t0,z2_100_0); + + /* 2^201 - 2^1 */ fsquare(t0,t1); + /* 2^202 - 2^2 */ fsquare(t1,t0); + /* 2^250 - 2^50 */ for (i = 2;i < 50;i += 2) { fsquare(t0,t1); fsquare(t1,t0); } + /* 2^250 - 2^0 */ fmul(t0,t1,z2_50_0); + + /* 2^251 - 2^1 */ fsquare(t1,t0); + /* 2^252 - 2^2 */ fsquare(t0,t1); + /* 2^253 - 2^3 */ fsquare(t1,t0); + /* 2^254 - 2^4 */ fsquare(t0,t1); + /* 2^255 - 2^5 */ fsquare(t1,t0); + /* 2^255 - 21 */ fmul(out,t1,z11); +} + +static void crypto_scalarmult(u8 *mypublic, const u8 *secret, const u8 *basepoint) { + limb bp[10], x[10], z[11], zmone[10]; + uint8_t e[32]; + int i; + + for (i = 0; i < 32; ++i) e[i] = secret[i]; + e[0] &= 248; + e[31] &= 127; + e[31] |= 64; + + fexpand(bp, basepoint); + cmult(x, z, e, bp); + crecip(zmone, z); + fmul(z, x, zmone); + fcontract(mypublic, z); +} + +#if 0 void add(unsigned int out[32],const unsigned int a[32],const unsigned int b[32]) { unsigned int j; @@ -285,11 +994,12 @@ int crypto_scalarmult(unsigned char *q,const unsigned char *n,const unsigned cha for (i = 0;i < 32;++i) q[i] = work[64 + i]; return 0; } +#endif static const unsigned char base[32] = {9}; -int crypto_scalarmult_base(unsigned char *q,const unsigned char *n) +static inline void crypto_scalarmult_base(unsigned char *q,const unsigned char *n) { - return crypto_scalarmult(q,n,base); + crypto_scalarmult(q,n,base); } ////////////////////////////////////////////////////////////////////////////// @@ -792,7 +1502,6 @@ void sc25519_mul(sc25519 *r, const sc25519 *x, const sc25519 *y) for(j=0;j<32;j++) t[i+j] += x->v[i] * y->v[j]; - /* Reduce coefficients */ for(i=0;i<63;i++) { carry = t[i] >> 8; @@ -1996,7 +2705,7 @@ void get_hram(unsigned char *hram, const unsigned char *sm, const unsigned char } // anonymous namespace #ifdef ZT_USE_FAST_X64_ED25519 -extern "C" void ed25519_amd64_asm_sign(const unsigned char *sk,const unsigned char *pk,const unsigned char *m,const unsigned int mlen,unsigned char *sig); +extern "C" void ed25519_amd64_asm_sign(const unsigned char *sk,const unsigned char *pk,const unsigned char *digest,unsigned char *sig); #endif namespace ZeroTier { @@ -2019,8 +2728,11 @@ void C25519::agree(const C25519::Private &mine,const C25519::Public &their,void void C25519::sign(const C25519::Private &myPrivate,const C25519::Public &myPublic,const void *msg,unsigned int len,void *signature) { + unsigned char digest[64]; // we sign the first 32 bytes of SHA-512(msg) + SHA512::hash(digest,msg,len); + #ifdef ZT_USE_FAST_X64_ED25519 - ed25519_amd64_asm_sign(myPrivate.data + 32,myPublic.data + 32,(const unsigned char *)msg,len,(unsigned char *)signature); + ed25519_amd64_asm_sign(myPrivate.data + 32,myPublic.data + 32,digest,(unsigned char *)signature); #else sc25519 sck, scs, scsk; ge25519 ger; @@ -2030,9 +2742,6 @@ void C25519::sign(const C25519::Private &myPrivate,const C25519::Public &myPubli unsigned char hmg[crypto_hash_sha512_BYTES]; unsigned char hram[crypto_hash_sha512_BYTES]; unsigned char *sig = (unsigned char *)signature; - unsigned char digest[64]; // we sign the first 32 bytes of SHA-512(msg) - - SHA512::hash(digest,msg,len); SHA512::hash(extsk,myPrivate.data + 32,32); extsk[0] &= 248; @@ -2071,18 +2780,17 @@ void C25519::sign(const C25519::Private &myPrivate,const C25519::Public &myPubli bool C25519::verify(const C25519::Public &their,const void *msg,unsigned int len,const void *signature) { + const unsigned char *const sig = (const unsigned char *)signature; + unsigned char digest[64]; // we sign the first 32 bytes of SHA-512(msg) + SHA512::hash(digest,msg,len); + if (!Utils::secureEq(sig + 64,digest,32)) + return false; + unsigned char t2[32]; ge25519 get1, get2; sc25519 schram, scs; unsigned char hram[crypto_hash_sha512_BYTES]; unsigned char m[96]; - unsigned char digest[64]; // we sign the first 32 bytes of SHA-512(msg) - const unsigned char *sig = (const unsigned char *)signature; - - // First check the message's integrity - SHA512::hash(digest,msg,len); - if (!Utils::secureEq(sig + 64,digest,32)) - return false; if (ge25519_unpackneg_vartime(&get1,their.data + 32)) return false; diff --git a/node/C25519.hpp b/node/C25519.hpp index 2e9184a0..640aedf5 100644 --- a/node/C25519.hpp +++ b/node/C25519.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Capability.cpp b/node/Capability.cpp index fb52be8a..fdf98bae 100644 --- a/node/Capability.cpp +++ b/node/Capability.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Capability.hpp b/node/Capability.hpp index 91a46566..0704c513 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -52,7 +52,7 @@ class RuntimeEnvironment; * (1) Evaluates its capabilities in ascending order of ID to determine * which capability allows it to transmit this packet. * (2) If it has not done so lately, it then sends this capability to the - * receving peer ("presents" it). + * receiving peer ("presents" it). * (3) The sender then sends the packet. * * On the receiving side the receiver evaluates the capabilities presented @@ -64,7 +64,7 @@ class RuntimeEnvironment; * * Capabilities support a chain of custody. This is currently unused but * in the future would allow the publication of capabilities that can be - * handed off between nodes. Limited transferrability of capabilities is + * handed off between nodes. Limited transferability of capabilities is * a feature of true capability based security. */ class Capability : public Credential @@ -72,29 +72,34 @@ class Capability : public Credential public: static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_CAPABILITY; } - Capability() + Capability() : + _nwid(0), + _ts(0), + _id(0), + _maxCustodyChainLength(0), + _ruleCount(0) { - memset(this,0,sizeof(Capability)); + memset(_rules,0,sizeof(_rules)); + memset(_custody,0,sizeof(_custody)); } /** * @param id Capability ID * @param nwid Network ID * @param ts Timestamp (at controller) - * @param mccl Maximum custody chain length (1 to create non-transferrable capability) + * @param mccl Maximum custody chain length (1 to create non-transferable capability) * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ - Capability(uint32_t id,uint64_t nwid,int64_t ts,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + Capability(uint32_t id,uint64_t nwid,int64_t ts,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) : + _nwid(nwid), + _ts(ts), + _id(id), + _maxCustodyChainLength((mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1), + _ruleCount((ruleCount < ZT_MAX_CAPABILITY_RULES) ? ruleCount : ZT_MAX_CAPABILITY_RULES) { - memset(this,0,sizeof(Capability)); - _nwid = nwid; - _ts = ts; - _id = id; - _maxCustodyChainLength = (mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1; - _ruleCount = (ruleCount < ZT_MAX_CAPABILITY_RULES) ? ruleCount : ZT_MAX_CAPABILITY_RULES; - if (_ruleCount) - ZT_FAST_MEMCPY(_rules,rules,sizeof(ZT_VirtualNetworkRule) * _ruleCount); + if (_ruleCount > 0) + memcpy(_rules,rules,sizeof(ZT_VirtualNetworkRule) * _ruleCount); } /** @@ -320,16 +325,16 @@ public: break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: case ZT_NETWORK_RULE_MATCH_MAC_DEST: - ZT_FAST_MEMCPY(rules[ruleCount].v.mac,b.field(p,6),6); + memcpy(rules[ruleCount].v.mac,b.field(p,6),6); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - ZT_FAST_MEMCPY(&(rules[ruleCount].v.ipv4.ip),b.field(p,4),4); + memcpy(&(rules[ruleCount].v.ipv4.ip),b.field(p,4),4); rules[ruleCount].v.ipv4.mask = (uint8_t)b[p + 4]; break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - ZT_FAST_MEMCPY(rules[ruleCount].v.ipv6.ip,b.field(p,16),16); + memcpy(rules[ruleCount].v.ipv6.ip,b.field(p,16),16); rules[ruleCount].v.ipv6.mask = (uint8_t)b[p + 16]; break; case ZT_NETWORK_RULE_MATCH_IP_TOS: @@ -423,7 +428,7 @@ public: template<unsigned int C> inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0) { - memset(this,0,sizeof(Capability)); + *this = Capability(); unsigned int p = startAt; @@ -452,7 +457,7 @@ public: if (b.template at<uint16_t>(p) != ZT_C25519_SIGNATURE_LEN) throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; p += 2; - ZT_FAST_MEMCPY(_custody[i].signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + memcpy(_custody[i].signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else { p += 2 + b.template at<uint16_t>(p); } diff --git a/node/CertificateOfMembership.cpp b/node/CertificateOfMembership.cpp index 8184ce99..614c6887 100644 --- a/node/CertificateOfMembership.cpp +++ b/node/CertificateOfMembership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index b5a90007..b2c63f9d 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -112,15 +112,8 @@ public: /** * Create an empty certificate of membership */ - CertificateOfMembership() - { - memset(this,0,sizeof(CertificateOfMembership)); - } - - CertificateOfMembership(const CertificateOfMembership &c) - { - ZT_FAST_MEMCPY(this,&c,sizeof(CertificateOfMembership)); - } + CertificateOfMembership() : + _qualifierCount(0) {} /** * Create from required fields common to all networks @@ -145,12 +138,6 @@ public: memset(_signature.data,0,ZT_C25519_SIGNATURE_LEN); } - inline CertificateOfMembership &operator=(const CertificateOfMembership &c) - { - ZT_FAST_MEMCPY(this,&c,sizeof(CertificateOfMembership)); - return *this; - } - /** * Create from binary-serialized COM in buffer * @@ -243,7 +230,7 @@ public: * Compare two certificates for parameter agreement * * This compares this certificate with the other and returns true if all - * paramters in this cert are present in the other and if they agree to + * parameters in this cert are present in the other and if they agree to * within this cert's max delta value for each given parameter. * * Tuples present in other but not in this cert are ignored, but any @@ -329,7 +316,7 @@ public: p += ZT_ADDRESS_LENGTH; if (_signedBy) { - ZT_FAST_MEMCPY(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp index 8ee67865..d7266cd2 100644 --- a/node/CertificateOfOwnership.cpp +++ b/node/CertificateOfOwnership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 278ae863..fdffec3b 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -69,19 +69,16 @@ public: CertificateOfOwnership() { - memset(this,0,sizeof(CertificateOfOwnership)); + memset(reinterpret_cast<void *>(this),0,sizeof(CertificateOfOwnership)); } - CertificateOfOwnership(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id) : - _networkId(nwid), - _ts(ts), - _flags(0), - _id(id), - _thingCount(0), - _issuedTo(issuedTo) + CertificateOfOwnership(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id) { - memset(_thingTypes,0,sizeof(_thingTypes)); - memset(_thingValues,0,sizeof(_thingValues)); + memset(reinterpret_cast<void *>(this),0,sizeof(CertificateOfOwnership)); + _networkId = nwid; + _ts = ts; + _id = id; + _issuedTo = issuedTo; } inline uint64_t networkId() const { return _networkId; } @@ -115,11 +112,11 @@ public: if (_thingCount >= ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS) return; if (ip.ss_family == AF_INET) { _thingTypes[_thingCount] = THING_IPV4_ADDRESS; - ZT_FAST_MEMCPY(_thingValues[_thingCount],&(reinterpret_cast<const struct sockaddr_in *>(&ip)->sin_addr.s_addr),4); + memcpy(_thingValues[_thingCount],&(reinterpret_cast<const struct sockaddr_in *>(&ip)->sin_addr.s_addr),4); ++_thingCount; } else if (ip.ss_family == AF_INET6) { _thingTypes[_thingCount] = THING_IPV6_ADDRESS; - ZT_FAST_MEMCPY(_thingValues[_thingCount],reinterpret_cast<const struct sockaddr_in6 *>(&ip)->sin6_addr.s6_addr,16); + memcpy(_thingValues[_thingCount],reinterpret_cast<const struct sockaddr_in6 *>(&ip)->sin6_addr.s6_addr,16); ++_thingCount; } } @@ -188,7 +185,7 @@ public: { unsigned int p = startAt; - memset(this,0,sizeof(CertificateOfOwnership)); + *this = CertificateOfOwnership(); _networkId = b.template at<uint64_t>(p); p += 8; _ts = b.template at<uint64_t>(p); p += 8; @@ -198,7 +195,7 @@ public: for(unsigned int i=0,j=_thingCount;i<j;++i) { if (i < ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS) { _thingTypes[i] = (uint8_t)b[p++]; - ZT_FAST_MEMCPY(_thingValues[i],b.field(p,ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE),ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE); + memcpy(_thingValues[i],b.field(p,ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE),ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE); p += ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE; } } @@ -209,7 +206,7 @@ public: if (b.template at<uint16_t>(p) != ZT_C25519_SIGNATURE_LEN) throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; p += 2; - ZT_FAST_MEMCPY(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else { p += 2 + b.template at<uint16_t>(p); } diff --git a/node/Constants.hpp b/node/Constants.hpp index 03b04e68..d58e408f 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -72,14 +72,6 @@ #include <machine/endian.h> #endif -// Defined this macro to disable "type punning" on a number of targets that -// have issues with unaligned memory access. -#if defined(__arm__) || defined(__ARMEL__) || (defined(__APPLE__) && ( (defined(TARGET_OS_IPHONE) && (TARGET_OS_IPHONE != 0)) || (defined(TARGET_OS_WATCH) && (TARGET_OS_WATCH != 0)) || (defined(TARGET_IPHONE_SIMULATOR) && (TARGET_IPHONE_SIMULATOR != 0)) ) ) -#ifndef ZT_NO_TYPE_PUNNING -#define ZT_NO_TYPE_PUNNING -#endif -#endif - #if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) #ifndef __UNIX_LIKE__ #define __UNIX_LIKE__ @@ -107,13 +99,23 @@ #pragma warning(disable : 4101) #undef __UNIX_LIKE__ #undef __BSD__ -#define ZT_PATH_SEPARATOR '\\' -#define ZT_PATH_SEPARATOR_S "\\" -#define ZT_EOL_S "\r\n" #include <WinSock2.h> #include <Windows.h> #endif +#ifdef __NetBSD__ +#ifndef RTF_MULTICAST +#define RTF_MULTICAST 0x20000000 +#endif +#endif + +// Define ZT_NO_TYPE_PUNNING to disable reckless casts on anything other than x86/x64. +#if (!(defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_AMD64) || defined(_M_X64) || defined(i386) || defined(__i386) || defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_M_IX86) || defined(__X86__) || defined(_X86_) || defined(__I86__) || defined(__INTEL__) || defined(__386))) +#ifndef ZT_NO_TYPE_PUNNING +#define ZT_NO_TYPE_PUNNING +#endif +#endif + // Assume little endian if not defined #if (defined(__APPLE__) || defined(__WINDOWS__)) && (!defined(__BYTE_ORDER)) #undef __BYTE_ORDER @@ -124,7 +126,11 @@ #define __BYTE_ORDER 1234 #endif -#ifdef __UNIX_LIKE__ +#ifdef __WINDOWS__ +#define ZT_PATH_SEPARATOR '\\' +#define ZT_PATH_SEPARATOR_S "\\" +#define ZT_EOL_S "\r\n" +#else #define ZT_PATH_SEPARATOR '/' #define ZT_PATH_SEPARATOR_S "/" #define ZT_EOL_S "\n" @@ -134,10 +140,6 @@ #include <endian.h> #endif -#ifdef __NetBSD__ -#define RTF_MULTICAST 0x20000000 -#endif - #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) #ifndef likely #define likely(x) __builtin_expect((x),1) @@ -187,19 +189,13 @@ /** * Size of RX queue - * - * This is about 2mb, and can be decreased for small devices. A queue smaller - * than about 4 is probably going to cause a lot of lost packets. */ -#define ZT_RX_QUEUE_SIZE 64 +#define ZT_RX_QUEUE_SIZE 32 /** * Size of TX queue - * - * This is about 2mb, and can be decreased for small devices. A queue smaller - * than about 4 is probably going to cause a lot of lost packets. */ -#define ZT_TX_QUEUE_SIZE 64 +#define ZT_TX_QUEUE_SIZE 32 /** * Length of secret key in bytes -- 256-bit -- do not change @@ -232,11 +228,6 @@ #define ZT_RECEIVE_QUEUE_TIMEOUT 5000 /** - * Maximum latency to allow for OK(HELLO) before packet is discarded - */ -#define ZT_HELLO_MAX_ALLOWABLE_LATENCY 120000 - -/** * Maximum number of ZT hops allowed (this is not IP hops/TTL) * * The protocol allows up to 7, but we limit it to something smaller. @@ -259,11 +250,6 @@ #define ZT_MULTICAST_EXPLICIT_GATHER_DELAY (ZT_MULTICAST_LIKE_EXPIRE / 10) /** - * Expiration for credentials presented for MULTICAST_LIKE or MULTICAST_GATHER (for non-network-members) - */ -#define ZT_MULTICAST_CREDENTIAL_EXPIRATON ZT_MULTICAST_LIKE_EXPIRE - -/** * Timeout for outgoing multicasts * * This is how long we wait for explicit or implicit gather results. @@ -276,6 +262,189 @@ #define ZT_PING_CHECK_INVERVAL 5000 /** + * How often the local.conf file is checked for changes (service, should be moved there) + */ +#define ZT_LOCAL_CONF_FILE_CHECK_INTERVAL 10000 + +/** + * How frequently to check for changes to the system's network interfaces. When + * the service decides to use this constant it's because we want to react more + * quickly to new interfaces that pop up or go down. + */ +#define ZT_MULTIPATH_BINDER_REFRESH_PERIOD 5000 + +/** + * Packets are only used for QoS/ACK statistical sampling if their packet ID is divisible by + * this integer. This is to provide a mechanism for both peers to agree on which packets need + * special treatment without having to exchange information. Changing this value would be + * a breaking change and would necessitate a protocol version upgrade. Since each incoming and + * outgoing packet ID is checked against this value its evaluation is of the form: + * (id & (divisor - 1)) == 0, thus the divisor must be a power of 2. + * + * This value is set at (16) so that given a normally-distributed RNG output we will sample + * 1/16th (or ~6.25%) of packets. + */ +#define ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR 0x10 + +/** + * Time horizon for VERB_QOS_MEASUREMENT and VERB_ACK packet processing cutoff + */ +#define ZT_PATH_QOS_ACK_CUTOFF_TIME 30000 + +/** + * Maximum number of VERB_QOS_MEASUREMENT and VERB_ACK packets allowed to be + * processed within cutoff time. Separate totals are kept for each type but + * the limit is the same for both. + * + * This limits how often this peer will compute statistical estimates + * of various QoS measures from a VERB_QOS_MEASUREMENT or VERB_ACK packets to + * CUTOFF_LIMIT times per CUTOFF_TIME milliseconds per peer to prevent + * this from being useful for DOS amplification attacks. + */ +#define ZT_PATH_QOS_ACK_CUTOFF_LIMIT 128 + +/** + * Path choice history window size. This is used to keep track of which paths were + * previously selected so that we can maintain a target allocation over time. + */ +#define ZT_MULTIPATH_PROPORTION_WIN_SZ 128 + +/** + * How often we will sample packet latency. Should be at least greater than ZT_PING_CHECK_INVERVAL + * since we will record a 0 bit/s measurement if no valid latency measurement was made within this + * window of time. + */ +#define ZT_PATH_LATENCY_SAMPLE_INTERVAL (ZT_MULTIPATH_PEER_PING_PERIOD * 2) + +/** + * Interval used for rate-limiting the computation of path quality estimates. + */ +#define ZT_PATH_QUALITY_COMPUTE_INTERVAL 1000 + +/** + * Number of samples to consider when computing real-time path statistics + */ +#define ZT_PATH_QUALITY_METRIC_REALTIME_CONSIDERATION_WIN_SZ 128 + +/** + * Number of samples to consider when computing performing long-term path quality analysis. + * By default this value is set to ZT_PATH_QUALITY_METRIC_REALTIME_CONSIDERATION_WIN_SZ but can + * be set to any value greater than that to observe longer-term path quality behavior. + */ +#define ZT_PATH_QUALITY_METRIC_WIN_SZ ZT_PATH_QUALITY_METRIC_REALTIME_CONSIDERATION_WIN_SZ + +/** + * Maximum acceptable Packet Delay Variance (PDV) over a path + */ +#define ZT_PATH_MAX_PDV 1000 + +/** + * Maximum acceptable time interval between expectation and receipt of at least one ACK over a path + */ +#define ZT_PATH_MAX_AGE 30000 + +/** + * Maximum acceptable mean latency over a path + */ +#define ZT_PATH_MAX_MEAN_LATENCY 1000 + +/** + * How much each factor contributes to the "stability" score of a path + */ +#define ZT_PATH_CONTRIB_PDV (1.0 / 3.0) +#define ZT_PATH_CONTRIB_LATENCY (1.0 / 3.0) +#define ZT_PATH_CONTRIB_THROUGHPUT_DISTURBANCE (1.0 / 3.0) + +/** + * How much each factor contributes to the "quality" score of a path + */ +#define ZT_PATH_CONTRIB_STABILITY (0.75 / 3.0) +#define ZT_PATH_CONTRIB_THROUGHPUT (1.50 / 3.0) +#define ZT_PATH_CONTRIB_SCOPE (0.75 / 3.0) + +/** + * How often a QoS packet is sent + */ +#define ZT_PATH_QOS_INTERVAL 3000 + +/** + * Min and max acceptable sizes for a VERB_QOS_MEASUREMENT packet + */ +#define ZT_PATH_MIN_QOS_PACKET_SZ 8 + 1 +#define ZT_PATH_MAX_QOS_PACKET_SZ 1400 + +/** + * How many ID:sojourn time pairs in a single QoS packet + */ +#define ZT_PATH_QOS_TABLE_SIZE ((ZT_PATH_MAX_QOS_PACKET_SZ * 8) / (64 + 16)) + +/** + * Maximum number of outgoing packets we monitor for QoS information + */ +#define ZT_PATH_MAX_OUTSTANDING_QOS_RECORDS 128 + +/** + * Timeout for QoS records + */ +#define ZT_PATH_QOS_TIMEOUT (ZT_PATH_QOS_INTERVAL * 2) + +/** + * How often the service tests the path throughput + */ +#define ZT_PATH_THROUGHPUT_MEASUREMENT_INTERVAL (ZT_PATH_ACK_INTERVAL * 8) + +/** + * Minimum amount of time between each ACK packet + */ +#define ZT_PATH_ACK_INTERVAL 1000 + +/** + * How often an aggregate link statistics report is emitted into this tracing system + */ +#define ZT_PATH_AGGREGATE_STATS_REPORT_INTERVAL 60000 + +/** + * How much an aggregate link's component paths can vary from their target allocation + * before the link is considered to be in a state of imbalance. + */ +#define ZT_PATH_IMBALANCE_THRESHOLD 0.20 + +/** + * Max allowable time spent in any queue + */ +#define ZT_QOS_TARGET 5 // ms + +/** + * Time period where the time spent in the queue by a packet should fall below + * target at least once + */ +#define ZT_QOS_INTERVAL 100 // ms + +/** + * The number of bytes that each queue is allowed to send during each DRR cycle. + * This approximates a single-byte-based fairness queuing scheme + */ +#define ZT_QOS_QUANTUM ZT_DEFAULT_MTU + +/** + * The maximum total number of packets that can be queued among all + * active/inactive, old/new queues + */ +#define ZT_QOS_MAX_ENQUEUED_PACKETS 1024 + +/** + * Number of QoS queues (buckets) + */ +#define ZT_QOS_NUM_BUCKETS 9 + +/** + * All unspecified traffic is put in this bucket. Anything in a bucket with a smaller + * value is de-prioritized. Anything in a bucket with a higher value is prioritized over + * other traffic. + */ +#define ZT_QOS_DEFAULT_BUCKET 0 + +/** * How frequently to send heartbeats over in-use paths */ #define ZT_PATH_HEARTBEAT_PERIOD 14000 @@ -291,6 +460,16 @@ #define ZT_PEER_PING_PERIOD 60000 /** + * Delay between full-fledge pings of directly connected peers. + * + * With multipath bonding enabled ping peers more often to measure + * packet loss and latency. This uses more bandwidth so is disabled + * by default to avoid increasing idle bandwidth use for regular + * links. + */ +#define ZT_MULTIPATH_PEER_PING_PERIOD 5000 + +/** * Paths are considered expired if they have not sent us a real packet in this long */ #define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 4) + 3000) @@ -303,7 +482,11 @@ /** * Timeout for overall peer activity (measured from last receive) */ +#ifndef ZT_SDK #define ZT_PEER_ACTIVITY_TIMEOUT 500000 +#else +#define ZT_PEER_ACTIVITY_TIMEOUT 30000 +#endif /** * General rate limit timeout for multiple packet types (HELLO, etc.) @@ -349,14 +532,19 @@ #define ZT_MAX_BRIDGE_ROUTES 67108864 /** - * If there is no known route, spam to up to this many active bridges + * If there is no known L2 bridging route, spam to up to this many active bridges */ #define ZT_MAX_BRIDGE_SPAM 32 /** * Interval between direct path pushes in milliseconds */ -#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000 +#define ZT_DIRECT_PATH_PUSH_INTERVAL 15000 + +/** + * Interval between direct path pushes in milliseconds if we already have a path + */ +#define ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH 120000 /** * Time horizon for push direct paths cutoff @@ -441,16 +629,7 @@ */ #define ZT_THREAD_MIN_STACK_SIZE 1048576 -/* Ethernet frame types that might be relevant to us */ -#define ZT_ETHERTYPE_IPV4 0x0800 -#define ZT_ETHERTYPE_ARP 0x0806 -#define ZT_ETHERTYPE_RARP 0x8035 -#define ZT_ETHERTYPE_ATALK 0x809b -#define ZT_ETHERTYPE_AARP 0x80f3 -#define ZT_ETHERTYPE_IPX_A 0x8137 -#define ZT_ETHERTYPE_IPX_B 0x8138 -#define ZT_ETHERTYPE_IPV6 0x86dd - +// Exceptions thrown in core ZT code #define ZT_EXCEPTION_OUT_OF_BOUNDS 100 #define ZT_EXCEPTION_OUT_OF_MEMORY 101 #define ZT_EXCEPTION_PRIVATE_KEY_REQUIRED 102 diff --git a/node/Credential.hpp b/node/Credential.hpp index 34e94162..e7962fbb 100644 --- a/node/Credential.hpp +++ b/node/Credential.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index f89b6ffc..8eb10a0b 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index 777e88dc..10aab736 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -399,6 +399,10 @@ private: { return ((unsigned long)i * (unsigned long)0x9e3779b1); } + static inline unsigned long _hc(const int i) + { + return ((unsigned long)i * (unsigned long)0x9e3379b1); + } inline void _grow() { diff --git a/node/Identity.cpp b/node/Identity.cpp index 03f27083..e914b270 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -50,8 +50,8 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub SHA512::hash(digest,publicKey,publicKeyBytes); // Initialize genmem[] using Salsa20 in a CBC-like configuration since - // ordinary Salsa20 is randomly seekable. This is good for a cipher - // but is not what we want for sequential memory-harndess. + // ordinary Salsa20 is randomly seek-able. This is good for a cipher + // but is not what we want for sequential memory-hardness. memset(genmem,0,ZT_IDENTITY_GEN_MEMORY); Salsa20 s20(digest,(char *)digest + 32); s20.crypt20((char *)genmem,(char *)genmem,64); diff --git a/node/Identity.hpp b/node/Identity.hpp index 74716aa0..f559bcc5 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -248,7 +248,7 @@ public: if (b[p++] != 0) throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE; - ZT_FAST_MEMCPY(_publicKey.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); + memcpy(_publicKey.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN; unsigned int privateKeyLength = (unsigned int)b[p++]; @@ -256,7 +256,7 @@ public: if (privateKeyLength != ZT_C25519_PRIVATE_KEY_LEN) throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; _privateKey = new C25519::Private(); - ZT_FAST_MEMCPY(_privateKey->data,b.field(p,ZT_C25519_PRIVATE_KEY_LEN),ZT_C25519_PRIVATE_KEY_LEN); + memcpy(_privateKey->data,b.field(p,ZT_C25519_PRIVATE_KEY_LEN),ZT_C25519_PRIVATE_KEY_LEN); p += ZT_C25519_PRIVATE_KEY_LEN; } diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ff4fc94b..b774f1df 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -80,6 +80,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) if (!trusted) { if (!dearmor(peer->key())) { RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops(),"invalid MAC"); + _path->recordInvalidPacket(); return true; } } @@ -90,29 +91,37 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) } const Packet::Verb v = verb(); + bool r = true; switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(tPtr,_path,hops(),packetId(),v,0,Packet::VERB_NOP,false,0); - return true; - case Packet::VERB_HELLO: return _doHELLO(RR,tPtr,true); - case Packet::VERB_ERROR: return _doERROR(RR,tPtr,peer); - case Packet::VERB_OK: return _doOK(RR,tPtr,peer); - case Packet::VERB_WHOIS: return _doWHOIS(RR,tPtr,peer); - case Packet::VERB_RENDEZVOUS: return _doRENDEZVOUS(RR,tPtr,peer); - case Packet::VERB_FRAME: return _doFRAME(RR,tPtr,peer); - case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,tPtr,peer); - case Packet::VERB_ECHO: return _doECHO(RR,tPtr,peer); - case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,tPtr,peer); - case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,tPtr,peer); - case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,tPtr,peer); - case Packet::VERB_NETWORK_CONFIG: return _doNETWORK_CONFIG(RR,tPtr,peer); - case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,tPtr,peer); - case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,tPtr,peer); - case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,tPtr,peer); - case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,tPtr,peer); - case Packet::VERB_REMOTE_TRACE: return _doREMOTE_TRACE(RR,tPtr,peer); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),v,0,Packet::VERB_NOP,false,0); + break; + case Packet::VERB_HELLO: r = _doHELLO(RR,tPtr,true); break; + case Packet::VERB_ACK: r = _doACK(RR,tPtr,peer); break; + case Packet::VERB_QOS_MEASUREMENT: r = _doQOS_MEASUREMENT(RR,tPtr,peer); break; + case Packet::VERB_ERROR: r = _doERROR(RR,tPtr,peer); break; + case Packet::VERB_OK: r = _doOK(RR,tPtr,peer); break; + case Packet::VERB_WHOIS: r = _doWHOIS(RR,tPtr,peer); break; + case Packet::VERB_RENDEZVOUS: r = _doRENDEZVOUS(RR,tPtr,peer); break; + case Packet::VERB_FRAME: r = _doFRAME(RR,tPtr,peer); break; + case Packet::VERB_EXT_FRAME: r = _doEXT_FRAME(RR,tPtr,peer); break; + case Packet::VERB_ECHO: r = _doECHO(RR,tPtr,peer); break; + case Packet::VERB_MULTICAST_LIKE: r = _doMULTICAST_LIKE(RR,tPtr,peer); break; + case Packet::VERB_NETWORK_CREDENTIALS: r = _doNETWORK_CREDENTIALS(RR,tPtr,peer); break; + case Packet::VERB_NETWORK_CONFIG_REQUEST: r = _doNETWORK_CONFIG_REQUEST(RR,tPtr,peer); break; + case Packet::VERB_NETWORK_CONFIG: r = _doNETWORK_CONFIG(RR,tPtr,peer); break; + case Packet::VERB_MULTICAST_GATHER: r = _doMULTICAST_GATHER(RR,tPtr,peer); break; + case Packet::VERB_MULTICAST_FRAME: r = _doMULTICAST_FRAME(RR,tPtr,peer); break; + case Packet::VERB_PUSH_DIRECT_PATHS: r = _doPUSH_DIRECT_PATHS(RR,tPtr,peer); break; + case Packet::VERB_USER_MESSAGE: r = _doUSER_MESSAGE(RR,tPtr,peer); break; + case Packet::VERB_REMOTE_TRACE: r = _doREMOTE_TRACE(RR,tPtr,peer); break; } + if (r) { + RR->node->statsLogVerb((unsigned int)v,(unsigned int)size()); + return true; + } + return false; } else { RR->sw->requestWhois(tPtr,RR->node->now(),sourceAddress); return false; @@ -169,7 +178,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar networkId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); const SharedPtr<Network> network(RR->node->network(networkId)); const int64_t now = RR->node->now(); - if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) + if ((network)&&(network->config().com)) network->pushCredentialsNow(tPtr,peer->address(),now); } break; @@ -194,7 +203,58 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar default: break; } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,networkId); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,networkId); + + return true; +} + +bool IncomingPacket::_doACK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer) +{ + if (!peer->rateGateACK(RR->node->now())) + return true; + /* Dissect incoming ACK packet. From this we can estimate current throughput of the path, establish known + * maximums and detect packet loss. */ + if (peer->localMultipathSupport()) { + int32_t ackedBytes; + if (payloadLength() != sizeof(ackedBytes)) { + return true; // ignore + } + memcpy(&ackedBytes, payload(), sizeof(ackedBytes)); + _path->receivedAck(RR->node->now(), Utils::ntoh(ackedBytes)); + peer->inferRemoteMultipathEnabled(); + } + + return true; +} + +bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer) +{ + if (!peer->rateGateQoS(RR->node->now())) + return true; + /* Dissect incoming QoS packet. From this we can compute latency values and their variance. + * The latency variance is used as a measure of "jitter". */ + if (peer->localMultipathSupport()) { + if (payloadLength() > ZT_PATH_MAX_QOS_PACKET_SZ || payloadLength() < ZT_PATH_MIN_QOS_PACKET_SZ) { + return true; // ignore + } + const int64_t now = RR->node->now(); + uint64_t rx_id[ZT_PATH_QOS_TABLE_SIZE]; + uint16_t rx_ts[ZT_PATH_QOS_TABLE_SIZE]; + char *begin = (char *)payload(); + char *ptr = begin; + int count = 0; + int len = payloadLength(); + // Read packet IDs and latency compensation intervals for each packet tracked by this QoS packet + while (ptr < (begin + len) && (count < ZT_PATH_QOS_TABLE_SIZE)) { + memcpy((void*)&rx_id[count], ptr, sizeof(uint64_t)); + ptr+=sizeof(uint64_t); + memcpy((void*)&rx_ts[count], ptr, sizeof(uint16_t)); + ptr+=sizeof(uint16_t); + count++; + } + _path->receivedQoS(now, count, rx_id, rx_ts); + peer->inferRemoteMultipathEnabled(); + } return true; } @@ -395,7 +455,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool _path->send(RR,tPtr,outp.data(),outp.size(),now); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),pid,payloadLength(),Packet::VERB_HELLO,0,Packet::VERB_NOP,false,0); return true; } @@ -413,9 +473,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP case Packet::VERB_HELLO: { const uint64_t latency = RR->node->now() - at<uint64_t>(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP); - if (latency > ZT_HELLO_MAX_ALLOWABLE_LATENCY) - return true; - const unsigned int vProto = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION]; const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION]; const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION]; @@ -445,8 +502,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP } } - if (!hops()) - _path->updateLatency((unsigned int)latency); + if (!hops()) { + _path->updateLatency((unsigned int)latency,RR->node->now()); + } peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); @@ -507,7 +565,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP default: break; } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false,networkId); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_OK,inRePacketId,inReVerb,false,networkId); return true; } @@ -542,7 +600,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,0); return true; } @@ -556,7 +614,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const const unsigned int port = at<uint16_t>(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { - const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); + InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localSocket(),atAddr)) { const uint64_t junk = RR->node->prng(); RR->node->putPacket(tPtr,_path->localSocket(),atAddr,&junk,4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls @@ -566,7 +624,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,0); return true; } @@ -589,13 +647,11 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar } } else { _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_FRAME,true); + return false; } - } else { - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished,nwid); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished,nwid); return true; } @@ -618,8 +674,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const if (!network->gate(tPtr,peer)) { RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,true); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); - return true; + return false; } if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { @@ -630,7 +685,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); if ((!from)||(from == network->mac())) { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } @@ -641,19 +696,19 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const network->learnBridgeRoute(from,peer->address()); } else { RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (remote)"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (to.isMulticast()) { if (network->config().multicastLimit == 0) { RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"multicast disabled"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } else if (!network->config().permitsBridging(RR->identity.address())) { RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (local)"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } @@ -673,9 +728,9 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); } else { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); } return true; @@ -695,7 +750,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share outp.armor(peer->key(),true); _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),pid,payloadLength(),Packet::VERB_ECHO,0,Packet::VERB_NOP,false,0); return true; } @@ -703,45 +758,25 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer) { const int64_t now = RR->node->now(); + bool authorized = false; + uint64_t lastNwid = 0; - uint64_t authOnNetwork[256]; // cache for approved network IDs - unsigned int authOnNetworkCount = 0; - SharedPtr<Network> network; - bool trustEstablished = false; - - // Iterate through 18-byte network,MAC,ADI tuples + // Packet contains a series of 18-byte network,MAC,ADI tuples for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18) { const uint64_t nwid = at<uint64_t>(ptr); - - bool auth = false; - for(unsigned int i=0;i<authOnNetworkCount;++i) { - if (nwid == authOnNetwork[i]) { - auth = true; - break; - } - } - if (!auth) { - if ((!network)||(network->id() != nwid)) - network = RR->node->network(nwid); - const bool authOnNet = ((network)&&(network->gate(tPtr,peer))); - if (!authOnNet) - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - trustEstablished |= authOnNet; - if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { - auth = true; - if (authOnNetworkCount < 256) // sanity check, packets can't really be this big - authOnNetwork[authOnNetworkCount++] = nwid; - } - } - - if (auth) { - const MulticastGroup group(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)); - RR->mc->add(tPtr,now,nwid,group,peer->address()); + if (nwid != lastNwid) { + lastNwid = nwid; + SharedPtr<Network> network(RR->node->network(nwid)); + if (network) + authorized = network->gate(tPtr,peer); + if (!authorized) + authorized = ((RR->topology->amUpstream())||(RR->node->localControllerHasAuthorized(now,nwid,peer->address()))); } + if (authorized) + RR->mc->add(tPtr,now,nwid,MulticastGroup(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)),peer->address()); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); - + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false,0); return true; } @@ -774,7 +809,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t case Membership::ADD_DEFERRED_FOR_WHOIS: return false; } - } else RR->mc->addCredential(tPtr,com,false); + } } } ++p; // skip trailing 0 after COMs if present @@ -863,7 +898,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); return true; } @@ -889,7 +924,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false,nwid); + peer->received(tPtr,_path,hopCount,requestPacketId,payloadLength(),Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false,nwid); return true; } @@ -910,7 +945,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,c } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false,(network) ? network->id() : 0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false,(network) ? network->id() : 0); return true; } @@ -928,18 +963,23 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr try { CertificateOfMembership com; com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM); - if (com) { - if (network) - network->addCredential(tPtr,com); - else RR->mc->addCredential(tPtr,com,false); - } + if ((com)&&(network)) + network->addCredential(tPtr,com); } catch ( ... ) {} // discard invalid COMs } - const bool trustEstablished = ((network)&&(network->gate(tPtr,peer))); - if (!trustEstablished) - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { + bool trustEstablished = false; + if (network) { + if (network->gate(tPtr,peer)) { + trustEstablished = true; + } else { + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + return false; + } + } + + const int64_t now = RR->node->now(); + if ((gatherLimit > 0)&&((trustEstablished)||(RR->topology->amUpstream())||(RR->node->localControllerHasAuthorized(now,nwid,peer->address())))) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); outp.append(packetId()); @@ -949,11 +989,11 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit); if (gatheredLocally > 0) { outp.armor(peer->key(),true); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),now); } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished,nwid); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished,nwid); return true; } @@ -977,10 +1017,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, } if (!network->gate(tPtr,peer)) { - RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,true); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); - return true; + return false; } unsigned int gatherLimit = 0; @@ -1003,19 +1041,19 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, if (network->config().multicastLimit == 0) { RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"multicast disabled"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); return true; } if ((frameLen > 0)&&(frameLen <= ZT_MAX_MTU)) { if (!to.mac().isMulticast()) { RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"destination not multicast"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } @@ -1029,7 +1067,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, network->learnBridgeRoute(from,peer->address()); } else { RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"bridging not allowed (remote)"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } @@ -1052,10 +1090,10 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); } else { _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); + return false; } return true; @@ -1067,7 +1105,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); return true; } @@ -1091,7 +1129,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt case 4: { const InetAddress a(field(ptr,4),4,at<uint16_t>(ptr + 4)); if ( - ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { @@ -1105,7 +1143,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt case 6: { const InetAddress a(field(ptr,16),16,at<uint16_t>(ptr + 16)); if ( - ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { @@ -1120,7 +1158,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt ptr += addrLen; } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); return true; } @@ -1136,7 +1174,7 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast<const void *>(&um)); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false,0); return true; } @@ -1160,23 +1198,20 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,false,0); + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,false,0); return true; } void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer,const uint64_t nwid) { - const int64_t now = RR->node->now(); - if (peer->rateGateOutgoingComRequest(now)) { - Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((uint8_t)verb()); - outp.append(packetId()); - outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); - outp.append(nwid); - outp.armor(peer->key(),true); - _path->send(RR,tPtr,outp.data(),outp.size(),now); - } + Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((uint8_t)verb()); + outp.append(packetId()); + outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); + outp.append(nwid); + outp.armor(peer->key(),true); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } } // namespace ZeroTier diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 88f4f066..5c4f896b 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -125,6 +125,8 @@ private: // been authenticated, decrypted, decompressed, and classified. bool _doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer); bool _doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated); + bool _doACK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer); + bool _doQOS_MEASUREMENT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer); bool _doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer); bool _doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer); bool _doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer); diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 36b4e434..15d2f878 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -1,11 +1,11 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or - * (at your oion) any later version. + * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -46,39 +46,38 @@ InetAddress::IpScope InetAddress::ipScope() const case AF_INET: { const uint32_t ip = Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr); switch(ip >> 24) { - case 0x00: return IP_SCOPE_NONE; // 0.0.0.0/8 (reserved, never used) - case 0x06: return IP_SCOPE_PSEUDOPRIVATE; // 6.0.0.0/8 (US Army) - case 0x0a: return IP_SCOPE_PRIVATE; // 10.0.0.0/8 - case 0x0b: return IP_SCOPE_PSEUDOPRIVATE; // 11.0.0.0/8 (US DoD) - case 0x15: return IP_SCOPE_PSEUDOPRIVATE; // 21.0.0.0/8 (US DDN-RVN) - case 0x16: return IP_SCOPE_PSEUDOPRIVATE; // 22.0.0.0/8 (US DISA) - case 0x19: return IP_SCOPE_PSEUDOPRIVATE; // 25.0.0.0/8 (UK Ministry of Defense) - case 0x1a: return IP_SCOPE_PSEUDOPRIVATE; // 26.0.0.0/8 (US DISA) - case 0x1c: return IP_SCOPE_PSEUDOPRIVATE; // 28.0.0.0/8 (US DSI-North) - case 0x1d: return IP_SCOPE_PSEUDOPRIVATE; // 29.0.0.0/8 (US DISA) - case 0x1e: return IP_SCOPE_PSEUDOPRIVATE; // 30.0.0.0/8 (US DISA) - case 0x2c: return IP_SCOPE_PSEUDOPRIVATE; // 44.0.0.0/8 (Amateur Radio) - case 0x33: return IP_SCOPE_PSEUDOPRIVATE; // 51.0.0.0/8 (UK Department of Social Security) - case 0x37: return IP_SCOPE_PSEUDOPRIVATE; // 55.0.0.0/8 (US DoD) - case 0x38: return IP_SCOPE_PSEUDOPRIVATE; // 56.0.0.0/8 (US Postal Service) + case 0x00: return IP_SCOPE_NONE; // 0.0.0.0/8 (reserved, never used) + case 0x06: return IP_SCOPE_PSEUDOPRIVATE; // 6.0.0.0/8 (US Army) + case 0x0a: return IP_SCOPE_PRIVATE; // 10.0.0.0/8 + case 0x0b: return IP_SCOPE_PSEUDOPRIVATE; // 11.0.0.0/8 (US DoD) + case 0x15: return IP_SCOPE_PSEUDOPRIVATE; // 21.0.0.0/8 (US DDN-RVN) + case 0x16: return IP_SCOPE_PSEUDOPRIVATE; // 22.0.0.0/8 (US DISA) + case 0x19: return IP_SCOPE_PSEUDOPRIVATE; // 25.0.0.0/8 (UK Ministry of Defense) + case 0x1a: return IP_SCOPE_PSEUDOPRIVATE; // 26.0.0.0/8 (US DISA) + case 0x1c: return IP_SCOPE_PSEUDOPRIVATE; // 28.0.0.0/8 (US DSI-North) + case 0x1d: return IP_SCOPE_PSEUDOPRIVATE; // 29.0.0.0/8 (US DISA) + case 0x1e: return IP_SCOPE_PSEUDOPRIVATE; // 30.0.0.0/8 (US DISA) + case 0x33: return IP_SCOPE_PSEUDOPRIVATE; // 51.0.0.0/8 (UK Department of Social Security) + case 0x37: return IP_SCOPE_PSEUDOPRIVATE; // 55.0.0.0/8 (US DoD) + case 0x38: return IP_SCOPE_PSEUDOPRIVATE; // 56.0.0.0/8 (US Postal Service) case 0x64: - if ((ip & 0xffc00000) == 0x64400000) return IP_SCOPE_PRIVATE; // 100.64.0.0/10 + if ((ip & 0xffc00000) == 0x64400000) return IP_SCOPE_PRIVATE; // 100.64.0.0/10 break; - case 0x7f: return IP_SCOPE_LOOPBACK; // 127.0.0.0/8 + case 0x7f: return IP_SCOPE_LOOPBACK; // 127.0.0.0/8 case 0xa9: - if ((ip & 0xffff0000) == 0xa9fe0000) return IP_SCOPE_LINK_LOCAL; // 169.254.0.0/16 + if ((ip & 0xffff0000) == 0xa9fe0000) return IP_SCOPE_LINK_LOCAL; // 169.254.0.0/16 break; case 0xac: - if ((ip & 0xfff00000) == 0xac100000) return IP_SCOPE_PRIVATE; // 172.16.0.0/12 + if ((ip & 0xfff00000) == 0xac100000) return IP_SCOPE_PRIVATE; // 172.16.0.0/12 break; case 0xc0: - if ((ip & 0xffff0000) == 0xc0a80000) return IP_SCOPE_PRIVATE; // 192.168.0.0/16 + if ((ip & 0xffff0000) == 0xc0a80000) return IP_SCOPE_PRIVATE; // 192.168.0.0/16 break; - case 0xff: return IP_SCOPE_NONE; // 255.0.0.0/8 (broadcast, or unused/unusable) + case 0xff: return IP_SCOPE_NONE; // 255.0.0.0/8 (broadcast, or unused/unusable) } switch(ip >> 28) { - case 0xe: return IP_SCOPE_MULTICAST; // 224.0.0.0/4 - case 0xf: return IP_SCOPE_PSEUDOPRIVATE; // 240.0.0.0/4 ("reserved," usually unusable) + case 0xe: return IP_SCOPE_MULTICAST; // 224.0.0.0/4 + case 0xf: return IP_SCOPE_PSEUDOPRIVATE; // 240.0.0.0/4 ("reserved," usually unusable) } return IP_SCOPE_GLOBAL; } break; @@ -86,21 +85,21 @@ InetAddress::IpScope InetAddress::ipScope() const case AF_INET6: { const unsigned char *ip = reinterpret_cast<const unsigned char *>(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr); if ((ip[0] & 0xf0) == 0xf0) { - if (ip[0] == 0xff) return IP_SCOPE_MULTICAST; // ff00::/8 + if (ip[0] == 0xff) return IP_SCOPE_MULTICAST; // ff00::/8 if ((ip[0] == 0xfe)&&((ip[1] & 0xc0) == 0x80)) { unsigned int k = 2; while ((!ip[k])&&(k < 15)) ++k; if ((k == 15)&&(ip[15] == 0x01)) - return IP_SCOPE_LOOPBACK; // fe80::1/128 - else return IP_SCOPE_LINK_LOCAL; // fe80::/10 + return IP_SCOPE_LOOPBACK; // fe80::1/128 + else return IP_SCOPE_LINK_LOCAL; // fe80::/10 } - if ((ip[0] & 0xfe) == 0xfc) return IP_SCOPE_PRIVATE; // fc00::/7 + if ((ip[0] & 0xfe) == 0xfc) return IP_SCOPE_PRIVATE; // fc00::/7 } unsigned int k = 0; while ((!ip[k])&&(k < 15)) ++k; if (k == 15) { // all 0's except last byte - if (ip[15] == 0x01) return IP_SCOPE_LOOPBACK; // ::1/128 - if (ip[15] == 0x00) return IP_SCOPE_NONE; // ::/128 + if (ip[15] == 0x01) return IP_SCOPE_LOOPBACK; // ::1/128 + if (ip[15] == 0x00) return IP_SCOPE_NONE; // ::/128 } return IP_SCOPE_GLOBAL; } break; @@ -115,13 +114,13 @@ void InetAddress::set(const void *ipBytes,unsigned int ipLen,unsigned int port) memset(this,0,sizeof(InetAddress)); if (ipLen == 4) { uint32_t ipb[1]; - ZT_FAST_MEMCPY(ipb,ipBytes,4); + memcpy(ipb,ipBytes,4); ss_family = AF_INET; reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr = ipb[0]; reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton((uint16_t)port); } else if (ipLen == 16) { ss_family = AF_INET6; - ZT_FAST_MEMCPY(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,ipBytes,16); + memcpy(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,ipBytes,16); reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_port = Utils::hton((uint16_t)port); } } @@ -139,33 +138,23 @@ char *InetAddress::toString(char buf[64]) const char *InetAddress::toIpString(char buf[64]) const { + buf[0] = (char)0; switch(ss_family) { case AF_INET: { - const uint8_t *a = reinterpret_cast<const uint8_t *>(&(reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr)); - char *p = buf; - for(int i=0;;++i) { - Utils::decimal((unsigned long)a[i],p); - if (i != 3) { - while (*p) ++p; - *(p++) = '.'; - } else break; - } - } break; +#ifdef _WIN32 + inet_ntop(AF_INET, (void*)&reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr, buf, INET_ADDRSTRLEN); +#else + inet_ntop(AF_INET, &reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr, buf, INET_ADDRSTRLEN); +#endif + } break; case AF_INET6: { - uint16_t a[8]; - ZT_FAST_MEMCPY(a,reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,16); - char *p = buf; - for(int i=0;i<8;++i) { - Utils::hex(Utils::ntoh(a[i]),p); - p[4] = (i == 7) ? (char)0 : ':'; - p += 5; - } +#ifdef _WIN32 + inet_ntop(AF_INET6, (void*)reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr, buf, INET6_ADDRSTRLEN); +#else + inet_ntop(AF_INET6, reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr, buf, INET6_ADDRSTRLEN); +#endif } break; - - default: - buf[0] = (char)0; - break; } return buf; } @@ -191,30 +180,16 @@ bool InetAddress::fromString(const char *ipSlashPort) } if (strchr(buf,':')) { - uint16_t a[8]; - unsigned int b = 0; - char *saveptr = (char *)0; - for(char *s=Utils::stok(buf,":",&saveptr);((s)&&(b<8));s=Utils::stok((char *)0,":",&saveptr)) - a[b++] = Utils::hton((uint16_t)(Utils::hexStrToUInt(s) & 0xffff)); - struct sockaddr_in6 *const in6 = reinterpret_cast<struct sockaddr_in6 *>(this); + inet_pton(AF_INET6, buf, &in6->sin6_addr.s6_addr); in6->sin6_family = AF_INET6; - ZT_FAST_MEMCPY(in6->sin6_addr.s6_addr,a,16); in6->sin6_port = Utils::hton((uint16_t)port); - return true; } else if (strchr(buf,'.')) { - uint8_t a[4]; - unsigned int b = 0; - char *saveptr = (char *)0; - for(char *s=Utils::stok(buf,".",&saveptr);((s)&&(b<4));s=Utils::stok((char *)0,".",&saveptr)) - a[b++] = (uint8_t)(Utils::strToUInt(s) & 0xff); - struct sockaddr_in *const in = reinterpret_cast<struct sockaddr_in *>(this); + inet_pton(AF_INET, buf, &in->sin_addr.s_addr); in->sin_family = AF_INET; - ZT_FAST_MEMCPY(&(in->sin_addr.s_addr),a,4); in->sin_port = Utils::hton((uint16_t)port); - return true; } else { return false; @@ -238,7 +213,7 @@ InetAddress InetAddress::netmask() const nm[0] = 0; nm[1] = 0; } - ZT_FAST_MEMCPY(reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,nm,16); + memcpy(reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,nm,16); } break; } return r; @@ -264,10 +239,10 @@ InetAddress InetAddress::network() const case AF_INET6: { uint64_t nm[2]; const unsigned int bits = netmaskBits(); - ZT_FAST_MEMCPY(nm,reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,16); + memcpy(nm,reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,16); nm[0] &= Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits)))); nm[1] &= Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits)))); - ZT_FAST_MEMCPY(reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,nm,16); + memcpy(reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,nm,16); } break; } return r; diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index f69b3cc2..ce65bcd0 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -91,8 +91,8 @@ struct InetAddress : public sockaddr_storage }; InetAddress() { memset(this,0,sizeof(InetAddress)); } - InetAddress(const InetAddress &a) { ZT_FAST_MEMCPY(this,&a,sizeof(InetAddress)); } - InetAddress(const InetAddress *a) { ZT_FAST_MEMCPY(this,a,sizeof(InetAddress)); } + InetAddress(const InetAddress &a) { memcpy(this,&a,sizeof(InetAddress)); } + InetAddress(const InetAddress *a) { memcpy(this,a,sizeof(InetAddress)); } InetAddress(const struct sockaddr_storage &ss) { *this = ss; } InetAddress(const struct sockaddr_storage *ss) { *this = ss; } InetAddress(const struct sockaddr &sa) { *this = sa; } @@ -108,28 +108,28 @@ struct InetAddress : public sockaddr_storage inline InetAddress &operator=(const InetAddress &a) { if (&a != this) - ZT_FAST_MEMCPY(this,&a,sizeof(InetAddress)); + memcpy(this,&a,sizeof(InetAddress)); return *this; } inline InetAddress &operator=(const InetAddress *a) { if (a != this) - ZT_FAST_MEMCPY(this,a,sizeof(InetAddress)); + memcpy(this,a,sizeof(InetAddress)); return *this; } inline InetAddress &operator=(const struct sockaddr_storage &ss) { if (reinterpret_cast<const InetAddress *>(&ss) != this) - ZT_FAST_MEMCPY(this,&ss,sizeof(InetAddress)); + memcpy(this,&ss,sizeof(InetAddress)); return *this; } inline InetAddress &operator=(const struct sockaddr_storage *ss) { if (reinterpret_cast<const InetAddress *>(ss) != this) - ZT_FAST_MEMCPY(this,ss,sizeof(InetAddress)); + memcpy(this,ss,sizeof(InetAddress)); return *this; } @@ -137,7 +137,7 @@ struct InetAddress : public sockaddr_storage { if (reinterpret_cast<const InetAddress *>(&sa) != this) { memset(this,0,sizeof(InetAddress)); - ZT_FAST_MEMCPY(this,&sa,sizeof(struct sockaddr_in)); + memcpy(this,&sa,sizeof(struct sockaddr_in)); } return *this; } @@ -146,7 +146,7 @@ struct InetAddress : public sockaddr_storage { if (reinterpret_cast<const InetAddress *>(sa) != this) { memset(this,0,sizeof(InetAddress)); - ZT_FAST_MEMCPY(this,sa,sizeof(struct sockaddr_in)); + memcpy(this,sa,sizeof(struct sockaddr_in)); } return *this; } @@ -155,7 +155,7 @@ struct InetAddress : public sockaddr_storage { if (reinterpret_cast<const InetAddress *>(&sa) != this) { memset(this,0,sizeof(InetAddress)); - ZT_FAST_MEMCPY(this,&sa,sizeof(struct sockaddr_in6)); + memcpy(this,&sa,sizeof(struct sockaddr_in6)); } return *this; } @@ -164,7 +164,7 @@ struct InetAddress : public sockaddr_storage { if (reinterpret_cast<const InetAddress *>(sa) != this) { memset(this,0,sizeof(InetAddress)); - ZT_FAST_MEMCPY(this,sa,sizeof(struct sockaddr_in6)); + memcpy(this,sa,sizeof(struct sockaddr_in6)); } return *this; } @@ -175,10 +175,10 @@ struct InetAddress : public sockaddr_storage memset(this,0,sizeof(InetAddress)); switch(sa.sa_family) { case AF_INET: - ZT_FAST_MEMCPY(this,&sa,sizeof(struct sockaddr_in)); + memcpy(this,&sa,sizeof(struct sockaddr_in)); break; case AF_INET6: - ZT_FAST_MEMCPY(this,&sa,sizeof(struct sockaddr_in6)); + memcpy(this,&sa,sizeof(struct sockaddr_in6)); break; } } @@ -191,10 +191,10 @@ struct InetAddress : public sockaddr_storage memset(this,0,sizeof(InetAddress)); switch(sa->sa_family) { case AF_INET: - ZT_FAST_MEMCPY(this,sa,sizeof(struct sockaddr_in)); + memcpy(this,sa,sizeof(struct sockaddr_in)); break; case AF_INET6: - ZT_FAST_MEMCPY(this,sa,sizeof(struct sockaddr_in6)); + memcpy(this,sa,sizeof(struct sockaddr_in6)); break; } } @@ -388,7 +388,7 @@ struct InetAddress : public sockaddr_storage break; case AF_INET6: r.ss_family = AF_INET6; - ZT_FAST_MEMCPY(reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,16); + memcpy(reinterpret_cast<struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,16); break; } return r; @@ -537,12 +537,12 @@ struct InetAddress : public sockaddr_storage return (unsigned int)(b.template at<uint16_t>(p) + 3); // other addresses begin with 16-bit non-inclusive length case 0x04: ss_family = AF_INET; - ZT_FAST_MEMCPY(&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr),b.field(p,4),4); p += 4; + memcpy(&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr),b.field(p,4),4); p += 4; reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2; break; case 0x06: ss_family = AF_INET6; - ZT_FAST_MEMCPY(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,b.field(p,16),16); p += 16; + memcpy(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,b.field(p,16),16); p += 16; reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2; break; default: diff --git a/node/MAC.hpp b/node/MAC.hpp index c08323a4..e1e20e37 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Membership.cpp b/node/Membership.cpp index affe7a71..92dd30cb 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -35,61 +35,42 @@ #include "Node.hpp" #include "Trace.hpp" -#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 3) - namespace ZeroTier { Membership::Membership() : _lastUpdatedMulticast(0), - _lastPushedCom(0), _comRevocationThreshold(0), + _lastPushedCredentials(0), _revocations(4), _remoteTags(4), _remoteCaps(4), _remoteCoos(4) { - resetPushState(); } -void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) +void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex) { - bool sendCom = ( (nconf.com) && ( ((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); - - const Capability *sendCap; - if (localCapabilityIndex >= 0) { - sendCap = &(nconf.capabilities[localCapabilityIndex]); - if ( ((now - _localCredLastPushed.cap[localCapabilityIndex]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) - _localCredLastPushed.cap[localCapabilityIndex] = now; - else sendCap = (const Capability *)0; - } else sendCap = (const Capability *)0; + const Capability *sendCap = (localCapabilityIndex >= 0) ? &(nconf.capabilities[localCapabilityIndex]) : (const Capability *)0; const Tag *sendTags[ZT_MAX_NETWORK_TAGS]; unsigned int sendTagCount = 0; - for(unsigned int t=0;t<nconf.tagCount;++t) { - if ( ((now - _localCredLastPushed.tag[t]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCredLastPushed.tag[t] = now; - sendTags[sendTagCount++] = &(nconf.tags[t]); - } - } + for(unsigned int t=0;t<nconf.tagCount;++t) + sendTags[sendTagCount++] = &(nconf.tags[t]); const CertificateOfOwnership *sendCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; unsigned int sendCooCount = 0; - for(unsigned int c=0;c<nconf.certificateOfOwnershipCount;++c) { - if ( ((now - _localCredLastPushed.coo[c]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCredLastPushed.coo[c] = now; - sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]); - } - } + for(unsigned int c=0;c<nconf.certificateOfOwnershipCount;++c) + sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]); unsigned int tagPtr = 0; unsigned int cooPtr = 0; + bool sendCom = (bool)(nconf.com); while ((tagPtr < sendTagCount)||(cooPtr < sendCooCount)||(sendCom)||(sendCap)) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (sendCom) { sendCom = false; nconf.com.serialize(outp); - _lastPushedCom = now; } outp.append((uint8_t)0x00); @@ -123,6 +104,8 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const i outp.compress(); RR->sw->send(tPtr,outp,true); } + + _lastPushedCredentials = now; } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com) diff --git a/node/Membership.hpp b/node/Membership.hpp index ad0bb73e..6f2d0339 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -67,10 +67,7 @@ public: Membership(); /** - * Send COM and other credentials to this peer if needed - * - * This checks last pushed times for our COM and for other credentials and - * sends VERB_NETWORK_CREDENTIALS if the recipient might need them. + * Send COM and other credentials to this peer * * @param RR Runtime environment * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call @@ -78,9 +75,16 @@ public: * @param peerAddress Address of member peer (the one that this Membership describes) * @param nconf My network config * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none - * @param force If true, send objects regardless of last push time */ - void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); + void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex); + + /** + * @return True if we haven't pushed credentials in a long time (to cause proactive credential push) + */ + inline bool shouldPushCredentials(const int64_t now) const + { + return ((now - _lastPushedCredentials) > ZT_PEER_ACTIVITY_TIMEOUT); + } /** * Check whether we should push MULTICAST_LIKEs to this peer, and update last sent time if true @@ -183,17 +187,6 @@ public: void clean(const int64_t now,const NetworkConfig &nconf); /** - * Reset last pushed time for local credentials - * - * This is done when we update our network configuration and our credentials have changed - */ - inline void resetPushState() - { - _lastPushedCom = 0; - memset(&_localCredLastPushed,0,sizeof(_localCredLastPushed)); - } - - /** * Generates a key for the internal use in indexing credentials by type and credential ID */ static uint64_t credentialKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); } @@ -211,7 +204,7 @@ private: } template<typename C> - void _cleanCredImpl(const NetworkConfig &nconf,Hashtable<uint32_t,C> &remoteCreds) + inline void _cleanCredImpl(const NetworkConfig &nconf,Hashtable<uint32_t,C> &remoteCreds) { uint32_t *k = (uint32_t *)0; C *v = (C *)0; @@ -225,12 +218,12 @@ private: // Last time we pushed MULTICAST_LIKE(s) int64_t _lastUpdatedMulticast; - // Last time we pushed our COM to this peer - int64_t _lastPushedCom; - // Revocation threshold for COM or 0 if none int64_t _comRevocationThreshold; + // Time we last pushed credentials + int64_t _lastPushedCredentials; + // Remote member's latest network COM CertificateOfMembership _com; @@ -242,13 +235,6 @@ private: Hashtable< uint32_t,Capability > _remoteCaps; Hashtable< uint32_t,CertificateOfOwnership > _remoteCoos; - // Time we last pushed our local credentials to this member - struct { - uint64_t tag[ZT_MAX_NETWORK_TAGS]; - uint64_t cap[ZT_MAX_NETWORK_CAPABILITIES]; - uint64_t coo[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; - } _localCredLastPushed; - public: class CapabilityIterator { diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index 0f4a621e..184df1ce 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -68,7 +68,7 @@ public: * Derive the multicast group used for address resolution (ARP/NDP) for an IP * * @param ip IP address (port field is ignored) - * @return Multicat group for ARP/NDP + * @return Multicast group for ARP/NDP */ static inline MulticastGroup deriveMulticastGroupForAddressResolution(const InetAddress &ip) { diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 753e4ee0..67e133c0 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -42,8 +42,7 @@ namespace ZeroTier { Multicaster::Multicaster(const RuntimeEnvironment *renv) : RR(renv), - _groups(256), - _gatherAuth(256) + _groups(32) { } @@ -190,7 +189,7 @@ void Multicaster::send( for(unsigned int i=0;i<multicastReplicatorCount;++i) { const SharedPtr<Peer> p(RR->topology->getPeerNoCache(multicastReplicators[i])); if ((p)&&(p->isAlive(now))) { - const SharedPtr<Path> pp(p->getBestPath(now,false)); + const SharedPtr<Path> pp(p->getAppropriatePath(now,false)); if ((pp)&&(pp->latency() < bestMulticastReplicatorLatency)) { bestMulticastReplicatorLatency = pp->latency(); bestMulticastReplicatorPath = pp; @@ -415,25 +414,6 @@ void Multicaster::clean(int64_t now) } } } - - { - Mutex::Lock _l(_gatherAuth_m); - _GatherAuthKey *k = (_GatherAuthKey *)0; - uint64_t *ts = NULL; - Hashtable<_GatherAuthKey,uint64_t>::Iterator i(_gatherAuth); - while (i.next(k,ts)) { - if ((now - *ts) >= ZT_MULTICAST_CREDENTIAL_EXPIRATON) - _gatherAuth.erase(*k); - } - } -} - -void Multicaster::addCredential(void *tPtr,const CertificateOfMembership &com,bool alreadyValidated) -{ - if ((alreadyValidated)||(com.verify(RR,tPtr) == 0)) { - Mutex::Lock _l(_gatherAuth_m); - _gatherAuth[_GatherAuthKey(com.networkId(),com.issuedTo())] = RR->node->now(); - } } void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) @@ -444,15 +424,17 @@ void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup if (member == RR->identity.address()) return; - for(std::vector<MulticastGroupMember>::iterator m(gs.members.begin());m!=gs.members.end();++m) { + std::vector<MulticastGroupMember>::iterator m(std::lower_bound(gs.members.begin(),gs.members.end(),member)); + if (m != gs.members.end()) { if (m->address == member) { m->timestamp = now; return; } + gs.members.insert(m,MulticastGroupMember(member,now)); + } else { + gs.members.push_back(MulticastGroupMember(member,now)); } - gs.members.push_back(MulticastGroupMember(member,now)); - for(std::list<OutboundMulticast>::iterator tx(gs.txQueue.begin());tx!=gs.txQueue.end();) { if (tx->atLimit()) gs.txQueue.erase(tx++); diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index e57f81fe..2cdcfb82 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -158,33 +158,6 @@ public: */ void clean(int64_t now); - /** - * Add an authorization credential - * - * The Multicaster keeps its own track of when valid credentials of network - * membership are presented. This allows it to control MULTICAST_LIKE - * GATHER authorization for networks this node does not belong to. - * - * @param com Certificate of membership - * @param alreadyValidated If true, COM has already been checked and found to be valid and signed - */ - void addCredential(void *tPtr,const CertificateOfMembership &com,bool alreadyValidated); - - /** - * Check authorization for GATHER and LIKE for non-network-members - * - * @param a Address of peer - * @param nwid Network ID - * @param now Current time - * @return True if GATHER and LIKE should be allowed - */ - bool cacheAuthorized(const Address &a,const uint64_t nwid,const int64_t now) const - { - Mutex::Lock _l(_gatherAuth_m); - const uint64_t *p = _gatherAuth.get(_GatherAuthKey(nwid,a)); - return ((p)&&((now - *p) < ZT_MULTICAST_CREDENTIAL_EXPIRATON)); - } - private: struct Key { @@ -203,6 +176,10 @@ private: MulticastGroupMember() {} MulticastGroupMember(const Address &a,uint64_t ts) : address(a),timestamp(ts) {} + inline bool operator<(const Address &a) const { return (address < a); } + inline bool operator==(const Address &a) const { return (address == a); } + inline bool operator!=(const Address &a) const { return (address != a); } + Address address; uint64_t timestamp; // time of last notification }; @@ -222,18 +199,6 @@ private: Hashtable<Multicaster::Key,MulticastGroupStatus> _groups; Mutex _groups_m; - - struct _GatherAuthKey - { - _GatherAuthKey() : member(0),networkId(0) {} - _GatherAuthKey(const uint64_t nwid,const Address &a) : member(a.toInt()),networkId(nwid) {} - inline unsigned long hashCode() const { return (unsigned long)(member ^ networkId); } - inline bool operator==(const _GatherAuthKey &k) const { return ((member == k.member)&&(networkId == k.networkId)); } - uint64_t member; - uint64_t networkId; - }; - Hashtable< _GatherAuthKey,uint64_t > _gatherAuth; - Mutex _gatherAuth_m; }; } // namespace ZeroTier diff --git a/node/Mutex.hpp b/node/Mutex.hpp index a60a00b2..1c8fb81e 100644 --- a/node/Mutex.hpp +++ b/node/Mutex.hpp @@ -1,6 +1,6 @@ /*
* ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/
+ * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@ * GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* --
*
diff --git a/node/Network.cpp b/node/Network.cpp index a75d9fd1..1e151b61 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -29,6 +29,8 @@ #include <stdlib.h> #include <math.h> +#include "../include/ZeroTierDebug.h" + #include "Constants.hpp" #include "../version.h" #include "Network.hpp" @@ -106,7 +108,8 @@ static _doZtFilterResult _doZtFilter( const unsigned int ruleCount, Address &cc, // MUTABLE -- set to TEE destination if TEE action is taken or left alone otherwise unsigned int &ccLength, // MUTABLE -- set to length of packet payload to TEE - bool &ccWatch) // MUTABLE -- set to true for WATCH target as opposed to normal TEE + bool &ccWatch, // MUTABLE -- set to true for WATCH target as opposed to normal TEE + uint8_t &qosBucket) // MUTABLE -- set to the value of the argument provided to PRIORITY { // Set to true if we are a TEE/REDIRECT/WATCH target bool superAccept = false; @@ -124,6 +127,10 @@ static _doZtFilterResult _doZtFilter( if ((unsigned int)rt <= (unsigned int)ZT_NETWORK_RULE_ACTION__MAX_ID) { if (thisSetMatches) { switch(rt) { + case ZT_NETWORK_RULE_ACTION_PRIORITY: + qosBucket = (rules[rn].v.qosBucket >= 0 || rules[rn].v.qosBucket <= 8) ? rules[rn].v.qosBucket : 4; // 4 = default bucket (no priority) + return DOZTFILTER_ACCEPT; + case ZT_NETWORK_RULE_ACTION_DROP: return DOZTFILTER_DROP; @@ -621,9 +628,9 @@ bool Network::filterOutgoingPacket( const uint8_t *frameData, const unsigned int frameLen, const unsigned int etherType, - const unsigned int vlanId) + const unsigned int vlanId, + uint8_t &qosBucket) { - const int64_t now = RR->node->now(); Address ztFinalDest(ztDest); int localCapabilityIndex = -1; int accept = 0; @@ -636,7 +643,7 @@ bool Network::filterOutgoingPacket( Membership *const membership = (ztDest) ? _memberships.get(ztDest) : (Membership *)0; - switch(_doZtFilter(RR,rrl,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { + switch(_doZtFilter(RR,rrl,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch,qosBucket)) { case DOZTFILTER_NO_MATCH: { for(unsigned int c=0;c<_config.capabilityCount;++c) { @@ -644,7 +651,7 @@ bool Network::filterOutgoingPacket( Address cc2; unsigned int ccLength2 = 0; bool ccWatch2 = false; - switch (_doZtFilter(RR,crrl,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),cc2,ccLength2,ccWatch2)) { + switch (_doZtFilter(RR,crrl,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),cc2,ccLength2,ccWatch2,qosBucket)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -656,9 +663,6 @@ bool Network::filterOutgoingPacket( accept = 1; if ((!noTee)&&(cc2)) { - Membership &m2 = _membership(cc2); - m2.pushCredentials(RR,tPtr,now,cc2,_config,localCapabilityIndex,false); - Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)(ccWatch2 ? 0x16 : 0x02)); @@ -693,13 +697,7 @@ bool Network::filterOutgoingPacket( } if (accept) { - if (membership) - membership->pushCredentials(RR,tPtr,now,ztDest,_config,localCapabilityIndex,false); - if ((!noTee)&&(cc)) { - Membership &m2 = _membership(cc); - m2.pushCredentials(RR,tPtr,now,cc,_config,localCapabilityIndex,false); - Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)(ccWatch ? 0x16 : 0x02)); @@ -712,9 +710,6 @@ bool Network::filterOutgoingPacket( } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { - Membership &m2 = _membership(ztFinalDest); - m2.pushCredentials(RR,tPtr,now,ztFinalDest,_config,localCapabilityIndex,false); - Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x04); @@ -759,11 +754,13 @@ int Network::filterIncomingPacket( bool ccWatch = false; const Capability *c = (Capability *)0; + uint8_t qosBucket = 255; // For incoming packets this is a dummy value + Mutex::Lock _l(_lock); Membership &membership = _membership(sourcePeer->address()); - switch (_doZtFilter(RR,rrl,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { + switch (_doZtFilter(RR,rrl,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch,qosBucket)) { case DOZTFILTER_NO_MATCH: { Membership::CapabilityIterator mci(membership,_config); @@ -772,7 +769,7 @@ int Network::filterIncomingPacket( Address cc2; unsigned int ccLength2 = 0; bool ccWatch2 = false; - switch(_doZtFilter(RR,crrl,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),cc2,ccLength2,ccWatch2)) { + switch(_doZtFilter(RR,crrl,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),cc2,ccLength2,ccWatch2,qosBucket)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -787,8 +784,6 @@ int Network::filterIncomingPacket( if (accept) { if (cc2) { - _membership(cc2).pushCredentials(RR,tPtr,RR->node->now(),cc2,_config,-1,false); - Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)(ccWatch2 ? 0x1c : 0x08)); @@ -820,8 +815,6 @@ int Network::filterIncomingPacket( if (accept) { if (cc) { - _membership(cc).pushCredentials(RR,tPtr,RR->node->now(),cc,_config,-1,false); - Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)(ccWatch ? 0x1c : 0x08)); @@ -834,8 +827,6 @@ int Network::filterIncomingPacket( } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { - _membership(ztFinalDest).pushCredentials(RR,tPtr,RR->node->now(),ztFinalDest,_config,-1,false); - Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x0a); @@ -985,7 +976,7 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add return false; c->haveChunkIds[c->haveChunks++] = chunkId; - ZT_FAST_MEMCPY(c->data.unsafeData() + chunkIndex,chunkData,chunkLen); + memcpy(c->data.unsafeData() + chunkIndex,chunkData,chunkLen); c->haveBytes += chunkLen; if (c->haveBytes == totalLength) { @@ -1040,18 +1031,12 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD _portInitialized = true; _externalConfig(&ctmp); - - Address *a = (Address *)0; - Membership *m = (Membership *)0; - Hashtable<Address,Membership>::Iterator i(_memberships); - while (i.next(a,m)) - m->resetPushState(); } _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); if (saveToDisk) { - Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *d = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>(); + Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *const d = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>(); try { if (nconf.toDictionary(*d,false)) { uint64_t tmp[2]; @@ -1257,7 +1242,6 @@ bool Network::gate(void *tPtr,const SharedPtr<Peer> &peer) if (!m) m = &(_membership(peer->address())); if (m->multicastLikeGate(now)) { - m->pushCredentials(RR,tPtr,now,peer->address(),_config,-1,false); _announceMulticastGroupsTo(tPtr,peer->address(),_allMulticastGroups()); } return true; @@ -1354,15 +1338,8 @@ Membership::AddCredentialResult Network::addCredential(void *tPtr,const Certific { if (com.networkId() != _id) return Membership::ADD_REJECTED; - const Address a(com.issuedTo()); Mutex::Lock _l(_lock); - Membership &m = _membership(a); - const Membership::AddCredentialResult result = m.addCredential(RR,tPtr,_config,com); - if ((result == Membership::ADD_ACCEPTED_NEW)||(result == Membership::ADD_ACCEPTED_REDUNDANT)) { - m.pushCredentials(RR,tPtr,RR->node->now(),a,_config,-1,false); - RR->mc->addCredential(tPtr,com,true); - } - return result; + return _membership(com.issuedTo()).addCredential(RR,tPtr,_config,com); } Membership::AddCredentialResult Network::addCredential(void *tPtr,const Address &sentFrom,const Revocation &rev) @@ -1440,7 +1417,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const ec->assignedAddressCount = 0; for(unsigned int i=0;i<ZT_MAX_ZT_ASSIGNED_ADDRESSES;++i) { if (i < _config.staticIpCount) { - ZT_FAST_MEMCPY(&(ec->assignedAddresses[i]),&(_config.staticIps[i]),sizeof(struct sockaddr_storage)); + memcpy(&(ec->assignedAddresses[i]),&(_config.staticIps[i]),sizeof(struct sockaddr_storage)); ++ec->assignedAddressCount; } else { memset(&(ec->assignedAddresses[i]),0,sizeof(struct sockaddr_storage)); @@ -1450,7 +1427,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const ec->routeCount = 0; for(unsigned int i=0;i<ZT_MAX_NETWORK_ROUTES;++i) { if (i < _config.routeCount) { - ZT_FAST_MEMCPY(&(ec->routes[i]),&(_config.routes[i]),sizeof(ZT_VirtualNetworkRoute)); + memcpy(&(ec->routes[i]),&(_config.routes[i]),sizeof(ZT_VirtualNetworkRoute)); ++ec->routeCount; } else { memset(&(ec->routes[i]),0,sizeof(ZT_VirtualNetworkRoute)); @@ -1485,7 +1462,8 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu std::sort(alwaysAnnounceTo.begin(),alwaysAnnounceTo.end()); for(std::vector<Address>::const_iterator a(alwaysAnnounceTo.begin());a!=alwaysAnnounceTo.end();++a) { - // push COM to non-members so they can do multicast request auth + /* + // push COM to non-members so they can do multicast request auth if ( (_config.com) && (!_memberships.contains(*a)) && (*a != RR->identity.address()) ) { Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); _config.com.serialize(outp); @@ -1496,6 +1474,7 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu outp.append((uint16_t)0); // no certificates of ownership RR->sw->send(tPtr,outp,true); } + */ _announceMulticastGroupsTo(tPtr,*a,groups); } } @@ -1505,7 +1484,6 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu Membership *m = (Membership *)0; Hashtable<Address,Membership>::Iterator i(_memberships); while (i.next(a,m)) { - m->pushCredentials(RR,tPtr,now,*a,_config,-1,false); if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config)) && (!std::binary_search(alwaysAnnounceTo.begin(),alwaysAnnounceTo.end(),*a)) ) _announceMulticastGroupsTo(tPtr,*a,groups); } @@ -1515,25 +1493,27 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu void Network::_announceMulticastGroupsTo(void *tPtr,const Address &peer,const std::vector<MulticastGroup> &allMulticastGroups) { // Assumes _lock is locked - Packet outp(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); + Packet *const outp = new Packet(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); for(std::vector<MulticastGroup>::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) { - if ((outp.size() + 24) >= ZT_PROTO_MAX_PACKET_LENGTH) { - outp.compress(); - RR->sw->send(tPtr,outp,true); - outp.reset(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); + if ((outp->size() + 24) >= ZT_PROTO_MAX_PACKET_LENGTH) { + outp->compress(); + RR->sw->send(tPtr,*outp,true); + outp->reset(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); } // network ID, MAC, ADI - outp.append((uint64_t)_id); - mg->mac().appendTo(outp); - outp.append((uint32_t)mg->adi()); + outp->append((uint64_t)_id); + mg->mac().appendTo(*outp); + outp->append((uint32_t)mg->adi()); } - if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) { - outp.compress(); - RR->sw->send(tPtr,outp,true); + if (outp->size() > ZT_PROTO_MIN_PACKET_LENGTH) { + outp->compress(); + RR->sw->send(tPtr,*outp,true); } + + delete outp; } std::vector<MulticastGroup> Network::_allMulticastGroups() const diff --git a/node/Network.hpp b/node/Network.hpp index 95b5483a..969e1f14 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -132,7 +132,8 @@ public: const uint8_t *frameData, const unsigned int frameLen, const unsigned int etherType, - const unsigned int vlanId); + const unsigned int vlanId, + uint8_t &qosBucket); /** * Apply filters to an incoming packet @@ -290,6 +291,11 @@ public: } /** + * @return True if QoS is in effect for this network + */ + inline bool qosEnabled() { return false; } + + /** * Set a bridge route * * @param mac MAC address of destination @@ -359,7 +365,22 @@ public: inline void pushCredentialsNow(void *tPtr,const Address &to,const int64_t now) { Mutex::Lock _l(_lock); - _membership(to).pushCredentials(RR,tPtr,now,to,_config,-1,true); + _membership(to).pushCredentials(RR,tPtr,now,to,_config,-1); + } + + /** + * Push credentials if we haven't done so in a very long time + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param to Destination peer address + * @param now Current time + */ + inline void pushCredentialsIfNeeded(void *tPtr,const Address &to,const int64_t now) + { + Mutex::Lock _l(_lock); + Membership &m = _membership(to); + if (m.shouldPushCredentials(now)) + m.pushCredentials(RR,tPtr,now,to,_config,-1); } /** @@ -438,6 +459,6 @@ private: AtomicCounter __refCount; }; -} // naemspace ZeroTier +} // namespace ZeroTier #endif diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index db051699..20ad2e25 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -200,10 +200,11 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d) { + static const NetworkConfig NIL_NC; Buffer<ZT_NETWORKCONFIG_DICT_CAPACITY> *tmp = new Buffer<ZT_NETWORKCONFIG_DICT_CAPACITY>(); try { - memset(this,0,sizeof(NetworkConfig)); + *this = NIL_NC; // Fields that are always present, new or old this->networkId = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,0); diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 44066c86..ddad52a5 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -221,9 +221,28 @@ namespace ZeroTier { class NetworkConfig { public: - NetworkConfig() { memset(this,0,sizeof(NetworkConfig)); } - NetworkConfig(const NetworkConfig &nc) { ZT_FAST_MEMCPY(this,&nc,sizeof(NetworkConfig)); } - inline NetworkConfig &operator=(const NetworkConfig &nc) { ZT_FAST_MEMCPY(this,&nc,sizeof(NetworkConfig)); return *this; } + NetworkConfig() : + networkId(0), + timestamp(0), + credentialTimeMaxDelta(0), + revision(0), + issuedTo(), + remoteTraceTarget(), + flags(0), + remoteTraceLevel(Trace::LEVEL_NORMAL), + mtu(0), + multicastLimit(0), + specialistCount(0), + routeCount(0), + staticIpCount(0), + ruleCount(0), + capabilityCount(0), + tagCount(0), + certificateOfOwnershipCount(0), + type(ZT_NETWORK_TYPE_PRIVATE) + { + name[0] = 0; + } /** * Write this network config to a dictionary for transport @@ -255,7 +274,18 @@ public: /** * @return True if frames should not be compressed */ - inline bool disableCompression() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_DISABLE_COMPRESSION) != 0); } + inline bool disableCompression() const + { +#ifndef ZT_DISABLE_COMPRESSION + return ((this->flags & ZT_NETWORKCONFIG_FLAG_DISABLE_COMPRESSION) != 0); +#else + /* Compression is disabled for libzt builds since it causes non-obvious chaotic + interference with lwIP's TCP congestion algorithm. Compression is also disabled + for some NAS builds due to the usage of low-performance processors in certain + older and budget models. */ + return false; +#endif + } /** * @return Network type is public (no access control) @@ -562,7 +592,7 @@ public: char name[ZT_MAX_NETWORK_SHORT_NAME_LENGTH + 1]; /** - * Certficiate of membership (for private networks) + * Certificate of membership (for private networks) */ CertificateOfMembership com; }; diff --git a/node/NetworkController.hpp b/node/NetworkController.hpp index 393bcc91..1cbbd3ff 100644 --- a/node/NetworkController.hpp +++ b/node/NetworkController.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Node.cpp b/node/Node.cpp index db511430..237da463 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -66,7 +66,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64 { if (callbacks->version != 0) throw ZT_EXCEPTION_INVALID_ARGUMENT; - ZT_FAST_MEMCPY(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); + memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); // Initialize non-cryptographic PRNG from a good random source Utils::getSecureRandom((void *)_prngState,sizeof(_prngState)); @@ -76,6 +76,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64 memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); + memset((void *)(&_stats),0,sizeof(_stats)); uint64_t idtmp[2]; idtmp[0] = 0; idtmp[1] = 0; @@ -234,7 +235,7 @@ public: } if ((!contacted)&&(_bestCurrentUpstream)) { - const SharedPtr<Path> up(_bestCurrentUpstream->getBestPath(_now,true)); + const SharedPtr<Path> up(_bestCurrentUpstream->getAppropriatePath(_now,true)); if (up) p->sendHELLO(_tPtr,up->localSocket(),up->address(),_now); } @@ -268,6 +269,15 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64 Hashtable< Address,std::vector<InetAddress> > alwaysContact; RR->topology->getUpstreamsToContact(alwaysContact); + // Uncomment to dump stats + /* + for(unsigned int i=0;i<32;i++) { + if (_stats.inVerbCounts[i] > 0) + printf("%.2x\t%12lld %lld\n",i,(unsigned long long)_stats.inVerbCounts[i],(unsigned long long)_stats.inVerbBytes[i]); + } + printf("\n"); + */ + // Check last receive time on designated upstreams to see if we seem to be online int64_t lastReceivedFromUpstream = 0; { @@ -281,6 +291,19 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64 } } + // Clean up any old local controller auth memorizations. + { + _localControllerAuthorizations_m.lock(); + Hashtable< _LocalControllerAuth,int64_t >::Iterator i(_localControllerAuthorizations); + _LocalControllerAuth *k = (_LocalControllerAuth *)0; + int64_t *v = (int64_t *)0; + while (i.next(k,v)) { + if ((*v - now) > (ZT_NETWORK_AUTOCONF_DELAY * 3)) + _localControllerAuthorizations.erase(*k); + } + _localControllerAuthorizations_m.unlock(); + } + // Get peers we should stay connected to according to network configs // Also get networks and whether they need config so we only have to do one pass over networks std::vector< std::pair< SharedPtr<Network>,bool > > networkConfigNeeded; @@ -368,6 +391,7 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) { Mutex::Lock _l(_networks_m); SharedPtr<Network> *nw = _networks.get(nwid); + RR->sw->removeNetworkQoSControlBlock(nwid); if (!nw) return ZT_RESULT_OK; if (uptr) @@ -450,6 +474,7 @@ ZT_PeerList *Node::peers() const for(std::vector< std::pair< Address,SharedPtr<Peer> > >::iterator pi(peers.begin());pi!=peers.end();++pi) { ZT_Peer *p = &(pl->peers[pl->peerCount++]); p->address = pi->second->address().toInt(); + p->hadAggregateLink = 0; if (pi->second->remoteVersionKnown()) { p->versionMajor = pi->second->remoteVersionMajor(); p->versionMinor = pi->second->remoteVersionMinor(); @@ -465,15 +490,27 @@ ZT_PeerList *Node::peers() const p->role = RR->topology->role(pi->second->identity().address()); std::vector< SharedPtr<Path> > paths(pi->second->paths(_now)); - SharedPtr<Path> bestp(pi->second->getBestPath(_now,false)); + SharedPtr<Path> bestp(pi->second->getAppropriatePath(_now,false)); + p->hadAggregateLink |= pi->second->hasAggregateLink(); p->pathCount = 0; for(std::vector< SharedPtr<Path> >::iterator path(paths.begin());path!=paths.end();++path) { - ZT_FAST_MEMCPY(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); + memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); p->paths[p->pathCount].lastSend = (*path)->lastOut(); p->paths[p->pathCount].lastReceive = (*path)->lastIn(); p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); p->paths[p->pathCount].expired = 0; p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0; + p->paths[p->pathCount].latency = (float)(*path)->latency(); + p->paths[p->pathCount].packetDelayVariance = (*path)->packetDelayVariance(); + p->paths[p->pathCount].throughputDisturbCoeff = (*path)->throughputDisturbanceCoefficient(); + p->paths[p->pathCount].packetErrorRatio = (*path)->packetErrorRatio(); + p->paths[p->pathCount].packetLossRatio = (*path)->packetLossRatio(); + p->paths[p->pathCount].stability = (*path)->lastComputedStability(); + p->paths[p->pathCount].throughput = (*path)->meanThroughput(); + p->paths[p->pathCount].maxThroughput = (*path)->maxLifetimeThroughput(); + p->paths[p->pathCount].allocation = (float)(*path)->allocation() / (float)255; + p->paths[p->pathCount].ifname = (*path)->getName(); + ++p->pathCount; } } @@ -619,6 +656,10 @@ std::vector<World> Node::moons() const void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) { + _localControllerAuthorizations_m.lock(); + _localControllerAuthorizations[_LocalControllerAuth(nwid,destination)] = now(); + _localControllerAuthorizations_m.unlock(); + if (destination == RR->identity.address()) { SharedPtr<Network> n(network(nwid)); if (!n) return; diff --git a/node/Node.hpp b/node/Node.hpp index 79284b63..a6f92eb1 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -260,6 +260,25 @@ public: inline const Address &remoteTraceTarget() const { return _remoteTraceTarget; } inline Trace::Level remoteTraceLevel() const { return _remoteTraceLevel; } + inline void setMultipathMode(uint8_t mode) { _multipathMode = mode; } + inline uint8_t getMultipathMode() { return _multipathMode; } + + inline bool localControllerHasAuthorized(const int64_t now,const uint64_t nwid,const Address &addr) const + { + _localControllerAuthorizations_m.lock(); + const int64_t *const at = _localControllerAuthorizations.get(_LocalControllerAuth(nwid,addr)); + _localControllerAuthorizations_m.unlock(); + if (at) + return ((now - *at) < (ZT_NETWORK_AUTOCONF_DELAY * 3)); + return false; + } + + inline void statsLogVerb(const unsigned int v,const unsigned int bytes) + { + ++_stats.inVerbCounts[v]; + _stats.inVerbBytes[v] += (uint64_t)bytes; + } + private: RuntimeEnvironment _RR; RuntimeEnvironment *RR; @@ -273,6 +292,22 @@ private: // Time of last identity verification indexed by InetAddress.rateGateHash() -- used in IncomingPacket::_doHELLO() via rateGateIdentityVerification() int64_t _lastIdentityVerification[16384]; + // Statistics about stuff happening + volatile ZT_NodeStatistics _stats; + + // Map that remembers if we have recently sent a network config to someone + // querying us as a controller. + struct _LocalControllerAuth + { + uint64_t nwid,address; + _LocalControllerAuth(const uint64_t nwid_,const Address &address_) : nwid(nwid_),address(address_.toInt()) {} + inline unsigned long hashCode() const { return (unsigned long)(nwid ^ address); } + inline bool operator==(const _LocalControllerAuth &a) const { return ((a.nwid == nwid)&&(a.address == address)); } + inline bool operator!=(const _LocalControllerAuth &a) const { return ((a.nwid != nwid)||(a.address != address)); } + }; + Hashtable< _LocalControllerAuth,int64_t > _localControllerAuthorizations; + Mutex _localControllerAuthorizations_m; + Hashtable< uint64_t,SharedPtr<Network> > _networks; Mutex _networks_m; @@ -284,6 +319,8 @@ private: Address _remoteTraceTarget; enum Trace::Level _remoteTraceLevel; + uint8_t _multipathMode; + volatile int64_t _now; int64_t _lastPingCheck; int64_t _lastHousekeepingRun; diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index d7a7b4d8..a162c8a5 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -78,20 +78,20 @@ void OutboundMulticast::init( if (!disableCompression) _packet.compress(); - ZT_FAST_MEMCPY(_frameData,payload,_frameLen); + memcpy(_frameData,payload,_frameLen); } void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr) { const SharedPtr<Network> nw(RR->node->network(_nwid)); - const Address toAddr2(toAddr); - if ((nw)&&(nw->filterOutgoingPacket(tPtr,true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { + uint8_t QoSBucket = 255; // Dummy value + if ((nw)&&(nw->filterOutgoingPacket(tPtr,true,RR->identity.address(),toAddr,_macSrc,_macDest,_frameData,_frameLen,_etherType,0,QoSBucket))) { + nw->pushCredentialsIfNeeded(tPtr,toAddr,RR->node->now()); _packet.newInitializationVector(); - _packet.setDestination(toAddr2); + _packet.setDestination(toAddr); RR->node->expectReplyTo(_packet.packetId()); - - Packet tmp(_packet); // make a copy of packet so as not to garble the original -- GitHub issue #461 - RR->sw->send(tPtr,tmp,true); + _tmp = _packet; + RR->sw->send(tPtr,_tmp,true); } } diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index a735f52b..42063995 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -161,7 +161,7 @@ private: unsigned int _limit; unsigned int _frameLen; unsigned int _etherType; - Packet _packet; + Packet _packet,_tmp; std::vector<Address> _alreadySentTo; uint8_t _frameData[ZT_MAX_MTU]; }; diff --git a/node/Packet.cpp b/node/Packet.cpp index c83131ca..5d0885d3 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -42,8 +42,8 @@ #ifdef _MSC_VER #define FORCE_INLINE static __forceinline #include <intrin.h> -#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #else #define FORCE_INLINE static inline #endif @@ -155,16 +155,16 @@ namespace { A library is provided to take care of it, see lz4frame.h. */ -#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) #define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE #define LZ4_QUOTE(str) #str #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) #define LZ4_MEMORY_USAGE 14 -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ @@ -173,7 +173,7 @@ static inline void LZ4_resetStream (LZ4_stream_t* streamPtr); #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ typedef struct { uint32_t hashTable[LZ4_HASH_SIZE_U32]; @@ -192,14 +192,14 @@ typedef struct { } LZ4_streamDecode_t_internal; #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) union LZ4_stream_u { unsigned long long table[LZ4_STREAMSIZE_U64]; LZ4_stream_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_stream_t */ #define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; LZ4_streamDecode_t_internal internal_donotuse; @@ -224,8 +224,8 @@ union LZ4_streamDecode_u { #endif #define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#define MEM_INIT memset +#define FREEMEM free +#define MEM_INIT memset typedef uint8_t BYTE; typedef uint16_t U16; @@ -257,50 +257,50 @@ static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = val #else /* safe and portable access through memcpy() */ static inline U16 LZ4_read16(const void* memPtr) { - U16 val; ZT_FAST_MEMCPY(&val, memPtr, sizeof(val)); return val; + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; } static inline U32 LZ4_read32(const void* memPtr) { - U32 val; ZT_FAST_MEMCPY(&val, memPtr, sizeof(val)); return val; + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; } static inline reg_t LZ4_read_ARCH(const void* memPtr) { - reg_t val; ZT_FAST_MEMCPY(&val, memPtr, sizeof(val)); return val; + reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; } static inline void LZ4_write16(void* memPtr, U16 value) { - ZT_FAST_MEMCPY(memPtr, &value, sizeof(value)); + memcpy(memPtr, &value, sizeof(value)); } static inline void LZ4_write32(void* memPtr, U32 value) { - ZT_FAST_MEMCPY(memPtr, &value, sizeof(value)); + memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ static inline U16 LZ4_readLE16(const void* memPtr) { if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); + return LZ4_read16(memPtr); } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); } } static inline void LZ4_writeLE16(void* memPtr, U16 value) { if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); + LZ4_write16(memPtr, value); } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); } } static inline void LZ4_copy8(void* dst, const void* src) { - ZT_FAST_MEMCPY(dst,src,8); + memcpy(dst,src,8); } static inline void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) @@ -330,63 +330,63 @@ static const int LZ4_minLength = (MFLIMIT+1); #define RUN_BITS (8-ML_BITS) #define RUN_MASK ((1U<<RUN_BITS)-1) -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ static inline unsigned LZ4_NbCommonBytes (reg_t val) { if (LZ4_isLittleEndian()) { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } } else /* Big Endian CPU */ { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } } @@ -396,10 +396,10 @@ static inline unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE const BYTE* const pStart = pIn; while (likely(pIn<pInLimit-(STEPSIZE-1))) { - reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); - if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } - pIn += LZ4_NbCommonBytes(diff); - return (unsigned)(pIn - pStart); + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } + pIn += LZ4_NbCommonBytes(diff); + return (unsigned)(pIn - pStart); } if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } @@ -425,9 +425,9 @@ static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize static inline U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) - return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } static inline U32 LZ4_hash5(U64 sequence, tableType_t const tableType) @@ -436,9 +436,9 @@ static inline U32 LZ4_hash5(U64 sequence, tableType_t const tableType) static const U64 prime8bytes = 11400714785074694791ULL; const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; if (LZ4_isLittleEndian()) - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); else - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) @@ -477,16 +477,16 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy } FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dict, - const dictIssue_directive dictIssue, - const U32 acceleration) + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) { const BYTE* ip = (const BYTE*) source; const BYTE* base; @@ -511,20 +511,20 @@ FORCE_INLINE int LZ4_compress_generic( { case noDict: default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source - cctx->dictSize; + break; case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source; + break; } if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* First Byte */ LZ4_putPosition(ip, cctx->hashTable, tableType, base); @@ -532,143 +532,143 @@ FORCE_INLINE int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - ptrdiff_t refDelta = 0; - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } - - /* Encode Literals */ - { unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; - *token = (RUN_MASK<<ML_BITS); - for(; len >= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength<<ML_BITS); - - /* Copy Literals */ - LZ4_wildCopy(op, anchor, op+litLength); - op+=litLength; - } + ptrdiff_t refDelta = 0; + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; + if (litLength >= RUN_MASK) { + int len = (int)litLength-RUN_MASK; + *token = (RUN_MASK<<ML_BITS); + for(; len >= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<<ML_BITS); + + /* Copy Literals */ + LZ4_wildCopy(op, anchor, op+litLength); + op+=litLength; + } _next_match: - /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; - - /* Encode MatchLength */ - { unsigned matchCode; - - if ((dict==usingExtDict) && (lowLimit==dictionary)) { - const BYTE* limit; - match += refDelta; - limit = ip + (dictEnd-match); - if (limit > matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; - } - } else { - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - } - - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; - op += matchCode / 255; - *op++ = (BYTE)(matchCode % 255); - } else - *token += (BYTE)(matchCode); - } - - anchor = ip; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); + /* Encode Offset */ + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + + /* Encode MatchLength */ + { unsigned matchCode; + + if ((dict==usingExtDict) && (lowLimit==dictionary)) { + const BYTE* limit; + match += refDelta; + limit = ip + (dictEnd-match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchCode; + if (ip==limit) { + unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchCode; + } + + if ( outputLimited && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) + return 0; + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); } _last_literals: /* Encode Last Literals */ { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRun<<ML_BITS); - } - ZT_FAST_MEMCPY(op, anchor, lastRun); - op += lastRun; + if ( (outputLimited) && /* Check output buffer overflow */ + ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) + return 0; + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun<<ML_BITS); + } + memcpy(op, anchor, lastRun); + op += lastRun; } /* End */ @@ -682,15 +682,15 @@ static inline int LZ4_compress_fast_extState(void* state, const char* source, ch //if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); } } @@ -717,19 +717,19 @@ static inline void LZ4_resetStream (LZ4_stream_t* LZ4_stream) } FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) { /* Local Variables */ const BYTE* ip = (const BYTE*) source; @@ -750,125 +750,125 @@ FORCE_INLINE int LZ4_decompress_generic( /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); /* Main Loop : decode sequences */ while (1) { - size_t length; - const BYTE* match; - size_t offset; - - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) ); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ - } - - /* copy literals */ - cpy = op+length; - if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - ZT_FAST_MEMCPY(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - ZT_FAST_MEMCPY(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - ZT_FAST_MEMCPY(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - ZT_FAST_MEMCPY(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; - - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op<cpy) *op++ = *match++; - } else { - LZ4_copy8(op, match); - if (length>16) LZ4_wildCopy(op+8, match+8, cpy); - } - op=cpy; /* correction */ + size_t length; + const BYTE* match; + size_t offset; + + /* get literal length */ + unsigned const token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) { + unsigned s; + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) ); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + } + + /* copy literals */ + cpy = op+length; + if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + if (unlikely(offset<8)) { + const int dec64 = dec64table[offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[offset]; + memcpy(op+4, match, 4); + match -= dec64; + } else { LZ4_copy8(op, match); match+=8; } + op += 8; + + if (unlikely(cpy>oend-12)) { + BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op<cpy) *op++ = *match++; + } else { + LZ4_copy8(op, match); + if (length>16) LZ4_wildCopy(op+8, match+8, cpy); + } + op=cpy; /* correction */ } /* end of decoding */ if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ + return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ else return (int) (((const char*)ip)-source); /* Nb of input bytes read */ @@ -921,7 +921,7 @@ void Packet::armor(const void *key,bool encryptPayload) s20.crypt12(payload,payload,payloadLen); uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,macKey); - ZT_FAST_MEMCPY(data + ZT_PACKET_IDX_MAC,mac,8); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); } } @@ -989,11 +989,11 @@ bool Packet::compress() if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 64))) { // don't bother compressing tiny packets int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress_fast(data + ZT_PACKET_IDX_PAYLOAD,buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); + int cl = LZ4_compress_fast(data + ZT_PACKET_IDX_PAYLOAD,buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,1); if ((cl > 0)&&(cl < pl)) { data[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); - ZT_FAST_MEMCPY(data + ZT_PACKET_IDX_PAYLOAD,buf,cl); + memcpy(data + ZT_PACKET_IDX_PAYLOAD,buf,cl); return true; } } @@ -1013,7 +1013,7 @@ bool Packet::uncompress() int ucl = LZ4_decompress_safe((const char *)data + ZT_PACKET_IDX_PAYLOAD,buf,compLen,sizeof(buf)); if ((ucl > 0)&&(ucl <= (int)(capacity() - ZT_PACKET_IDX_PAYLOAD))) { setSize((unsigned int)ucl + ZT_PACKET_IDX_PAYLOAD); - ZT_FAST_MEMCPY(data + ZT_PACKET_IDX_PAYLOAD,buf,ucl); + memcpy(data + ZT_PACKET_IDX_PAYLOAD,buf,ucl); } else { return false; } diff --git a/node/Packet.hpp b/node/Packet.hpp index 27da6fb5..fcb81489 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -45,31 +45,33 @@ /** * Protocol version -- incremented only for major changes * - * 1 - 0.2.0 ... 0.2.5 - * 2 - 0.3.0 ... 0.4.5 - * + Added signature and originating peer to multicast frame - * + Double size of multicast frame bloom filter - * 3 - 0.5.0 ... 0.6.0 - * + Yet another multicast redesign - * + New crypto completely changes key agreement cipher - * 4 - 0.6.0 ... 1.0.6 - * + BREAKING CHANGE: New identity format based on hashcash design - * 5 - 1.1.0 ... 1.1.5 - * + Supports echo - * + Supports in-band world (root server definition) updates - * + Clustering! (Though this will work with protocol v4 clients.) - * + Otherwise backward compatible with protocol v4 - * 6 - 1.1.5 ... 1.1.10 - * + Network configuration format revisions including binary values - * 7 - 1.1.10 ... 1.1.17 - * + Introduce trusted paths for local SDN use - * 8 - 1.1.17 ... 1.2.0 - * + Multipart network configurations for large network configs - * + Tags and Capabilities - * + Inline push of CertificateOfMembership deprecated - * 9 - 1.2.0 ... CURRENT + * 1 - 0.2.0 ... 0.2.5 + * 2 - 0.3.0 ... 0.4.5 + * + Added signature and originating peer to multicast frame + * + Double size of multicast frame bloom filter + * 3 - 0.5.0 ... 0.6.0 + * + Yet another multicast redesign + * + New crypto completely changes key agreement cipher + * 4 - 0.6.0 ... 1.0.6 + * + BREAKING CHANGE: New identity format based on hashcash design + * 5 - 1.1.0 ... 1.1.5 + * + Supports echo + * + Supports in-band world (root server definition) updates + * + Clustering! (Though this will work with protocol v4 clients.) + * + Otherwise backward compatible with protocol v4 + * 6 - 1.1.5 ... 1.1.10 + * + Network configuration format revisions including binary values + * 7 - 1.1.10 ... 1.1.17 + * + Introduce trusted paths for local SDN use + * 8 - 1.1.17 ... 1.2.0 + * + Multipart network configurations for large network configs + * + Tags and Capabilities + * + Inline push of CertificateOfMembership deprecated + * 9 - 1.2.0 ... 1.2.14 + * 10 - 1.4.0 ... CURRENT + * + Multipath capability and load balancing */ -#define ZT_PROTO_VERSION 9 +#define ZT_PROTO_VERSION 10 /** * Minimum supported protocol version @@ -150,7 +152,7 @@ * * In cryptography, a "break" means something different from what it means in * common discussion. If a cipher is 256 bits strong and someone finds a way - * to reduce key search to 254 bits, this constitues a "break" in the academic + * to reduce key search to 254 bits, this constitutes a "break" in the academic * literature. 254 bits is still far beyond what can be leveraged to accomplish * a "break" as most people would understand it -- the actual decryption and * reading of traffic. @@ -249,7 +251,7 @@ */ #define ZT_PROTO_MIN_FRAGMENT_LENGTH ZT_PACKET_FRAGMENT_IDX_PAYLOAD -// Field incides for parsing verbs ------------------------------------------- +// Field indices for parsing verbs ------------------------------------------- // Some verbs have variable-length fields. Those aren't fully defined here // yet-- instead they are parsed using relative indexes in IncomingPacket. @@ -419,7 +421,7 @@ public: template<unsigned int C2> Fragment(const Buffer<C2> &b) : - Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b) + Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b) { } @@ -734,7 +736,7 @@ public: * Credentials can be for any number of networks. * * The use of a zero byte to terminate the COM section is for legacy - * backward compatiblity. Newer fields are prefixed with a length. + * backward compatibility. Newer fields are prefixed with a length. * * OK/ERROR are not generated. */ @@ -751,11 +753,11 @@ public: * This message requests network configuration from a node capable of * providing it. * - * Respones to this are always whole configs intended for the recipient. + * Responses to this are always whole configs intended for the recipient. * For patches and other updates a NETWORK_CONFIG is sent instead. * * It would be valid and correct as of 1.2.0 to use NETWORK_CONFIG always, - * but OK(NTEWORK_CONFIG_REQUEST) should be sent for compatibility. + * but OK(NETWORK_CONFIG_REQUEST) should be sent for compatibility. * * OK response payload: * <[8] 64-bit network ID> @@ -884,7 +886,7 @@ public: * <[6] MAC address of multicast group> * <[4] 32-bit ADI for multicast group> * <[1] flags> - * [<[...] network certficate of membership (DEPRECATED)>] + * [<[...] network certificate of membership (DEPRECATED)>] * [<[...] implicit gather results if flag 0x01 is set>] * * OK flags (same bits as request flags): @@ -930,7 +932,53 @@ public: */ VERB_PUSH_DIRECT_PATHS = 0x10, - // 0x11, 0x12 -- deprecated + // 0x11 -- deprecated + + /** + * An acknowledgment of receipt of a series of recent packets from another + * peer. This is used to calculate relative throughput values and to detect + * packet loss. Only VERB_FRAME and VERB_EXT_FRAME packets are counted. + * + * ACK response format: + * <[4] 32-bit number of bytes received since last ACK> + * + * Upon receipt of this packet, the local peer will verify that the correct + * number of bytes were received by the remote peer. If these values do + * not agree that could be an indicator of packet loss. + * + * Additionally, the local peer knows the interval of time that has + * elapsed since the last received ACK. With this information it can compute + * a rough estimate of the current throughput. + * + * This is sent at a maximum rate of once per every ZT_PATH_ACK_INTERVAL + */ + VERB_ACK = 0x12, + + /** + * A packet containing timing measurements useful for estimating path quality. + * Composed of a list of <packet ID:internal sojourn time> pairs for an + * arbitrary set of recent packets. This is used to sample for latency and + * packet delay variance (PDV, "jitter"). + * + * QoS record format: + * + * <[8] 64-bit packet ID of previously-received packet> + * <[1] 8-bit packet sojourn time> + * <...repeat until end of max 1400 byte packet...> + * + * The number of possible records per QoS packet is: (1400 * 8) / 72 = 155 + * This packet should be sent very rarely (every few seconds) as it can be + * somewhat large if the connection is saturated. Future versions might use + * a bloom table to probabilistically determine these values in a vastly + * more space-efficient manner. + * + * Note: The 'internal packet sojourn time' is a slight misnomer as it is a + * measure of the amount of time between when a packet was received and the + * egress time of its tracking QoS packet. + * + * This is sent at a maximum rate of once per every ZT_PATH_QOS_INTERVAL + */ + VERB_QOS_MEASUREMENT = 0x13, /** * A message with arbitrary user-definable content: @@ -954,7 +1002,7 @@ public: * * This message contains a remote trace event. Remote trace events can * be sent to observers configured at the network level for those that - * pertain directly to actiity on a network, or to global observers if + * pertain directly to activity on a network, or to global observers if * locally configured. * * The instance ID is a random 64-bit value generated by each ZeroTier @@ -999,7 +1047,7 @@ public: template<unsigned int C2> Packet(const Buffer<C2> &b) : - Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b) + Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(b) { } @@ -1251,7 +1299,7 @@ public: * Encrypt/decrypt a separately armored portion of a packet * * This is currently only used to mask portions of HELLO as an extra - * security precation since most of that message is sent in the clear. + * security precaution since most of that message is sent in the clear. * * This must NEVER be used more than once in the same packet, as doing * so will result in re-use of the same key stream. diff --git a/node/Path.cpp b/node/Path.cpp index b1b3dd06..2adb351c 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Path.hpp b/node/Path.hpp index e12328ff..ed3e907f 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -39,6 +39,10 @@ #include "SharedPtr.hpp" #include "AtomicCounter.hpp" #include "Utils.hpp" +#include "RingBuffer.hpp" +#include "Packet.hpp" + +#include "../osdep/Phy.hpp" /** * Maximum return value of preferenceRank() @@ -55,6 +59,7 @@ class RuntimeEnvironment; class Path { friend class SharedPtr<Path>; + Phy<Path *> *_phy; public: /** @@ -72,10 +77,10 @@ public: _k[1] = (uint64_t)reinterpret_cast<const struct sockaddr_in *>(&r)->sin_port; _k[2] = (uint64_t)l; } else if (r.ss_family == AF_INET6) { - ZT_FAST_MEMCPY(_k,reinterpret_cast<const struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,16); + memcpy(_k,reinterpret_cast<const struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,16); _k[2] = ((uint64_t)reinterpret_cast<const struct sockaddr_in6 *>(&r)->sin6_port << 32) ^ (uint64_t)l; } else { - ZT_FAST_MEMCPY(_k,&r,std::min(sizeof(_k),sizeof(InetAddress))); + memcpy(_k,&r,std::min(sizeof(_k),sizeof(InetAddress))); _k[2] += (uint64_t)l; } } @@ -93,22 +98,69 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _lastPathQualityComputeTime(0), _localSocket(-1), _latency(0xffff), _addr(), - _ipScope(InetAddress::IP_SCOPE_NONE) + _ipScope(InetAddress::IP_SCOPE_NONE), + _lastAck(0), + _lastThroughputEstimation(0), + _lastQoSMeasurement(0), + _lastQoSRecordPurge(0), + _unackedBytes(0), + _expectingAckAsOf(0), + _packetsReceivedSinceLastAck(0), + _packetsReceivedSinceLastQoS(0), + _maxLifetimeThroughput(0), + _lastComputedMeanThroughput(0), + _bytesAckedSinceLastThroughputEstimation(0), + _lastComputedMeanLatency(0.0), + _lastComputedPacketDelayVariance(0.0), + _lastComputedPacketErrorRatio(0.0), + _lastComputedPacketLossRatio(0), + _lastComputedStability(0.0), + _lastComputedRelativeQuality(0), + _lastComputedThroughputDistCoeff(0.0), + _lastAllocation(0) { + memset(_ifname, 0, 16); + memset(_addrString, 0, sizeof(_addrString)); } Path(const int64_t localSocket,const InetAddress &addr) : _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _lastPathQualityComputeTime(0), _localSocket(localSocket), _latency(0xffff), _addr(addr), - _ipScope(addr.ipScope()) + _ipScope(addr.ipScope()), + _lastAck(0), + _lastThroughputEstimation(0), + _lastQoSMeasurement(0), + _lastQoSRecordPurge(0), + _unackedBytes(0), + _expectingAckAsOf(0), + _packetsReceivedSinceLastAck(0), + _packetsReceivedSinceLastQoS(0), + _maxLifetimeThroughput(0), + _lastComputedMeanThroughput(0), + _bytesAckedSinceLastThroughputEstimation(0), + _lastComputedMeanLatency(0.0), + _lastComputedPacketDelayVariance(0.0), + _lastComputedPacketErrorRatio(0.0), + _lastComputedPacketLossRatio(0), + _lastComputedStability(0.0), + _lastComputedRelativeQuality(0), + _lastComputedThroughputDistCoeff(0.0), + _lastAllocation(0) { + memset(_ifname, 0, 16); + memset(_addrString, 0, sizeof(_addrString)); + if (_localSocket != -1) { + _phy->getIfName((PhySocket *) ((uintptr_t) _localSocket), _ifname, 16); + } } /** @@ -147,12 +199,16 @@ public: * * @param l Measured latency */ - inline void updateLatency(const unsigned int l) + inline void updateLatency(const unsigned int l, int64_t now) { unsigned int pl = _latency; - if (pl < 0xffff) + if (pl < 0xffff) { _latency = (pl + l) / 2; - else _latency = l; + } + else { + _latency = l; + } + _latencySamples.push(l); } /** @@ -241,6 +297,332 @@ public: } /** + * Record statistics on outgoing packets. Used later to estimate QoS metrics. + * + * @param now Current time + * @param packetId ID of packet + * @param payloadLength Length of payload + * @param verb Packet verb + */ + inline void recordOutgoingPacket(int64_t now, int64_t packetId, uint16_t payloadLength, Packet::Verb verb) + { + Mutex::Lock _l(_statistics_m); + if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) { + if ((packetId & (ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR - 1)) == 0) { + _unackedBytes += payloadLength; + // Take note that we're expecting a VERB_ACK on this path as of a specific time + _expectingAckAsOf = ackAge(now) > ZT_PATH_ACK_INTERVAL ? _expectingAckAsOf : now; + if (_outQoSRecords.size() < ZT_PATH_MAX_OUTSTANDING_QOS_RECORDS) { + _outQoSRecords[packetId] = now; + } + } + } + } + + /** + * Record statistics on incoming packets. Used later to estimate QoS metrics. + * + * @param now Current time + * @param packetId ID of packet + * @param payloadLength Length of payload + * @param verb Packet verb + */ + inline void recordIncomingPacket(int64_t now, int64_t packetId, uint16_t payloadLength, Packet::Verb verb) + { + Mutex::Lock _l(_statistics_m); + if (verb != Packet::VERB_ACK && verb != Packet::VERB_QOS_MEASUREMENT) { + if ((packetId & (ZT_PATH_QOS_ACK_PROTOCOL_DIVISOR - 1)) == 0) { + _inACKRecords[packetId] = payloadLength; + _packetsReceivedSinceLastAck++; + _inQoSRecords[packetId] = now; + _packetsReceivedSinceLastQoS++; + } + _packetValiditySamples.push(true); + } + } + + /** + * Record that we've received a VERB_ACK on this path, also compute throughput if required. + * + * @param now Current time + * @param ackedBytes Number of bytes acknowledged by other peer + */ + inline void receivedAck(int64_t now, int32_t ackedBytes) + { + _expectingAckAsOf = 0; + _unackedBytes = (ackedBytes > _unackedBytes) ? 0 : _unackedBytes - ackedBytes; + int64_t timeSinceThroughputEstimate = (now - _lastThroughputEstimation); + if (timeSinceThroughputEstimate >= ZT_PATH_THROUGHPUT_MEASUREMENT_INTERVAL) { + uint64_t throughput = (uint64_t)((float)(_bytesAckedSinceLastThroughputEstimation * 8) / ((float)timeSinceThroughputEstimate / (float)1000)); + _throughputSamples.push(throughput); + _maxLifetimeThroughput = throughput > _maxLifetimeThroughput ? throughput : _maxLifetimeThroughput; + _lastThroughputEstimation = now; + _bytesAckedSinceLastThroughputEstimation = 0; + } else { + _bytesAckedSinceLastThroughputEstimation += ackedBytes; + } + } + + /** + * @return Number of bytes this peer is responsible for ACKing since last ACK + */ + inline int32_t bytesToAck() + { + Mutex::Lock _l(_statistics_m); + int32_t bytesToAck = 0; + std::map<uint64_t,uint16_t>::iterator it = _inACKRecords.begin(); + while (it != _inACKRecords.end()) { + bytesToAck += it->second; + it++; + } + return bytesToAck; + } + + /** + * @return Number of bytes thus far sent that have not been acknowledged by the remote peer + */ + inline int64_t unackedSentBytes() + { + return _unackedBytes; + } + + /** + * Account for the fact that an ACK was just sent. Reset counters, timers, and clear statistics buffers + * + * @param Current time + */ + inline void sentAck(int64_t now) + { + Mutex::Lock _l(_statistics_m); + _inACKRecords.clear(); + _packetsReceivedSinceLastAck = 0; + _lastAck = now; + } + + /** + * Receive QoS data, match with recorded egress times from this peer, compute latency + * estimates. + * + * @param now Current time + * @param count Number of records + * @param rx_id table of packet IDs + * @param rx_ts table of holding times + */ + inline void receivedQoS(int64_t now, int count, uint64_t *rx_id, uint16_t *rx_ts) + { + Mutex::Lock _l(_statistics_m); + // Look up egress times and compute latency values for each record + std::map<uint64_t,uint64_t>::iterator it; + for (int j=0; j<count; j++) { + it = _outQoSRecords.find(rx_id[j]); + if (it != _outQoSRecords.end()) { + uint16_t rtt = (uint16_t)(now - it->second); + uint16_t rtt_compensated = rtt - rx_ts[j]; + uint16_t latency = rtt_compensated / 2; + updateLatency(latency, now); + _outQoSRecords.erase(it); + } + } + } + + /** + * Generate the contents of a VERB_QOS_MEASUREMENT packet. + * + * @param now Current time + * @param qosBuffer destination buffer + * @return Size of payload + */ + inline int32_t generateQoSPacket(int64_t now, char *qosBuffer) + { + Mutex::Lock _l(_statistics_m); + int32_t len = 0; + std::map<uint64_t,uint64_t>::iterator it = _inQoSRecords.begin(); + int i=0; + while (i<_packetsReceivedSinceLastQoS && it != _inQoSRecords.end()) { + uint64_t id = it->first; + memcpy(qosBuffer, &id, sizeof(uint64_t)); + qosBuffer+=sizeof(uint64_t); + uint16_t holdingTime = (uint16_t)(now - it->second); + memcpy(qosBuffer, &holdingTime, sizeof(uint16_t)); + qosBuffer+=sizeof(uint16_t); + len+=sizeof(uint64_t)+sizeof(uint16_t); + _inQoSRecords.erase(it++); + i++; + } + return len; + } + + /** + * Account for the fact that a VERB_QOS_MEASUREMENT was just sent. Reset timers. + * + * @param Current time + */ + inline void sentQoS(int64_t now) { + _packetsReceivedSinceLastQoS = 0; + _lastQoSMeasurement = now; + } + + /** + * @param now Current time + * @return Whether an ACK (VERB_ACK) packet needs to be emitted at this time + */ + inline bool needsToSendAck(int64_t now) { + return ((now - _lastAck) >= ZT_PATH_ACK_INTERVAL || + (_packetsReceivedSinceLastAck == ZT_PATH_QOS_TABLE_SIZE)) && _packetsReceivedSinceLastAck; + } + + /** + * @param now Current time + * @return Whether a QoS (VERB_QOS_MEASUREMENT) packet needs to be emitted at this time + */ + inline bool needsToSendQoS(int64_t now) { + return ((_packetsReceivedSinceLastQoS >= ZT_PATH_QOS_TABLE_SIZE) || + ((now - _lastQoSMeasurement) > ZT_PATH_QOS_INTERVAL)) && _packetsReceivedSinceLastQoS; + } + + /** + * How much time has elapsed since we've been expecting a VERB_ACK on this path. This value + * is used to determine a more relevant path "age". This lets us penalize paths which are no + * longer ACKing, but not those that simple aren't being used to carry traffic at the + * current time. + */ + inline int64_t ackAge(int64_t now) { return _expectingAckAsOf ? now - _expectingAckAsOf : 0; } + + /** + * The maximum observed throughput (in bits/s) for this path + */ + inline uint64_t maxLifetimeThroughput() { return _maxLifetimeThroughput; } + + /** + * @return The mean throughput (in bits/s) of this link + */ + inline uint64_t meanThroughput() { return _lastComputedMeanThroughput; } + + /** + * Assign a new relative quality value for this path in the aggregate link + * + * @param rq Quality of this path in comparison to other paths available to this peer + */ + inline void updateRelativeQuality(float rq) { _lastComputedRelativeQuality = rq; } + + /** + * @return Quality of this path compared to others in the aggregate link + */ + inline float relativeQuality() { return _lastComputedRelativeQuality; } + + /** + * Assign a new allocation value for this path in the aggregate link + * + * @param allocation Percentage of traffic to be sent over this path to a peer + */ + inline void updateComponentAllocationOfAggregateLink(unsigned char allocation) { _lastAllocation = allocation; } + + /** + * @return Percentage of traffic allocated to this path in the aggregate link + */ + inline unsigned char allocation() { return _lastAllocation; } + + /** + * @return Stability estimates can become expensive to compute, we cache the most recent result. + */ + inline float lastComputedStability() { return _lastComputedStability; } + + /** + * @return A pointer to a cached copy of the human-readable name of the interface this Path's localSocket is bound to + */ + inline char *getName() { return _ifname; } + + /** + * @return Packet delay variance + */ + inline float packetDelayVariance() { return _lastComputedPacketDelayVariance; } + + /** + * @return Previously-computed mean latency + */ + inline float meanLatency() { return _lastComputedMeanLatency; } + + /** + * @return Packet loss rate (PLR) + */ + inline float packetLossRatio() { return _lastComputedPacketLossRatio; } + + /** + * @return Packet error ratio (PER) + */ + inline float packetErrorRatio() { return _lastComputedPacketErrorRatio; } + + /** + * Record an invalid incoming packet. This packet failed MAC/compression/cipher checks and will now + * contribute to a Packet Error Ratio (PER). + */ + inline void recordInvalidPacket() { _packetValiditySamples.push(false); } + + /** + * @return A pointer to a cached copy of the address string for this Path (For debugging only) + */ + inline char *getAddressString() { return _addrString; } + + /** + * @return The current throughput disturbance coefficient + */ + inline float throughputDisturbanceCoefficient() { return _lastComputedThroughputDistCoeff; } + + /** + * Compute and cache stability and performance metrics. The resultant stability coefficient is a measure of how "well behaved" + * this path is. This figure is substantially different from (but required for the estimation of the path's overall "quality". + * + * @param now Current time + */ + inline void processBackgroundPathMeasurements(const int64_t now) + { + if (now - _lastPathQualityComputeTime > ZT_PATH_QUALITY_COMPUTE_INTERVAL) { + Mutex::Lock _l(_statistics_m); + _lastPathQualityComputeTime = now; + address().toString(_addrString); + _lastComputedMeanLatency = _latencySamples.mean(); + _lastComputedPacketDelayVariance = _latencySamples.stddev(); // Similar to "jitter" (SEE: RFC 3393, RFC 4689) + _lastComputedMeanThroughput = (uint64_t)_throughputSamples.mean(); + + // If no packet validity samples, assume PER==0 + _lastComputedPacketErrorRatio = 1 - (_packetValiditySamples.count() ? _packetValiditySamples.mean() : 1); + + // Compute path stability + // Normalize measurements with wildly different ranges into a reasonable range + float normalized_pdv = Utils::normalize(_lastComputedPacketDelayVariance, 0, ZT_PATH_MAX_PDV, 0, 10); + float normalized_la = Utils::normalize(_lastComputedMeanLatency, 0, ZT_PATH_MAX_MEAN_LATENCY, 0, 10); + float throughput_cv = _throughputSamples.mean() > 0 ? _throughputSamples.stddev() / _throughputSamples.mean() : 1; + + // Form an exponential cutoff and apply contribution weights + float pdv_contrib = expf((-1.0f)*normalized_pdv) * (float)ZT_PATH_CONTRIB_PDV; + float latency_contrib = expf((-1.0f)*normalized_la) * (float)ZT_PATH_CONTRIB_LATENCY; + + // Throughput Disturbance Coefficient + float throughput_disturbance_contrib = expf((-1.0f)*throughput_cv) * (float)ZT_PATH_CONTRIB_THROUGHPUT_DISTURBANCE; + _throughputDisturbanceSamples.push(throughput_cv); + _lastComputedThroughputDistCoeff = _throughputDisturbanceSamples.mean(); + + // Obey user-defined ignored contributions + pdv_contrib = ZT_PATH_CONTRIB_PDV > 0.0 ? pdv_contrib : 1; + latency_contrib = ZT_PATH_CONTRIB_LATENCY > 0.0 ? latency_contrib : 1; + throughput_disturbance_contrib = ZT_PATH_CONTRIB_THROUGHPUT_DISTURBANCE > 0.0 ? throughput_disturbance_contrib : 1; + + // Stability + _lastComputedStability = pdv_contrib + latency_contrib + throughput_disturbance_contrib; + _lastComputedStability *= 1 - _lastComputedPacketErrorRatio; + + // Prevent QoS records from sticking around for too long + std::map<uint64_t,uint64_t>::iterator it = _outQoSRecords.begin(); + while (it != _outQoSRecords.end()) { + // Time since egress of tracked packet + if ((now - it->second) >= ZT_PATH_QOS_TIMEOUT) { + _outQoSRecords.erase(it++); + } else { it++; } + } + } + } + + /** * @return True if this path is alive (receiving heartbeats) */ inline bool alive(const int64_t now) const { return ((now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000)); } @@ -266,14 +648,56 @@ public: inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } private: + Mutex _statistics_m; + volatile int64_t _lastOut; volatile int64_t _lastIn; volatile int64_t _lastTrustEstablishedPacketReceived; + volatile int64_t _lastPathQualityComputeTime; int64_t _localSocket; volatile unsigned int _latency; InetAddress _addr; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often AtomicCounter __refCount; + + std::map<uint64_t,uint64_t> _outQoSRecords; // id:egress_time + std::map<uint64_t,uint64_t> _inQoSRecords; // id:now + std::map<uint64_t,uint16_t> _inACKRecords; // id:len + + int64_t _lastAck; + int64_t _lastThroughputEstimation; + int64_t _lastQoSMeasurement; + int64_t _lastQoSRecordPurge; + + int64_t _unackedBytes; + int64_t _expectingAckAsOf; + int16_t _packetsReceivedSinceLastAck; + int16_t _packetsReceivedSinceLastQoS; + + uint64_t _maxLifetimeThroughput; + uint64_t _lastComputedMeanThroughput; + uint64_t _bytesAckedSinceLastThroughputEstimation; + + float _lastComputedMeanLatency; + float _lastComputedPacketDelayVariance; + + float _lastComputedPacketErrorRatio; + float _lastComputedPacketLossRatio; + + // cached estimates + float _lastComputedStability; + float _lastComputedRelativeQuality; + float _lastComputedThroughputDistCoeff; + unsigned char _lastAllocation; + + // cached human-readable strings for tracing purposes + char _ifname[16]; + char _addrString[256]; + + RingBuffer<uint64_t,ZT_PATH_QUALITY_METRIC_WIN_SZ> _throughputSamples; + RingBuffer<uint32_t,ZT_PATH_QUALITY_METRIC_WIN_SZ> _latencySamples; + RingBuffer<bool,ZT_PATH_QUALITY_METRIC_WIN_SZ> _packetValiditySamples; + RingBuffer<float,ZT_PATH_QUALITY_METRIC_WIN_SZ> _throughputDisturbanceSamples; }; } // namespace ZeroTier diff --git a/node/Peer.cpp b/node/Peer.cpp index 71afd852..5f5d1462 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -25,7 +25,6 @@ */ #include "../version.h" - #include "Constants.hpp" #include "Peer.hpp" #include "Node.hpp" @@ -35,9 +34,13 @@ #include "Packet.hpp" #include "Trace.hpp" #include "InetAddress.hpp" +#include "RingBuffer.hpp" +#include "Utils.hpp" namespace ZeroTier { +static unsigned char s_freeRandomByteCounter = 0; + Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : RR(renv), _lastReceive(0), @@ -48,18 +51,29 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastCredentialRequestSent(0), _lastWhoisRequestReceived(0), _lastEchoRequestReceived(0), - _lastComRequestReceived(0), - _lastComRequestSent(0), _lastCredentialsReceived(0), _lastTrustEstablishedPacketReceived(0), _lastSentFullHello(0), + _lastACKWindowReset(0), + _lastQoSWindowReset(0), + _lastMultipathCompatibilityCheck(0), + _freeRandomByte((unsigned char)((uintptr_t)this >> 4) ^ ++s_freeRandomByteCounter), + _uniqueAlivePathCount(0), + _localMultipathSupported(false), + _remoteMultipathSupported(false), + _canUseMultipath(false), _vProto(0), _vMajor(0), _vMinor(0), _vRevision(0), _id(peerIdentity), _directPathPushCutoffCount(0), - _credentialsCutoffCount(0) + _credentialsCutoffCount(0), + _linkIsBalanced(false), + _linkIsRedundant(false), + _remotePeerMultipathEnabled(false), + _lastAggregateStatsReport(0), + _lastAggregateAllocation(0) { if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) throw ZT_EXCEPTION_INVALID_ARGUMENT; @@ -70,6 +84,7 @@ void Peer::received( const SharedPtr<Path> &path, const unsigned int hops, const uint64_t packetId, + const unsigned int payloadLength, const Packet::Verb verb, const uint64_t inRePacketId, const Packet::Verb inReVerb, @@ -87,7 +102,8 @@ void Peer::received( case Packet::VERB_MULTICAST_FRAME: _lastNontrivialReceive = now; break; - default: break; + default: + break; } if (trustEstablished) { @@ -95,9 +111,25 @@ void Peer::received( path->trustedPacketReceived(now); } + { + Mutex::Lock _l(_paths_m); + + recordIncomingPacket(tPtr, path, packetId, payloadLength, verb, now); + + if (_canUseMultipath) { + if (path->needsToSendQoS(now)) { + sendQOS_MEASUREMENT(tPtr, path, path->localSocket(), path->address(), now); + } + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + _paths[i].p->processBackgroundPathMeasurements(now); + } + } + } + } + if (hops == 0) { // If this is a direct packet (no hops), update existing paths or learn new ones - bool havePath = false; { Mutex::Lock _l(_paths_m); @@ -116,20 +148,27 @@ void Peer::received( if ((!havePath)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) { Mutex::Lock _l(_paths_m); - // Paths are redunant if they duplicate an alive path to the same IP or + // Paths are redundant if they duplicate an alive path to the same IP or // with the same local socket and address family. bool redundant = false; + unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS; for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { if (_paths[i].p) { - if ( (_paths[i].p->alive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual2(path->address())) ) ) { + if ( (_paths[i].p->alive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual2(path->address())) ) ) { redundant = true; break; } + // If the path is the same address and port, simply assume this is a replacement + if ( (_paths[i].p->address().ipsEqual2(path->address()))) { + replacePath = i; + break; + } } else break; } - if (!redundant) { - unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS; + // If the path isn't a duplicate of the same localSocket AND we haven't already determined a replacePath, + // then find the worst path and replace it. + if (!redundant && replacePath == ZT_MAX_PEER_NETWORK_PATHS) { int replacePathQuality = 0; for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { if (_paths[i].p) { @@ -143,16 +182,16 @@ void Peer::received( break; } } + } - if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) { - if (verb == Packet::VERB_OK) { - RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId); - _paths[replacePath].lr = now; - _paths[replacePath].p = path; - _paths[replacePath].priority = 1; - } else { - attemptToContact = true; - } + if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) { + if (verb == Packet::VERB_OK) { + RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId); + _paths[replacePath].lr = now; + _paths[replacePath].p = path; + _paths[replacePath].priority = 1; + } else { + attemptToContact = true; } } } @@ -165,39 +204,20 @@ void Peer::received( } // If we have a trust relationship periodically push a message enumerating - // all known external addresses for ourselves. We now do this even if we - // have a current path since we'll want to use new ones too. + // all known external addresses for ourselves. If we already have a path this + // is done less frequently. if (this->trustEstablished(now)) { - if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) { + const int64_t sinceLastPush = now - _lastDirectPathPushSent; + if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) { _lastDirectPathPushSent = now; - - std::vector<InetAddress> pathsToPush; - - std::vector<InetAddress> dps(RR->node->directPaths()); - for(std::vector<InetAddress>::const_iterator i(dps.begin());i!=dps.end();++i) - pathsToPush.push_back(*i); - - // Do symmetric NAT prediction if we are communicating indirectly. - if (hops > 0) { - std::vector<InetAddress> sym(RR->sa->getSymmetricNatPredictions()); - for(unsigned long i=0,added=0;i<sym.size();++i) { - InetAddress tmp(sym[(unsigned long)RR->node->prng() % sym.size()]); - if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { - pathsToPush.push_back(tmp); - if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) - break; - } - } - } - + std::vector<InetAddress> pathsToPush(RR->node->directPaths()); if (pathsToPush.size() > 0) { std::vector<InetAddress>::const_iterator p(pathsToPush.begin()); while (p != pathsToPush.end()) { - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); - outp.addSize(2); // leave room for count - + Packet *const outp = new Packet(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); + outp->addSize(2); // leave room for count unsigned int count = 0; - while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) { + while ((p != pathsToPush.end())&&((outp->size() + 24) < 1200)) { uint8_t addressType = 4; switch(p->ss_family) { case AF_INET: @@ -210,51 +230,287 @@ void Peer::received( continue; } - outp.append((uint8_t)0); // no flags - outp.append((uint16_t)0); // no extensions - outp.append(addressType); - outp.append((uint8_t)((addressType == 4) ? 6 : 18)); - outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16)); - outp.append((uint16_t)p->port()); + outp->append((uint8_t)0); // no flags + outp->append((uint16_t)0); // no extensions + outp->append(addressType); + outp->append((uint8_t)((addressType == 4) ? 6 : 18)); + outp->append(p->rawIpData(),((addressType == 4) ? 4 : 16)); + outp->append((uint16_t)p->port()); ++count; ++p; } - if (count) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); - outp.armor(_key,true); - path->send(RR,tPtr,outp.data(),outp.size(),now); + outp->setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); + outp->compress(); + outp->armor(_key,true); + path->send(RR,tPtr,outp->data(),outp->size(),now); } + delete outp; } } } } } -SharedPtr<Path> Peer::getBestPath(int64_t now,bool includeExpired) const +void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId, + uint16_t payloadLength, const Packet::Verb verb, int64_t now) { - Mutex::Lock _l(_paths_m); + _freeRandomByte += (unsigned char)(packetId >> 8); // grab entropy to use in path selection logic for multipath + if (_canUseMultipath) { + path->recordOutgoingPacket(now, packetId, payloadLength, verb); + } +} + +void Peer::recordIncomingPacket(void *tPtr, const SharedPtr<Path> &path, const uint64_t packetId, + uint16_t payloadLength, const Packet::Verb verb, int64_t now) +{ + if (_canUseMultipath) { + if (path->needsToSendAck(now)) { + sendACK(tPtr, path, path->localSocket(), path->address(), now); + } + path->recordIncomingPacket(now, packetId, payloadLength, verb); + } +} + +void Peer::computeAggregateProportionalAllocation(int64_t now) +{ + float maxStability = 0; + float totalRelativeQuality = 0; + float maxThroughput = 1; + float maxScope = 0; + float relStability[ZT_MAX_PEER_NETWORK_PATHS]; + float relThroughput[ZT_MAX_PEER_NETWORK_PATHS]; + memset(&relStability, 0, sizeof(relStability)); + memset(&relThroughput, 0, sizeof(relThroughput)); + // Survey all paths + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + relStability[i] = _paths[i].p->lastComputedStability(); + relThroughput[i] = (float)_paths[i].p->maxLifetimeThroughput(); + maxStability = relStability[i] > maxStability ? relStability[i] : maxStability; + maxThroughput = relThroughput[i] > maxThroughput ? relThroughput[i] : maxThroughput; + maxScope = _paths[i].p->ipScope() > maxScope ? _paths[i].p->ipScope() : maxScope; + } + } + // Convert to relative values + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + relStability[i] /= maxStability ? maxStability : 1; + relThroughput[i] /= maxThroughput ? maxThroughput : 1; + float normalized_ma = Utils::normalize((float)_paths[i].p->ackAge(now), 0, ZT_PATH_MAX_AGE, 0, 10); + float age_contrib = exp((-1)*normalized_ma); + float relScope = ((float)(_paths[i].p->ipScope()+1) / (maxScope + 1)); + float relQuality = + (relStability[i] * (float)ZT_PATH_CONTRIB_STABILITY) + + (fmaxf(1.0f, relThroughput[i]) * (float)ZT_PATH_CONTRIB_THROUGHPUT) + + relScope * (float)ZT_PATH_CONTRIB_SCOPE; + relQuality *= age_contrib; + // Arbitrary cutoffs + relQuality = relQuality > (1.00f / 100.0f) ? relQuality : 0.0f; + relQuality = relQuality < (99.0f / 100.0f) ? relQuality : 1.0f; + totalRelativeQuality += relQuality; + _paths[i].p->updateRelativeQuality(relQuality); + } + } + // Convert set of relative performances into an allocation set + for(uint16_t i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + _paths[i].p->updateComponentAllocationOfAggregateLink((unsigned char)((_paths[i].p->relativeQuality() / totalRelativeQuality) * 255)); + } + } +} + +int Peer::computeAggregateLinkPacketDelayVariance() +{ + float pdv = 0.0; + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + pdv += _paths[i].p->relativeQuality() * _paths[i].p->packetDelayVariance(); + } + } + return (int)pdv; +} + +int Peer::computeAggregateLinkMeanLatency() +{ + int ml = 0; + int pathCount = 0; + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + pathCount++; + ml += (int)(_paths[i].p->relativeQuality() * _paths[i].p->meanLatency()); + } + } + return ml / pathCount; +} + +int Peer::aggregateLinkPhysicalPathCount() +{ + std::map<std::string, bool> ifnamemap; + int pathCount = 0; + int64_t now = RR->node->now(); + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p && _paths[i].p->alive(now)) { + if (!ifnamemap[_paths[i].p->getName()]) { + ifnamemap[_paths[i].p->getName()] = true; + pathCount++; + } + } + } + return pathCount; +} + +int Peer::aggregateLinkLogicalPathCount() +{ + int pathCount = 0; + int64_t now = RR->node->now(); + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p && _paths[i].p->alive(now)) { + pathCount++; + } + } + return pathCount; +} +SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired) +{ + Mutex::Lock _l(_paths_m); unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS; - long bestPathQuality = 2147483647; + + /** + * Send traffic across the highest quality path only. This algorithm will still + * use the old path quality metric from protocol version 9. + */ + if (!_canUseMultipath) { + long bestPathQuality = 2147483647; + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + if ((includeExpired)||((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION)) { + const long q = _paths[i].p->quality(now) / _paths[i].priority; + if (q <= bestPathQuality) { + bestPathQuality = q; + bestPath = i; + } + } + } else break; + } + if (bestPath != ZT_MAX_PEER_NETWORK_PATHS) { + return _paths[bestPath].p; + } + return SharedPtr<Path>(); + } + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { if (_paths[i].p) { - if ((includeExpired)||((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION)) { - const long q = _paths[i].p->quality(now) / _paths[i].priority; - if (q <= bestPathQuality) { - bestPathQuality = q; - bestPath = i; + _paths[i].p->processBackgroundPathMeasurements(now); + } + } + + /** + * Randomly distribute traffic across all paths + */ + int numAlivePaths = 0; + int numStalePaths = 0; + if (RR->node->getMultipathMode() == ZT_MULTIPATH_RANDOM) { + int alivePaths[ZT_MAX_PEER_NETWORK_PATHS]; + int stalePaths[ZT_MAX_PEER_NETWORK_PATHS]; + memset(&alivePaths, -1, sizeof(alivePaths)); + memset(&stalePaths, -1, sizeof(stalePaths)); + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + if (_paths[i].p->alive(now)) { + alivePaths[numAlivePaths] = i; + numAlivePaths++; + } + else { + stalePaths[numStalePaths] = i; + numStalePaths++; } } - } else break; + } + unsigned int r = _freeRandomByte; + if (numAlivePaths > 0) { + int rf = r % numAlivePaths; + return _paths[alivePaths[rf]].p; + } + else if(numStalePaths > 0) { + // Resort to trying any non-expired path + int rf = r % numStalePaths; + return _paths[stalePaths[rf]].p; + } } - if (bestPath != ZT_MAX_PEER_NETWORK_PATHS) - return _paths[bestPath].p; + /** + * Proportionally allocate traffic according to dynamic path quality measurements + */ + if (RR->node->getMultipathMode() == ZT_MULTIPATH_PROPORTIONALLY_BALANCED) { + if ((now - _lastAggregateAllocation) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) { + _lastAggregateAllocation = now; + computeAggregateProportionalAllocation(now); + } + // Randomly choose path according to their allocations + float rf = _freeRandomByte; + for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + if (rf < _paths[i].p->allocation()) { + bestPath = i; + _pathChoiceHist.push(bestPath); // Record which path we chose + break; + } + rf -= _paths[i].p->allocation(); + } + } + if (bestPath < ZT_MAX_PEER_NETWORK_PATHS) { + return _paths[bestPath].p; + } + } return SharedPtr<Path>(); } +char *Peer::interfaceListStr() +{ + std::map<std::string, int> ifnamemap; + char tmp[32]; + const int64_t now = RR->node->now(); + char *ptr = _interfaceListStr; + bool imbalanced = false; + memset(_interfaceListStr, 0, sizeof(_interfaceListStr)); + int alivePathCount = aggregateLinkLogicalPathCount(); + for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p && _paths[i].p->alive(now)) { + int ipv = _paths[i].p->address().isV4(); + // If this is acting as an aggregate link, check allocations + float targetAllocation = 1.0f / (float)alivePathCount; + float currentAllocation = 1.0f; + if (alivePathCount > 1) { + currentAllocation = (float)_pathChoiceHist.countValue(i) / (float)_pathChoiceHist.count(); + if (fabs(targetAllocation - currentAllocation) > ZT_PATH_IMBALANCE_THRESHOLD) { + imbalanced = true; + } + } + char *ipvStr = ipv ? (char*)"ipv4" : (char*)"ipv6"; + sprintf(tmp, "(%s, %s, %.3f)", _paths[i].p->getName(), ipvStr, currentAllocation); + // Prevent duplicates + if(ifnamemap[_paths[i].p->getName()] != ipv) { + memcpy(ptr, tmp, strlen(tmp)); + ptr += strlen(tmp); + *ptr = ' '; + ptr++; + ifnamemap[_paths[i].p->getName()] = ipv; + } + } + } + ptr--; // Overwrite trailing space + if (imbalanced) { + sprintf(tmp, ", is asymmetrical"); + memcpy(ptr, tmp, sizeof(tmp)); + } else { + *ptr = '\0'; + } + return _interfaceListStr; +} + void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &other) const { unsigned int myBestV4ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; @@ -376,6 +632,73 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &o } } +inline void Peer::processBackgroundPeerTasks(const int64_t now) +{ + // Determine current multipath compatibility with other peer + if ((now - _lastMultipathCompatibilityCheck) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) { + // + // Cache number of available paths so that we can short-circuit multipath logic elsewhere + // + // We also take notice of duplicate paths (same IP only) because we may have + // recently received a direct path push from a peer and our list might contain + // a dead path which hasn't been fully recognized as such. In this case we + // don't want the duplicate to trigger execution of multipath code prematurely. + // + // This is done to support the behavior of auto multipath enable/disable + // without user intervention. + // + int currAlivePathCount = 0; + int duplicatePathsFound = 0; + for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { + if (_paths[i].p) { + currAlivePathCount++; + for (unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) { + if (_paths[i].p && _paths[j].p && _paths[i].p->address().ipsEqual2(_paths[j].p->address()) && i != j) { + duplicatePathsFound+=1; + break; + } + } + } + } + _uniqueAlivePathCount = (currAlivePathCount - (duplicatePathsFound / 2)); + _lastMultipathCompatibilityCheck = now; + _localMultipathSupported = ((RR->node->getMultipathMode() != ZT_MULTIPATH_NONE) && (ZT_PROTO_VERSION > 9)); + _remoteMultipathSupported = _vProto > 9; + // If both peers support multipath and more than one path exist, we can use multipath logic + _canUseMultipath = _localMultipathSupported && _remoteMultipathSupported && (_uniqueAlivePathCount > 1); + } +} + +void Peer::sendACK(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,const InetAddress &atAddress,int64_t now) +{ + Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ACK); + uint32_t bytesToAck = path->bytesToAck(); + outp.append<uint32_t>(bytesToAck); + if (atAddress) { + outp.armor(_key,false); + RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size()); + } else { + RR->sw->send(tPtr,outp,false); + } + path->sentAck(now); +} + +void Peer::sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,const InetAddress &atAddress,int64_t now) +{ + const int64_t _now = RR->node->now(); + Packet outp(_id.address(),RR->identity.address(),Packet::VERB_QOS_MEASUREMENT); + char qosData[ZT_PATH_MAX_QOS_PACKET_SZ]; + int16_t len = path->generateQoSPacket(_now,qosData); + outp.append(qosData,len); + if (atAddress) { + outp.armor(_key,false); + RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size()); + } else { + RR->sw->send(tPtr,outp,false); + } + path->sentQoS(now); +} + void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); @@ -444,12 +767,30 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now) unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) { unsigned int sent = 0; - Mutex::Lock _l(_paths_m); const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD); _lastSentFullHello = now; + processBackgroundPeerTasks(now); + + // Emit traces regarding aggregate link status + if (_canUseMultipath) { + int alivePathCount = aggregateLinkPhysicalPathCount(); + if ((now - _lastAggregateStatsReport) > ZT_PATH_AGGREGATE_STATS_REPORT_INTERVAL) { + _lastAggregateStatsReport = now; + if (alivePathCount) { + RR->t->peerLinkAggregateStatistics(NULL,*this); + } + } if (alivePathCount < 2 && _linkIsRedundant) { + _linkIsRedundant = !_linkIsRedundant; + RR->t->peerLinkNoLongerRedundant(NULL,*this); + } if (alivePathCount > 1 && !_linkIsRedundant) { + _linkIsRedundant = !_linkIsRedundant; + RR->t->peerLinkNowRedundant(NULL,*this); + } + } + // Right now we only keep pinging links that have the maximum priority. The // priority is used to track cluster redirections, meaning that when a cluster // redirects us its redirect target links override all other links and we @@ -477,13 +818,14 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) } } else break; } - while(j < ZT_MAX_PEER_NETWORK_PATHS) { - _paths[j].lr = 0; - _paths[j].p.zero(); - _paths[j].priority = 1; - ++j; + if (canUseMultipath()) { + while(j < ZT_MAX_PEER_NETWORK_PATHS) { + _paths[j].lr = 0; + _paths[j].p.zero(); + _paths[j].priority = 1; + ++j; + } } - return sent; } diff --git a/node/Peer.hpp b/node/Peer.hpp index b6f3c695..b4cbe057 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -27,18 +27,13 @@ #ifndef ZT_PEER_HPP #define ZT_PEER_HPP -#include <stdint.h> - -#include "Constants.hpp" - -#include <algorithm> -#include <utility> #include <vector> -#include <stdexcept> #include "../include/ZeroTierOne.h" +#include "Constants.hpp" #include "RuntimeEnvironment.hpp" +#include "Node.hpp" #include "Path.hpp" #include "Address.hpp" #include "Utils.hpp" @@ -108,6 +103,7 @@ public: const SharedPtr<Path> &path, const unsigned int hops, const uint64_t packetId, + const unsigned int payloadLength, const Packet::Verb verb, const uint64_t inRePacketId, const Packet::Verb inReVerb, @@ -145,20 +141,95 @@ public: */ inline bool sendDirect(void *tPtr,const void *data,unsigned int len,int64_t now,bool force) { - SharedPtr<Path> bp(getBestPath(now,force)); + SharedPtr<Path> bp(getAppropriatePath(now,force)); if (bp) return bp->send(RR,tPtr,data,len,now); return false; } /** - * Get the best current direct path + * Record statistics on outgoing packets + * + * @param path Path over which packet was sent + * @param id Packet ID + * @param len Length of packet payload + * @param verb Packet verb + * @param now Current time + */ + void recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, int64_t now); + + /** + * Record statistics on incoming packets + * + * @param path Path over which packet was sent + * @param id Packet ID + * @param len Length of packet payload + * @param verb Packet verb + * @param now Current time + */ + void recordIncomingPacket(void *tPtr, const SharedPtr<Path> &path, const uint64_t packetId, uint16_t payloadLength, const Packet::Verb verb, int64_t now); + + /** + * Send an ACK to peer for the most recent packets received + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param localSocket Raw socket the ACK packet will be sent over + * @param atAddress Destination for the ACK packet + * @param now Current time + */ + void sendACK(void *tPtr, const SharedPtr<Path> &path, const int64_t localSocket,const InetAddress &atAddress,int64_t now); + + /** + * Send a QoS packet to peer so that it can evaluate the quality of this link + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param localSocket Raw socket the QoS packet will be sent over + * @param atAddress Destination for the QoS packet + * @param now Current time + */ + void sendQOS_MEASUREMENT(void *tPtr, const SharedPtr<Path> &path, const int64_t localSocket,const InetAddress &atAddress,int64_t now); + + /** + * Compute relative quality values and allocations for the components of the aggregate link + * + * @param now Current time + */ + void computeAggregateProportionalAllocation(int64_t now); + + /** + * @return The aggregate link Packet Delay Variance (PDV) + */ + int computeAggregateLinkPacketDelayVariance(); + + /** + * @return The aggregate link mean latency + */ + int computeAggregateLinkMeanLatency(); + + /** + * @return The number of currently alive "physical" paths in the aggregate link + */ + int aggregateLinkPhysicalPathCount(); + + /** + * @return The number of currently alive "logical" paths in the aggregate link + */ + int aggregateLinkLogicalPathCount(); + + /** + * Get the most appropriate direct path based on current multipath and QoS configuration * * @param now Current time * @param includeExpired If true, include even expired paths * @return Best current path or NULL if none */ - SharedPtr<Path> getBestPath(int64_t now,bool includeExpired) const; + SharedPtr<Path> getAppropriatePath(int64_t now, bool includeExpired); + + /** + * Generate a human-readable string of interface names making up the aggregate link, also include + * moving allocation and IP version number for each (for tracing) + */ + char *interfaceListStr(); /** * Send VERB_RENDEZVOUS to this and another peer via the best common IP scope and path @@ -213,6 +284,16 @@ public: unsigned int doPingAndKeepalive(void *tPtr,int64_t now); /** + * Clear paths whose localSocket(s) are in a CLOSED state or have an otherwise INVALID state. + * This should be called frequently so that we can detect and remove unproductive or invalid paths. + * + * Under the hood this is done periodically based on ZT_CLOSED_PATH_PRUNING_INTERVAL. + * + * @return Number of paths that were pruned this round + */ + unsigned int prunePaths(); + + /** * Process a cluster redirect sent by this peer * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call @@ -268,14 +349,18 @@ public: inline int64_t isActive(int64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** - * @return Latency in milliseconds of best path or 0xffff if unknown / no paths + * @return Latency in milliseconds of best/aggregate path or 0xffff if unknown / no paths */ - inline unsigned int latency(const int64_t now) const + inline unsigned int latency(const int64_t now) { - SharedPtr<Path> bp(getBestPath(now,false)); - if (bp) - return bp->latency(); - return 0xffff; + if (_canUseMultipath) { + return (int)computeAggregateLinkMeanLatency(); + } else { + SharedPtr<Path> bp(getAppropriatePath(now,false)); + if (bp) + return bp->latency(); + return 0xffff; + } } /** @@ -289,7 +374,7 @@ public: * * @return Relay quality score computed from latency and other factors, lower is better */ - inline unsigned int relayQuality(const int64_t now) const + inline unsigned int relayQuality(const int64_t now) { const uint64_t tsr = now - _lastReceive; if (tsr >= ZT_PEER_ACTIVITY_TIMEOUT) @@ -329,6 +414,37 @@ public: inline bool remoteVersionKnown() const { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); } /** + * Periodically update known multipath activation constraints. This is done so that we know when and when + * not to use multipath logic. Doing this once every few seconds is sufficient. + * + * @param now Current time + */ + inline void processBackgroundPeerTasks(const int64_t now); + + /** + * Record that the remote peer does have multipath enabled. As is evident by the receipt of a VERB_ACK + * or a VERB_QOS_MEASUREMENT packet at some point in the past. Until this flag is set, the local client + * shall assume that multipath is not enabled and should only use classical Protocol 9 logic. + */ + inline void inferRemoteMultipathEnabled() { _remotePeerMultipathEnabled = true; } + + /** + * @return Whether the local client supports and is configured to use multipath + */ + inline bool localMultipathSupport() { return _localMultipathSupported; } + + /** + * @return Whether the remote peer supports and is configured to use multipath + */ + inline bool remoteMultipathSupport() { return _remoteMultipathSupported; } + + /** + * @return Whether this client can use multipath to communicate with this peer. True if both peers are using + * the correct protocol and if both peers have multipath enabled. False if otherwise. + */ + inline bool canUseMultipath() { return _canUseMultipath; } + + /** * @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms */ inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } @@ -394,27 +510,38 @@ public: } /** - * Rate gate incoming requests for network COM + * Rate limit gate for VERB_ACK */ - inline bool rateGateIncomingComRequest(const int64_t now) + inline bool rateGateACK(const int64_t now) { - if ((now - _lastComRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { - _lastComRequestReceived = now; - return true; + if ((now - _lastACKWindowReset) >= ZT_PATH_QOS_ACK_CUTOFF_TIME) { + _lastACKWindowReset = now; + _ACKCutoffCount = 0; + } else { + ++_ACKCutoffCount; } - return false; + return (_ACKCutoffCount < ZT_PATH_QOS_ACK_CUTOFF_LIMIT); } /** - * Rate gate outgoing requests for network COM + * Rate limit gate for VERB_QOS_MEASUREMENT */ - inline bool rateGateOutgoingComRequest(const int64_t now) + inline bool rateGateQoS(const int64_t now) { - if ((now - _lastComRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { - _lastComRequestSent = now; - return true; + if ((now - _lastQoSWindowReset) >= ZT_PATH_QOS_ACK_CUTOFF_TIME) { + _lastQoSWindowReset = now; + _QoSCutoffCount = 0; + } else { + ++_QoSCutoffCount; } - return false; + return (_QoSCutoffCount < ZT_PATH_QOS_ACK_CUTOFF_LIMIT); + } + + /** + * @return Whether this peer is reachable via an aggregate link + */ + inline bool hasAggregateLink() { + return _localMultipathSupported && _remoteMultipathSupported && _remotePeerMultipathEnabled; } /** @@ -510,11 +637,21 @@ private: int64_t _lastCredentialRequestSent; int64_t _lastWhoisRequestReceived; int64_t _lastEchoRequestReceived; - int64_t _lastComRequestReceived; - int64_t _lastComRequestSent; int64_t _lastCredentialsReceived; int64_t _lastTrustEstablishedPacketReceived; int64_t _lastSentFullHello; + int64_t _lastPathPrune; + int64_t _lastACKWindowReset; + int64_t _lastQoSWindowReset; + int64_t _lastMultipathCompatibilityCheck; + + unsigned char _freeRandomByte; + + int _uniqueAlivePathCount; + + bool _localMultipathSupported; + bool _remoteMultipathSupported; + bool _canUseMultipath; uint16_t _vProto; uint16_t _vMajor; @@ -528,8 +665,21 @@ private: unsigned int _directPathPushCutoffCount; unsigned int _credentialsCutoffCount; + unsigned int _QoSCutoffCount; + unsigned int _ACKCutoffCount; AtomicCounter __refCount; + + RingBuffer<int,ZT_MULTIPATH_PROPORTION_WIN_SZ> _pathChoiceHist; + + bool _linkIsBalanced; + bool _linkIsRedundant; + bool _remotePeerMultipathEnabled; + + int64_t _lastAggregateStatsReport; + int64_t _lastAggregateAllocation; + + char _interfaceListStr[256]; // 16 characters * 16 paths in a link }; } // namespace ZeroTier diff --git a/node/Poly1305.cpp b/node/Poly1305.cpp index eceb57b3..c670e5d0 100644 --- a/node/Poly1305.cpp +++ b/node/Poly1305.cpp @@ -18,115 +18,6 @@ Public domain. namespace ZeroTier { -#if 0 - -// "Naive" implementation, which is slower... might still want this on some older -// or weird platforms if the later versions have issues. - -static inline void add(unsigned int h[17],const unsigned int c[17]) -{ - unsigned int j; - unsigned int u; - u = 0; - for (j = 0;j < 17;++j) { u += h[j] + c[j]; h[j] = u & 255; u >>= 8; } -} - -static inline void squeeze(unsigned int h[17]) -{ - unsigned int j; - unsigned int u; - u = 0; - for (j = 0;j < 16;++j) { u += h[j]; h[j] = u & 255; u >>= 8; } - u += h[16]; h[16] = u & 3; - u = 5 * (u >> 2); - for (j = 0;j < 16;++j) { u += h[j]; h[j] = u & 255; u >>= 8; } - u += h[16]; h[16] = u; -} - -static const unsigned int minusp[17] = { - 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252 -} ; - -static inline void freeze(unsigned int h[17]) -{ - unsigned int horig[17]; - unsigned int j; - unsigned int negative; - for (j = 0;j < 17;++j) horig[j] = h[j]; - add(h,minusp); - negative = -(h[16] >> 7); - for (j = 0;j < 17;++j) h[j] ^= negative & (horig[j] ^ h[j]); -} - -static inline void mulmod(unsigned int h[17],const unsigned int r[17]) -{ - unsigned int hr[17]; - unsigned int i; - unsigned int j; - unsigned int u; - - for (i = 0;i < 17;++i) { - u = 0; - for (j = 0;j <= i;++j) u += h[j] * r[i - j]; - for (j = i + 1;j < 17;++j) u += 320 * h[j] * r[i + 17 - j]; - hr[i] = u; - } - for (i = 0;i < 17;++i) h[i] = hr[i]; - squeeze(h); -} - -static inline int crypto_onetimeauth(unsigned char *out,const unsigned char *in,unsigned long long inlen,const unsigned char *k) -{ - unsigned int j; - unsigned int r[17]; - unsigned int h[17]; - unsigned int c[17]; - - r[0] = k[0]; - r[1] = k[1]; - r[2] = k[2]; - r[3] = k[3] & 15; - r[4] = k[4] & 252; - r[5] = k[5]; - r[6] = k[6]; - r[7] = k[7] & 15; - r[8] = k[8] & 252; - r[9] = k[9]; - r[10] = k[10]; - r[11] = k[11] & 15; - r[12] = k[12] & 252; - r[13] = k[13]; - r[14] = k[14]; - r[15] = k[15] & 15; - r[16] = 0; - - for (j = 0;j < 17;++j) h[j] = 0; - - while (inlen > 0) { - for (j = 0;j < 17;++j) c[j] = 0; - for (j = 0;(j < 16) && (j < inlen);++j) c[j] = in[j]; - c[j] = 1; - in += j; inlen -= j; - add(h,c); - mulmod(h,r); - } - - freeze(h); - - for (j = 0;j < 16;++j) c[j] = k[j + 16]; - c[16] = 0; - add(h,c); - for (j = 0;j < 16;++j) out[j] = h[j]; - return 0; -} - -void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key) -{ - crypto_onetimeauth((unsigned char *)auth,(const unsigned char *)data,len,(const unsigned char *)key); -} - -#endif - namespace { typedef struct poly1305_context { @@ -215,8 +106,7 @@ static inline void U64TO8(unsigned char *p, unsigned long long v) #define U64TO8(p,v) ((*reinterpret_cast<unsigned long long *>(p)) = (v)) #endif -static inline void -poly1305_init(poly1305_context *ctx, const unsigned char key[32]) { +static inline void poly1305_init(poly1305_context *ctx, const unsigned char key[32]) { poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx; unsigned long long t0,t1; @@ -241,8 +131,7 @@ poly1305_init(poly1305_context *ctx, const unsigned char key[32]) { st->final = 0; } -static inline void -poly1305_blocks(poly1305_state_internal_t *st, const unsigned char *m, size_t bytes) { +static inline void poly1305_blocks(poly1305_state_internal_t *st, const unsigned char *m, size_t bytes) { const unsigned long long hibit = (st->final) ? 0 : ((unsigned long long)1 << 40); /* 1 << 128 */ unsigned long long r0,r1,r2; unsigned long long s1,s2; @@ -293,8 +182,7 @@ poly1305_blocks(poly1305_state_internal_t *st, const unsigned char *m, size_t by st->h[2] = h2; } -static inline void -poly1305_finish(poly1305_context *ctx, unsigned char mac[16]) { +static inline void poly1305_finish(poly1305_context *ctx, unsigned char mac[16]) { poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx; unsigned long long h0,h1,h2,c; unsigned long long g0,g1,g2; @@ -582,8 +470,7 @@ poly1305_finish(poly1305_context *ctx, unsigned char mac[16]) { #endif // MSC/GCC or not -static inline void -poly1305_update(poly1305_context *ctx, const unsigned char *m, size_t bytes) { +static inline void poly1305_update(poly1305_context *ctx, const unsigned char *m, size_t bytes) { poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx; size_t i; diff --git a/node/Poly1305.hpp b/node/Poly1305.hpp index adcc2410..4614826e 100644 --- a/node/Poly1305.hpp +++ b/node/Poly1305.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Revocation.cpp b/node/Revocation.cpp index 78098f8c..04342569 100644 --- a/node/Revocation.cpp +++ b/node/Revocation.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Revocation.hpp b/node/Revocation.hpp index eaf01915..1d001a40 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -58,9 +58,17 @@ class Revocation : public Credential public: static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_REVOCATION; } - Revocation() + Revocation() : + _id(0), + _credentialId(0), + _networkId(0), + _threshold(0), + _flags(0), + _target(), + _signedBy(), + _type(Credential::CREDENTIAL_TYPE_NULL) { - memset(this,0,sizeof(Revocation)); + memset(_signature.data,0,sizeof(_signature.data)); } /** @@ -80,7 +88,10 @@ public: _flags(fl), _target(tgt), _signedBy(), - _type(ct) {} + _type(ct) + { + memset(_signature.data,0,sizeof(_signature.data)); + } inline uint32_t id() const { return _id; } inline uint32_t credentialId() const { return _credentialId; } @@ -148,7 +159,7 @@ public: template<unsigned int C> inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0) { - memset(this,0,sizeof(Revocation)); + *this = Revocation(); unsigned int p = startAt; @@ -166,7 +177,7 @@ public: if (b[p++] == 1) { if (b.template at<uint16_t>(p) == ZT_C25519_SIGNATURE_LEN) { p += 2; - ZT_FAST_MEMCPY(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; } else { diff --git a/node/RingBuffer.hpp b/node/RingBuffer.hpp new file mode 100644 index 00000000..0f29a89a --- /dev/null +++ b/node/RingBuffer.hpp @@ -0,0 +1,334 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_RINGBUFFER_H +#define ZT_RINGBUFFER_H + +#include <typeinfo> +#include <cstdint> +#include <stdlib.h> +#include <memory.h> +#include <algorithm> +#include <math.h> + +namespace ZeroTier { + +/** + * A circular buffer + * + * For fast handling of continuously-evolving variables (such as path quality metrics). + * Using this, we can maintain longer sliding historical windows for important path + * metrics without the need for potentially expensive calls to memcpy/memmove. + * + * Some basic statistical functionality is implemented here in an attempt + * to reduce the complexity of code needed to interact with this type of buffer. + */ + +template <class T,size_t S> +class RingBuffer +{ +private: + T buf[S]; + size_t begin; + size_t end; + bool wrap; + +public: + RingBuffer() : + begin(0), + end(0), + wrap(false) + { + memset(buf,0,sizeof(T)*S); + } + + /** + * @return A pointer to the underlying buffer + */ + inline T *get_buf() + { + return buf + begin; + } + + /** + * Adjust buffer index pointer as if we copied data in + * @param n Number of elements to copy in + * @return Number of elements we copied in + */ + inline size_t produce(size_t n) + { + n = std::min(n, getFree()); + if (n == 0) { + return n; + } + const size_t first_chunk = std::min(n, S - end); + end = (end + first_chunk) % S; + if (first_chunk < n) { + const size_t second_chunk = n - first_chunk; + end = (end + second_chunk) % S; + } + if (begin == end) { + wrap = true; + } + return n; + } + + /** + * Fast erase, O(1). + * Merely reset the buffer pointer, doesn't erase contents + */ + inline void reset() { consume(count()); } + + /** + * adjust buffer index pointer as if we copied data out + * @param n Number of elements we copied from the buffer + * @return Number of elements actually available from the buffer + */ + inline size_t consume(size_t n) + { + n = std::min(n, count()); + if (n == 0) { + return n; + } + if (wrap) { + wrap = false; + } + const size_t first_chunk = std::min(n, S - begin); + begin = (begin + first_chunk) % S; + if (first_chunk < n) { + const size_t second_chunk = n - first_chunk; + begin = (begin + second_chunk) % S; + } + return n; + } + + /** + * @param data Buffer that is to be written to the ring + * @param n Number of elements to write to the buffer + */ + inline size_t write(const T * data, size_t n) + { + n = std::min(n, getFree()); + if (n == 0) { + return n; + } + const size_t first_chunk = std::min(n, S - end); + memcpy(buf + end, data, first_chunk * sizeof(T)); + end = (end + first_chunk) % S; + if (first_chunk < n) { + const size_t second_chunk = n - first_chunk; + memcpy(buf + end, data + first_chunk, second_chunk * sizeof(T)); + end = (end + second_chunk) % S; + } + if (begin == end) { + wrap = true; + } + return n; + } + + /** + * Place a single value on the buffer. If the buffer is full, consume a value first. + * + * @param value A single value to be placed in the buffer + */ + inline void push(const T value) + { + if (count() == S) { + consume(1); + } + const size_t first_chunk = std::min((size_t)1, S - end); + *(buf + end) = value; + end = (end + first_chunk) % S; + if (begin == end) { + wrap = true; + } + } + + /** + * @return The most recently pushed element on the buffer + */ + inline T get_most_recent() { return *(buf + end); } + + /** + * @param dest Destination buffer + * @param n Size (in terms of number of elements) of the destination buffer + * @return Number of elements read from the buffer + */ + inline size_t read(T *dest,size_t n) + { + n = std::min(n, count()); + if (n == 0) { + return n; + } + if (wrap) { + wrap = false; + } + const size_t first_chunk = std::min(n, S - begin); + memcpy(dest, buf + begin, first_chunk * sizeof(T)); + begin = (begin + first_chunk) % S; + if (first_chunk < n) { + const size_t second_chunk = n - first_chunk; + memcpy(dest + first_chunk, buf + begin, second_chunk * sizeof(T)); + begin = (begin + second_chunk) % S; + } + return n; + } + + /** + * Return how many elements are in the buffer, O(1). + * + * @return The number of elements in the buffer + */ + inline size_t count() + { + if (end == begin) { + return wrap ? S : 0; + } + else if (end > begin) { + return end - begin; + } + else { + return S + end - begin; + } + } + + /** + * @return The number of slots that are unused in the buffer + */ + inline size_t getFree() { return S - count(); } + + /** + * @return The arithmetic mean of the contents of the buffer + */ + inline float mean() + { + size_t iterator = begin; + float subtotal = 0; + size_t curr_cnt = count(); + for (size_t i=0; i<curr_cnt; i++) { + iterator = (iterator + S - 1) % curr_cnt; + subtotal += (float)*(buf + iterator); + } + return curr_cnt ? subtotal / (float)curr_cnt : 0; + } + + /** + * @return The arithmetic mean of the most recent 'n' elements of the buffer + */ + inline float mean(size_t n) + { + n = n < S ? n : S; + size_t iterator = begin; + float subtotal = 0; + size_t curr_cnt = count(); + for (size_t i=0; i<n; i++) { + iterator = (iterator + S - 1) % curr_cnt; + subtotal += (float)*(buf + iterator); + } + return curr_cnt ? subtotal / (float)curr_cnt : 0; + } + + /** + * @return The sample standard deviation of element values + */ + inline float stddev() { return sqrt(variance()); } + + /** + * @return The variance of element values + */ + inline float variance() + { + size_t iterator = begin; + float cached_mean = mean(); + size_t curr_cnt = count(); + T sum_of_squared_deviations = 0; + for (size_t i=0; i<curr_cnt; i++) { + iterator = (iterator + S - 1) % curr_cnt; + float deviation = (buf[i] - cached_mean); + sum_of_squared_deviations += (T)(deviation*deviation); + } + float variance = (float)sum_of_squared_deviations / (float)(S - 1); + return variance; + } + + /** + * @return The number of elements of zero value + */ + inline size_t zeroCount() + { + size_t iterator = begin; + size_t zeros = 0; + size_t curr_cnt = count(); + for (size_t i=0; i<curr_cnt; i++) { + iterator = (iterator + S - 1) % curr_cnt; + if (*(buf + iterator) == 0) { + zeros++; + } + } + return zeros; + } + + /** + * @param value Value to match against in buffer + * @return The number of values held in the ring buffer which match a given value + */ + inline size_t countValue(T value) + { + size_t iterator = begin; + size_t cnt = 0; + size_t curr_cnt = count(); + for (size_t i=0; i<curr_cnt; i++) { + iterator = (iterator + S - 1) % curr_cnt; + if (*(buf + iterator) == value) { + cnt++; + } + } + return cnt; + } + + /** + * Print the contents of the buffer + */ + /* + inline void dump() + { + size_t iterator = begin; + for (size_t i=0; i<S; i++) { + iterator = (iterator + S - 1) % S; + if (typeid(T) == typeid(int)) { + //DEBUG_INFO("buf[%2zu]=%2d", iterator, (int)*(buf + iterator)); + } + else { + //DEBUG_INFO("buf[%2zu]=%2f", iterator, (float)*(buf + iterator)); + } + } + } + */ +}; + +} // namespace ZeroTier + +#endif diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index 46350b4a..e8ee4c59 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/SHA512.hpp b/node/SHA512.hpp index eedc284a..cb7b40a8 100644 --- a/node/SHA512.hpp +++ b/node/SHA512.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Salsa20.cpp b/node/Salsa20.cpp index 1d4117e3..ca9e4718 100644 --- a/node/Salsa20.cpp +++ b/node/Salsa20.cpp @@ -14,7 +14,7 @@ #define XOR(v,w) ((v) ^ (w)) #define PLUS(v,w) ((uint32_t)((v) + (w))) -// Set up laod/store macros with appropriate endianness (we don't use these in SSE mode) +// Set up load/store macros with appropriate endianness (we don't use these in SSE mode) #ifndef ZT_SALSA20_SSE #if __BYTE_ORDER == __LITTLE_ENDIAN diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index c4f107fb..0fb9e4a4 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -124,87 +124,4 @@ void SelfAwareness::clean(int64_t now) } } -std::vector<InetAddress> SelfAwareness::getSymmetricNatPredictions() -{ - /* This is based on ideas and strategies found here: - * https://tools.ietf.org/html/draft-takeda-symmetric-nat-traversal-00 - * - * For each IP address reported by a trusted (upstream) peer, we find - * the external port most recently reported by ANY peer for that IP. - * - * We only do any of this for global IPv4 addresses since private IPs - * and IPv6 are not going to have symmetric NAT. - * - * SECURITY NOTE: - * - * We never use IPs reported by non-trusted peers, since this could lead - * to a minor vulnerability whereby a peer could poison our cache with - * bad external surface reports via OK(HELLO) and then possibly coax us - * into suggesting their IP to other peers via PUSH_DIRECT_PATHS. This - * in turn could allow them to MITM flows. - * - * Since flows are encrypted and authenticated they could not actually - * read or modify traffic, but they could gather meta-data for forensics - * purpsoes or use this as a DOS attack vector. */ - - std::map< uint32_t,unsigned int > maxPortByIp; - InetAddress theOneTrueSurface; - { - Mutex::Lock _l(_phy_m); - - // First check to see if this is a symmetric NAT and enumerate external IPs learned from trusted peers - bool symmetric = false; - { - Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); - PhySurfaceKey *k = (PhySurfaceKey *)0; - PhySurfaceEntry *e = (PhySurfaceEntry *)0; - while (i.next(k,e)) { - if ((e->trusted)&&(e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { - if (!theOneTrueSurface) - theOneTrueSurface = e->mySurface; - else if (theOneTrueSurface != e->mySurface) - symmetric = true; - maxPortByIp[reinterpret_cast<const struct sockaddr_in *>(&(e->mySurface))->sin_addr.s_addr] = e->mySurface.port(); - } - } - } - if (!symmetric) - return std::vector<InetAddress>(); - - { // Then find the highest issued port per IP - Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); - PhySurfaceKey *k = (PhySurfaceKey *)0; - PhySurfaceEntry *e = (PhySurfaceEntry *)0; - while (i.next(k,e)) { - if ((e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { - const unsigned int port = e->mySurface.port(); - std::map< uint32_t,unsigned int >::iterator mp(maxPortByIp.find(reinterpret_cast<const struct sockaddr_in *>(&(e->mySurface))->sin_addr.s_addr)); - if ((mp != maxPortByIp.end())&&(mp->second < port)) - mp->second = port; - } - } - } - } - - std::vector<InetAddress> r; - - // Try next port up from max for each - for(std::map< uint32_t,unsigned int >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) { - unsigned int p = i->second + 1; - if (p > 65535) p -= 64511; - const InetAddress pred(&(i->first),4,p); - if (std::find(r.begin(),r.end(),pred) == r.end()) - r.push_back(pred); - } - - // Try a random port for each -- there are only 65535 so eventually it should work - for(std::map< uint32_t,unsigned int >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) { - const InetAddress pred(&(i->first),4,1024 + ((unsigned int)RR->node->prng() % 64511)); - if (std::find(r.begin(),r.end(),pred) == r.end()) - r.push_back(pred); - } - - return r; -} - } // namespace ZeroTier diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index ce6e8c76..7940f5ac 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -64,13 +64,6 @@ public: */ void clean(int64_t now); - /** - * If we appear to be behind a symmetric NAT, get predictions for possible external endpoints - * - * @return Symmetric NAT predictions or empty vector if none - */ - std::vector<InetAddress> getSymmetricNatPredictions(); - private: struct PhySurfaceKey { diff --git a/node/SharedPtr.hpp b/node/SharedPtr.hpp index 2f0e50c9..c7f9ea97 100644 --- a/node/SharedPtr.hpp +++ b/node/SharedPtr.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Switch.cpp b/node/Switch.cpp index eeeca5db..a6852d9f 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -269,6 +269,8 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const } } + uint8_t qosBucket = ZT_QOS_DEFAULT_BUCKET; + if (to.isMulticast()) { MulticastGroup multicastGroup(to,0); @@ -386,7 +388,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const network->learnBridgedMulticastGroup(tPtr,multicastGroup,RR->node->now()); // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. - if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) { RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -410,11 +412,13 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this SharedPtr<Peer> toPeer(RR->topology->getPeer(tPtr,toZT)); - if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) { RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked"); return; } + network->pushCredentialsIfNeeded(tPtr,toZT,RR->node->now()); + if (fromBridged) { Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(network->id()); @@ -425,7 +429,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(tPtr,outp,true); + aqm_enqueue(tPtr,network,outp,true,qosBucket); } else { Packet outp(toZT,RR->identity.address(),Packet::VERB_FRAME); outp.append(network->id()); @@ -433,16 +437,15 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(tPtr,outp,true); + aqm_enqueue(tPtr,network,outp,true,qosBucket); } - } else { // Destination is bridged behind a remote peer // We filter with a NULL destination ZeroTier address first. Filtrations // for each ZT destination are also done below. This is the same rationale // and design as for multicast. - if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) { RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -480,7 +483,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const } for(unsigned int b=0;b<numBridges;++b) { - if (network->filterOutgoingPacket(tPtr,true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (network->filterOutgoingPacket(tPtr,true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) { Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(network->id()); outp.append((uint8_t)0x00); @@ -490,7 +493,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(tPtr,outp,true); + aqm_enqueue(tPtr,network,outp,true,qosBucket); } else { RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked (bridge replication)"); } @@ -498,6 +501,263 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const } } +void Switch::aqm_enqueue(void *tPtr, const SharedPtr<Network> &network, Packet &packet,bool encrypt,int qosBucket) +{ + if(!network->qosEnabled()) { + send(tPtr, packet, encrypt); + return; + } + NetworkQoSControlBlock *nqcb = _netQueueControlBlock[network->id()]; + if (!nqcb) { + // DEBUG_INFO("creating network QoS control block (NQCB) for network %llx", network->id()); + nqcb = new NetworkQoSControlBlock(); + _netQueueControlBlock[network->id()] = nqcb; + // Initialize ZT_QOS_NUM_BUCKETS queues and place them in the INACTIVE list + // These queues will be shuffled between the new/old/inactive lists by the enqueue/dequeue algorithm + for (int i=0; i<ZT_QOS_NUM_BUCKETS; i++) { + nqcb->inactiveQueues.push_back(new ManagedQueue(i)); + } + } + + if (packet.verb() != Packet::VERB_FRAME && packet.verb() != Packet::VERB_EXT_FRAME) { + // DEBUG_INFO("skipping, no QoS for this packet, verb=%x", packet.verb()); + // just send packet normally, no QoS for ZT protocol traffic + send(tPtr, packet, encrypt); + } + + _aqm_m.lock(); + + // Enqueue packet and move queue to appropriate list + + const Address dest(packet.destination()); + TXQueueEntry *txEntry = new TXQueueEntry(dest,RR->node->now(),packet,encrypt); + + ManagedQueue *selectedQueue = nullptr; + for (size_t i=0; i<ZT_QOS_NUM_BUCKETS; i++) { + if (i < nqcb->oldQueues.size()) { // search old queues first (I think this is best since old would imply most recent usage of the queue) + if (nqcb->oldQueues[i]->id == qosBucket) { + selectedQueue = nqcb->oldQueues[i]; + } + } if (i < nqcb->newQueues.size()) { // search new queues (this would imply not often-used queues) + if (nqcb->newQueues[i]->id == qosBucket) { + selectedQueue = nqcb->newQueues[i]; + } + } if (i < nqcb->inactiveQueues.size()) { // search inactive queues + if (nqcb->inactiveQueues[i]->id == qosBucket) { + selectedQueue = nqcb->inactiveQueues[i]; + // move queue to end of NEW queue list + selectedQueue->byteCredit = ZT_QOS_QUANTUM; + // DEBUG_INFO("moving q=%p from INACTIVE to NEW list", selectedQueue); + nqcb->newQueues.push_back(selectedQueue); + nqcb->inactiveQueues.erase(nqcb->inactiveQueues.begin() + i); + } + } + } + if (!selectedQueue) { + return; + } + + selectedQueue->q.push_back(txEntry); + selectedQueue->byteLength+=txEntry->packet.payloadLength(); + nqcb->_currEnqueuedPackets++; + + // DEBUG_INFO("nq=%2lu, oq=%2lu, iq=%2lu, nqcb.size()=%3d, bucket=%2d, q=%p", nqcb->newQueues.size(), nqcb->oldQueues.size(), nqcb->inactiveQueues.size(), nqcb->_currEnqueuedPackets, qosBucket, selectedQueue); + + // Drop a packet if necessary + ManagedQueue *selectedQueueToDropFrom = nullptr; + if (nqcb->_currEnqueuedPackets > ZT_QOS_MAX_ENQUEUED_PACKETS) + { + // DEBUG_INFO("too many enqueued packets (%d), finding packet to drop", nqcb->_currEnqueuedPackets); + int maxQueueLength = 0; + for (size_t i=0; i<ZT_QOS_NUM_BUCKETS; i++) { + if (i < nqcb->oldQueues.size()) { + if (nqcb->oldQueues[i]->byteLength > maxQueueLength) { + maxQueueLength = nqcb->oldQueues[i]->byteLength; + selectedQueueToDropFrom = nqcb->oldQueues[i]; + } + } if (i < nqcb->newQueues.size()) { + if (nqcb->newQueues[i]->byteLength > maxQueueLength) { + maxQueueLength = nqcb->newQueues[i]->byteLength; + selectedQueueToDropFrom = nqcb->newQueues[i]; + } + } if (i < nqcb->inactiveQueues.size()) { + if (nqcb->inactiveQueues[i]->byteLength > maxQueueLength) { + maxQueueLength = nqcb->inactiveQueues[i]->byteLength; + selectedQueueToDropFrom = nqcb->inactiveQueues[i]; + } + } + } + if (selectedQueueToDropFrom) { + // DEBUG_INFO("dropping packet from head of largest queue (%d payload bytes)", maxQueueLength); + int sizeOfDroppedPacket = selectedQueueToDropFrom->q.front()->packet.payloadLength(); + delete selectedQueueToDropFrom->q.front(); + selectedQueueToDropFrom->q.pop_front(); + selectedQueueToDropFrom->byteLength-=sizeOfDroppedPacket; + nqcb->_currEnqueuedPackets--; + } + } + _aqm_m.unlock(); + aqm_dequeue(tPtr); +} + +uint64_t Switch::control_law(uint64_t t, int count) +{ + return (uint64_t)(t + ZT_QOS_INTERVAL / sqrt(count)); +} + +Switch::dqr Switch::dodequeue(ManagedQueue *q, uint64_t now) +{ + dqr r; + r.ok_to_drop = false; + r.p = q->q.front(); + + if (r.p == NULL) { + q->first_above_time = 0; + return r; + } + uint64_t sojourn_time = now - r.p->creationTime; + if (sojourn_time < ZT_QOS_TARGET || q->byteLength <= ZT_DEFAULT_MTU) { + // went below - stay below for at least interval + q->first_above_time = 0; + } else { + if (q->first_above_time == 0) { + // just went above from below. if still above at + // first_above_time, will say it's ok to drop. + q->first_above_time = now + ZT_QOS_INTERVAL; + } else if (now >= q->first_above_time) { + r.ok_to_drop = true; + } + } + return r; +} + +Switch::TXQueueEntry * Switch::CoDelDequeue(ManagedQueue *q, bool isNew, uint64_t now) +{ + dqr r = dodequeue(q, now); + + if (q->dropping) { + if (!r.ok_to_drop) { + q->dropping = false; + } + while (now >= q->drop_next && q->dropping) { + q->q.pop_front(); // drop + r = dodequeue(q, now); + if (!r.ok_to_drop) { + // leave dropping state + q->dropping = false; + } else { + ++(q->count); + // schedule the next drop. + q->drop_next = control_law(q->drop_next, q->count); + } + } + } else if (r.ok_to_drop) { + q->q.pop_front(); // drop + r = dodequeue(q, now); + q->dropping = true; + q->count = (q->count > 2 && now - q->drop_next < 8*ZT_QOS_INTERVAL)? + q->count - 2 : 1; + q->drop_next = control_law(now, q->count); + } + return r.p; +} + +void Switch::aqm_dequeue(void *tPtr) +{ + // Cycle through network-specific QoS control blocks + for(std::map<uint64_t,NetworkQoSControlBlock*>::iterator nqcb(_netQueueControlBlock.begin());nqcb!=_netQueueControlBlock.end();) { + if (!(*nqcb).second->_currEnqueuedPackets) { + return; + } + + uint64_t now = RR->node->now(); + TXQueueEntry *entryToEmit = nullptr; + std::vector<ManagedQueue*> *currQueues = &((*nqcb).second->newQueues); + std::vector<ManagedQueue*> *oldQueues = &((*nqcb).second->oldQueues); + std::vector<ManagedQueue*> *inactiveQueues = &((*nqcb).second->inactiveQueues); + + _aqm_m.lock(); + + // Attempt dequeue from queues in NEW list + bool examiningNewQueues = true; + while (currQueues->size()) { + ManagedQueue *queueAtFrontOfList = currQueues->front(); + if (queueAtFrontOfList->byteCredit < 0) { + queueAtFrontOfList->byteCredit += ZT_QOS_QUANTUM; + // Move to list of OLD queues + // DEBUG_INFO("moving q=%p from NEW to OLD list", queueAtFrontOfList); + oldQueues->push_back(queueAtFrontOfList); + currQueues->erase(currQueues->begin()); + } else { + entryToEmit = CoDelDequeue(queueAtFrontOfList, examiningNewQueues, now); + if (!entryToEmit) { + // Move to end of list of OLD queues + // DEBUG_INFO("moving q=%p from NEW to OLD list", queueAtFrontOfList); + oldQueues->push_back(queueAtFrontOfList); + currQueues->erase(currQueues->begin()); + } + else { + int len = entryToEmit->packet.payloadLength(); + queueAtFrontOfList->byteLength -= len; + queueAtFrontOfList->byteCredit -= len; + // Send the packet! + queueAtFrontOfList->q.pop_front(); + send(tPtr, entryToEmit->packet, entryToEmit->encrypt); + (*nqcb).second->_currEnqueuedPackets--; + } + if (queueAtFrontOfList) { + //DEBUG_INFO("dequeuing from q=%p, len=%lu in NEW list (byteCredit=%d)", queueAtFrontOfList, queueAtFrontOfList->q.size(), queueAtFrontOfList->byteCredit); + } + break; + } + } + + // Attempt dequeue from queues in OLD list + examiningNewQueues = false; + currQueues = &((*nqcb).second->oldQueues); + while (currQueues->size()) { + ManagedQueue *queueAtFrontOfList = currQueues->front(); + if (queueAtFrontOfList->byteCredit < 0) { + queueAtFrontOfList->byteCredit += ZT_QOS_QUANTUM; + oldQueues->push_back(queueAtFrontOfList); + currQueues->erase(currQueues->begin()); + } else { + entryToEmit = CoDelDequeue(queueAtFrontOfList, examiningNewQueues, now); + if (!entryToEmit) { + //DEBUG_INFO("moving q=%p from OLD to INACTIVE list", queueAtFrontOfList); + // Move to inactive list of queues + inactiveQueues->push_back(queueAtFrontOfList); + currQueues->erase(currQueues->begin()); + } + else { + int len = entryToEmit->packet.payloadLength(); + queueAtFrontOfList->byteLength -= len; + queueAtFrontOfList->byteCredit -= len; + queueAtFrontOfList->q.pop_front(); + send(tPtr, entryToEmit->packet, entryToEmit->encrypt); + (*nqcb).second->_currEnqueuedPackets--; + } + if (queueAtFrontOfList) { + //DEBUG_INFO("dequeuing from q=%p, len=%lu in OLD list (byteCredit=%d)", queueAtFrontOfList, queueAtFrontOfList->q.size(), queueAtFrontOfList->byteCredit); + } + break; + } + } + nqcb++; + _aqm_m.unlock(); + } +} + +void Switch::removeNetworkQoSControlBlock(uint64_t nwid) +{ + NetworkQoSControlBlock *nq = _netQueueControlBlock[nwid]; + if (nq) { + _netQueueControlBlock.erase(nwid); + delete nq; + nq = NULL; + } +} + void Switch::send(void *tPtr,Packet &packet,bool encrypt) { const Address dest(packet.destination()); @@ -581,6 +841,7 @@ unsigned long Switch::doTimerTasks(void *tPtr,int64_t now) std::vector<Address> needWhois; { Mutex::Lock _l(_txQueue_m); + for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { if (_trySend(tPtr,txi->packet,txi->encrypt)) { _txQueue.erase(txi++); @@ -654,12 +915,12 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) const SharedPtr<Peer> peer(RR->topology->getPeer(tPtr,destination)); if (peer) { - viaPath = peer->getBestPath(now,false); + viaPath = peer->getAppropriatePath(now,false); if (!viaPath) { peer->tryMemorizedPath(tPtr,now); // periodically attempt memorized or statically defined paths, if any are known const SharedPtr<Peer> relay(RR->topology->getUpstreamPeer()); - if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { - if (!(viaPath = peer->getBestPath(now,true))) + if ( (!relay) || (!(viaPath = relay->getAppropriatePath(now,false))) ) { + if (!(viaPath = peer->getAppropriatePath(now,true))) return false; } } @@ -674,6 +935,8 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) unsigned int chunkSize = std::min(packet.size(),mtu); packet.setFragmented(chunkSize < packet.size()); + peer->recordOutgoingPacket(viaPath, packet.packetId(), packet.payloadLength(), packet.verb(), now); + if (trustedPathId) { packet.setTrusted(trustedPathId); } else { diff --git a/node/Switch.hpp b/node/Switch.hpp index 5de17fa0..a531b268 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -44,6 +44,16 @@ #include "IncomingPacket.hpp" #include "Hashtable.hpp" +/* Ethernet frame types that might be relevant to us */ +#define ZT_ETHERTYPE_IPV4 0x0800 +#define ZT_ETHERTYPE_ARP 0x0806 +#define ZT_ETHERTYPE_RARP 0x8035 +#define ZT_ETHERTYPE_ATALK 0x809b +#define ZT_ETHERTYPE_AARP 0x80f3 +#define ZT_ETHERTYPE_IPX_A 0x8137 +#define ZT_ETHERTYPE_IPX_B 0x8138 +#define ZT_ETHERTYPE_IPV6 0x86dd + namespace ZeroTier { class RuntimeEnvironment; @@ -59,6 +69,14 @@ class Peer; */ class Switch { + struct ManagedQueue; + struct TXQueueEntry; + + typedef struct { + TXQueueEntry *p; + bool ok_to_drop; + } dqr; + public: Switch(const RuntimeEnvironment *renv); @@ -88,6 +106,62 @@ public: void onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); /** + * Determines the next drop schedule for packets in the TX queue + * + * @param t Current time + * @param count Number of packets dropped this round + */ + uint64_t control_law(uint64_t t, int count); + + /** + * Selects a packet eligible for transmission from a TX queue. According to the control law, multiple packets + * may be intentionally dropped before a packet is returned to the AQM scheduler. + * + * @param q The TX queue that is being dequeued from + * @param now Current time + */ + dqr dodequeue(ManagedQueue *q, uint64_t now); + + /** + * Presents a packet to the AQM scheduler. + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param network Network that the packet shall be sent over + * @param packet Packet to be sent + * @param encrypt Encrypt packet payload? (always true except for HELLO) + * @param qosBucket Which bucket the rule-system determined this packet should fall into + */ + void aqm_enqueue(void *tPtr, const SharedPtr<Network> &network, Packet &packet,bool encrypt,int qosBucket); + + /** + * Performs a single AQM cycle and dequeues and transmits all eligible packets on all networks + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + */ + void aqm_dequeue(void *tPtr); + + /** + * Calls the dequeue mechanism and adjust queue state variables + * + * @param q The TX queue that is being dequeued from + * @param isNew Whether or not this queue is in the NEW list + * @param now Current time + */ + Switch::TXQueueEntry * CoDelDequeue(ManagedQueue *q, bool isNew, uint64_t now); + + /** + * Removes QoS Queues and flow state variables for a specific network. These queues are created + * automatically upon the transmission of the first packet from this peer to another peer on the + * given network. + * + * The reason for existence of queues and flow state variables specific to each network is so that + * each network's QoS rules function independently. + * + * @param nwid Network ID + */ + void removeNetworkQoSControlBlock(uint64_t nwid); + + /** * Send a packet to a ZeroTier address (destination in packet) * * The packet must be fully composed with source and destination but not @@ -200,6 +274,7 @@ private: }; std::list< TXQueueEntry > _txQueue; Mutex _txQueue_m; + Mutex _aqm_m; // Tracks sending of VERB_RENDEZVOUS to relaying peers struct _LastUniteKey @@ -221,6 +296,35 @@ private: }; Hashtable< _LastUniteKey,uint64_t > _lastUniteAttempt; // key is always sorted in ascending order, for set-like behavior Mutex _lastUniteAttempt_m; + + // Queue with additional flow state variables + struct ManagedQueue + { + ManagedQueue(int id) : + id(id), + byteCredit(ZT_QOS_QUANTUM), + byteLength(0), + dropping(false) + {} + int id; + int byteCredit; + int byteLength; + uint64_t first_above_time; + uint32_t count; + uint64_t drop_next; + bool dropping; + uint64_t drop_next_time; + std::list< TXQueueEntry *> q; + }; + // To implement fq_codel we need to maintain a queue of queues + struct NetworkQoSControlBlock + { + int _currEnqueuedPackets; + std::vector<ManagedQueue *> newQueues; + std::vector<ManagedQueue *> oldQueues; + std::vector<ManagedQueue *> inactiveQueues; + }; + std::map<uint64_t,NetworkQoSControlBlock*> _netQueueControlBlock; }; } // namespace ZeroTier diff --git a/node/Tag.cpp b/node/Tag.cpp index 62d9cb2e..6c78f3e4 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Tag.hpp b/node/Tag.hpp index d2e932c2..3f888ba9 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -58,16 +58,20 @@ class RuntimeEnvironment; * values. * * Unlike capabilities tags are signed only by the issuer and are never - * transferrable. + * transferable. */ class Tag : public Credential { public: static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_TAG; } - Tag() + Tag() : + _id(0), + _value(0), + _networkId(0), + _ts(0) { - memset(this,0,sizeof(Tag)); + memset(_signature.data,0,sizeof(_signature.data)); } /** @@ -85,6 +89,7 @@ public: _issuedTo(issuedTo), _signedBy() { + memset(_signature.data,0,sizeof(_signature.data)); } inline uint32_t id() const { return _id; } @@ -149,7 +154,7 @@ public: { unsigned int p = startAt; - memset(this,0,sizeof(Tag)); + *this = Tag(); _networkId = b.template at<uint64_t>(p); p += 8; _ts = b.template at<uint64_t>(p); p += 8; @@ -163,7 +168,7 @@ public: if (b.template at<uint16_t>(p) != ZT_C25519_SIGNATURE_LEN) throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; p += 2; - ZT_FAST_MEMCPY(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else { p += 2 + b.template at<uint16_t>(p); } diff --git a/node/Topology.cpp b/node/Topology.cpp index 12aa8d2c..6d6beda3 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -138,7 +138,7 @@ SharedPtr<Peer> Topology::getPeer(void *tPtr,const Address &zta) } return SharedPtr<Peer>(); } - } catch ( ... ) {} // ignore invalid identities or other strage failures + } catch ( ... ) {} // ignore invalid identities or other strange failures return SharedPtr<Peer>(); } diff --git a/node/Topology.hpp b/node/Topology.hpp index 63946a32..b6690f58 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -299,7 +299,7 @@ public: Address *a = (Address *)0; SharedPtr<Peer> *p = (SharedPtr<Peer> *)0; while (i.next(a,p)) { - const SharedPtr<Path> pp((*p)->getBestPath(now,false)); + const SharedPtr<Path> pp((*p)->getAppropriatePath(now,false)); if (pp) ++cnt; } diff --git a/node/Trace.cpp b/node/Trace.cpp index 386edaac..cccab9c9 100644 --- a/node/Trace.cpp +++ b/node/Trace.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -40,6 +40,7 @@ #include "Tag.hpp" #include "Capability.hpp" #include "Revocation.hpp" +#include "../include/ZeroTierDebug.h" namespace ZeroTier { @@ -106,6 +107,26 @@ void Trace::peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId, } } +void Trace::peerLinkNowRedundant(void *const tPtr,Peer &peer) +{ + ZT_LOCAL_TRACE(tPtr,RR,"link to peer %.10llx is fully redundant",peer.address().toInt()); +} + +void Trace::peerLinkNoLongerRedundant(void *const tPtr,Peer &peer) +{ + ZT_LOCAL_TRACE(tPtr,RR,"link to peer %.10llx is no longer redundant",peer.address().toInt()); +} + +void Trace::peerLinkAggregateStatistics(void *const tPtr,Peer &peer) +{ + ZT_LOCAL_TRACE(tPtr,RR,"link to peer %.10llx is composed of (%d) physical paths %s, has packet delay variance (%.0f ms), mean latency (%.0f ms)", + peer.address().toInt(), + peer.aggregateLinkPhysicalPathCount(), + peer.interfaceListStr(), + peer.computeAggregateLinkPacketDelayVariance(), + peer.computeAggregateLinkMeanLatency()); +} + void Trace::peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &newPath,const uint64_t packetId) { char tmp[128]; diff --git a/node/Trace.hpp b/node/Trace.hpp index 2a2fca6c..2effb7f0 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -121,6 +121,12 @@ public: void resettingPathsInScope(void *const tPtr,const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope); void peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &path,const uint64_t packetId,const Packet::Verb verb); + + void peerLinkNowRedundant(void *const tPtr,Peer &peer); + void peerLinkNoLongerRedundant(void *const tPtr,Peer &peer); + + void peerLinkAggregateStatistics(void *const tPtr,Peer &peer); + void peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &newPath,const uint64_t packetId); void peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &newPath); diff --git a/node/Utils.cpp b/node/Utils.cpp index a69a575e..16d81f65 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/node/Utils.hpp b/node/Utils.hpp index a24f2c9a..86db8b02 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -38,46 +38,12 @@ #include <vector> #include <map> -#include "Constants.hpp" - -#ifdef __LINUX__ -//#if (defined(_MSC_VER) || defined(__GNUC__)) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64)) -#if 0 -#include <emmintrin.h> -static inline void ZT_FAST_MEMCPY(void *a,const void *b,unsigned long k) -{ - char *aa = reinterpret_cast<char *>(a); - const char *bb = reinterpret_cast<const char *>(b); - while (k >= 64) { - __m128 t1 = _mm_loadu_ps(reinterpret_cast<const float *>(bb)); - __m128 t2 = _mm_loadu_ps(reinterpret_cast<const float *>(bb + 16)); - __m128 t3 = _mm_loadu_ps(reinterpret_cast<const float *>(bb + 32)); - __m128 t4 = _mm_loadu_ps(reinterpret_cast<const float *>(bb + 48)); - _mm_storeu_ps(reinterpret_cast<float *>(aa),t1); - _mm_storeu_ps(reinterpret_cast<float *>(aa + 16),t2); - _mm_storeu_ps(reinterpret_cast<float *>(aa + 32),t3); - _mm_storeu_ps(reinterpret_cast<float *>(aa + 48),t4); - bb += 64; - aa += 64; - k -= 64; - } - while (k >= 16) { - __m128 t1 = _mm_loadu_ps(reinterpret_cast<const float *>(bb)); - _mm_storeu_ps(reinterpret_cast<float *>(aa),t1); - bb += 16; - aa += 16; - k -= 16; - } - for(unsigned long i=0;i<k;++i) - aa[i] = bb[i]; -} -#else -#define ZT_FAST_MEMCPY(a,b,c) memcpy(a,b,c) -#endif -#else -#define ZT_FAST_MEMCPY(a,b,c) memcpy(a,b,c) +#if defined(__FreeBSD__) +#include <sys/endian.h> #endif +#include "Constants.hpp" + namespace ZeroTier { /** @@ -261,6 +227,14 @@ public: return l; } + static inline float normalize(float value, int64_t bigMin, int64_t bigMax, int32_t targetMin, int32_t targetMax) + { + int64_t bigSpan = bigMax - bigMin; + int64_t smallSpan = targetMax - targetMin; + float valueScaled = (value - (float)bigMin) / (float)bigSpan; + return (float)targetMin + valueScaled * (float)smallSpan; + } + /** * Generate secure random bytes * @@ -411,8 +385,12 @@ public: static inline uint64_t hton(uint64_t n) { #if __BYTE_ORDER == __LITTLE_ENDIAN -#if defined(__GNUC__) && (!defined(__OpenBSD__)) +#if defined(__GNUC__) +#if defined(__FreeBSD__) + return bswap64(n); +#elif (!defined(__OpenBSD__)) return __builtin_bswap64(n); +#endif #else return ( ((n & 0x00000000000000FFULL) << 56) | @@ -440,8 +418,12 @@ public: static inline uint64_t ntoh(uint64_t n) { #if __BYTE_ORDER == __LITTLE_ENDIAN -#if defined(__GNUC__) && !defined(__OpenBSD__) +#if defined(__GNUC__) +#if defined(__FreeBSD__) + return bswap64(n); +#elif (!defined(__OpenBSD__)) return __builtin_bswap64(n); +#endif #else return ( ((n & 0x00000000000000FFULL) << 56) | diff --git a/node/World.hpp b/node/World.hpp index 9edf2a6a..823d304d 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -217,8 +217,8 @@ public: _id = b.template at<uint64_t>(p); p += 8; _ts = b.template at<uint64_t>(p); p += 8; - ZT_FAST_MEMCPY(_updatesMustBeSignedBy.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN; - ZT_FAST_MEMCPY(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + memcpy(_updatesMustBeSignedBy.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; const unsigned int numRoots = (unsigned int)b[p++]; if (numRoots > ZT_WORLD_MAX_ROOTS) throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; @@ -30,7 +30,9 @@ ONE_OBJS=\ controller/EmbeddedNetworkController.o \ controller/DB.o \ controller/FileDB.o \ - controller/RethinkDB.o \ + controller/LFDB.o \ + controller/PostgreSQL.o \ + controller/RabbitMQ.o \ osdep/ManagedRoute.o \ osdep/Http.o \ osdep/OSUtils.o \ @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -58,15 +58,18 @@ #include <sys/prctl.h> #include <sys/syscall.h> #include <sys/wait.h> +#ifndef ZT_NO_CAPABILITIES #include <linux/capability.h> #include <linux/securebits.h> #endif #endif +#endif #include <string> #include <stdexcept> #include <iostream> #include <sstream> +#include <algorithm> #include "version.h" #include "include/ZeroTierOne.h" @@ -127,6 +130,7 @@ static void cliPrintHelp(const char *pn,FILE *out) fprintf(out,ZT_EOL_S"Available commands:" ZT_EOL_S); fprintf(out," info - Display status info" ZT_EOL_S); fprintf(out," listpeers - List all peers" ZT_EOL_S); + fprintf(out," peers - List all peers (prettier)" ZT_EOL_S); fprintf(out," listnetworks - List all networks" ZT_EOL_S); fprintf(out," join <network> - Join a network" ZT_EOL_S); fprintf(out," leave <network> - Leave a network" ZT_EOL_S); @@ -135,6 +139,12 @@ static void cliPrintHelp(const char *pn,FILE *out) fprintf(out," listmoons - List moons (federated root sets)" ZT_EOL_S); fprintf(out," orbit <world ID> <seed> - Join a moon via any member root" ZT_EOL_S); fprintf(out," deorbit <world ID> - Leave a moon" ZT_EOL_S); + fprintf(out,ZT_EOL_S"Available settings:" ZT_EOL_S); + fprintf(out," Settings to use with [get/set] may include property names from " ZT_EOL_S); + fprintf(out," the JSON output of \"zerotier-cli -j listnetworks\". Additionally, " ZT_EOL_S); + fprintf(out," (ip, ip4, ip6, ip6plane, and ip6prefix can be used). For instance:" ZT_EOL_S); + fprintf(out," zerotier-cli get <nwid> ip6plane will return the 6PLANE address" ZT_EOL_S); + fprintf(out," assigned to this node." ZT_EOL_S); } static std::string cliFixJsonCRs(const std::string &s) @@ -407,6 +417,73 @@ static int cli(int argc,char **argv) printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; } + } else if (command == "peers") { + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/peer",requestHeaders,responseHeaders,responseBody); + + if (scode == 0) { + printf("Error connecting to the ZeroTier service: %s\n\nPlease check that the service is running and that TCP port 9993 can be contacted via 127.0.0.1." ZT_EOL_S, responseBody.c_str()); + return 1; + } + + nlohmann::json j; + try { + j = OSUtils::jsonParse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + + if (scode == 200) { + if (json) { + printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); + } else { + printf("200 peers\n<ztaddr> <ver> <role> <lat> <link> <lastTX> <lastRX> <path>" ZT_EOL_S); + if (j.is_array()) { + for(unsigned long k=0;k<j.size();++k) { + nlohmann::json &p = j[k]; + std::string bestPath; + nlohmann::json &paths = p["paths"]; + if (paths.is_array()) { + for(unsigned long i=0;i<paths.size();++i) { + nlohmann::json &path = paths[i]; + if (path["preferred"]) { + char tmp[256]; + std::string addr = path["address"]; + const int64_t now = OSUtils::now(); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%-8lld %-8lld %s",now - (int64_t)path["lastSend"],now - (int64_t)path["lastReceive"],addr.c_str()); + bestPath = std::string("DIRECT ") + tmp; + break; + } + } + } + if (bestPath.length() == 0) bestPath = "RELAY"; + char ver[128]; + int64_t vmaj = p["versionMajor"]; + int64_t vmin = p["versionMinor"]; + int64_t vrev = p["versionRev"]; + if (vmaj >= 0) { + OSUtils::ztsnprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + } else { + ver[0] = '-'; + ver[1] = (char)0; + } + printf("%s %-6s %-6s %5d %s" ZT_EOL_S, + OSUtils::jsonString(p["address"],"-").c_str(), + ver, + OSUtils::jsonString(p["role"],"-").c_str(), + (int)OSUtils::jsonInt(p["latency"],0), + bestPath.c_str()); + } + } + } + return 0; + } else { + printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); + return 1; + } } else if (command == "listnetworks") { const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/network",requestHeaders,responseHeaders,responseBody); @@ -466,7 +543,7 @@ static int cli(int argc,char **argv) } } else if (command == "join") { if (arg1.length() != 16) { - printf("invalid network id" ZT_EOL_S); + printf("invalid network id" ZT_EOL_S); return 2; } requestHeaders["Content-Type"] = "application/json"; @@ -494,7 +571,7 @@ static int cli(int argc,char **argv) } } else if (command == "leave") { if (arg1.length() != 16) { - printf("invalid network id" ZT_EOL_S); + printf("invalid network id" ZT_EOL_S); return 2; } unsigned int scode = Http::DEL( @@ -592,7 +669,11 @@ static int cli(int argc,char **argv) } } else if (command == "set") { if (arg1.length() != 16) { - printf("invalid network id" ZT_EOL_S); + fprintf(stderr,"invalid format: must be a 16-digit (network) ID\n"); + return 2; + } + if (!arg2.length()) { + fprintf(stderr,"invalid format: include a property name to set\n"); return 2; } std::size_t eqidx = arg2.find('='); @@ -630,20 +711,18 @@ static int cli(int argc,char **argv) } } else if (command == "get") { if (arg1.length() != 16) { - fprintf(stderr,"invalid network ID format, must be a 16-digit hexidecimal number\n"); + fprintf(stderr,"invalid format: must be a 16-digit (network) ID\n"); + return 2; + } + if (!arg2.length()) { + fprintf(stderr,"invalid format: include a property name to get\n"); return 2; } - char jsons[1024], cl[128]; - OSUtils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); - requestHeaders["Content-Type"] = "application/json"; - requestHeaders["Content-Length"] = cl; const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/network",requestHeaders,responseHeaders,responseBody); - if (scode == 0) { printf("Error connecting to the ZeroTier service: %s\n\nPlease check that the service is running and that TCP port 9993 can be contacted via 127.0.0.1." ZT_EOL_S, responseBody.c_str()); return 1; } - nlohmann::json j; try { j = OSUtils::jsonParse(responseBody); @@ -660,8 +739,56 @@ static int cli(int argc,char **argv) nlohmann::json &n = j[i]; if (n.is_object()) { if (n["id"] == arg1) { - printf("%s\n", OSUtils::jsonString(n[arg2],"-").c_str()); bNetworkFound = true; + std::string aa; + if (arg2 != "ip" && arg2 != "ip4" && arg2 != "ip6" && arg2 != "ip6plane" && arg2 != "ip6prefix") { + aa.append(OSUtils::jsonString(n[arg2],"-")); // Standard network property field + if (aa == "-") { + printf("error, unknown property name\n"); + break; + } + printf("%s\n",aa.c_str()); + break; + } + nlohmann::json &assignedAddresses = n["assignedAddresses"]; + if (assignedAddresses.is_array()) { + int matchingIdxs[ZT_MAX_ZT_ASSIGNED_ADDRESSES]; + int addressCountOfType = 0; + for (int k = 0; k<std::min(ZT_MAX_ZT_ASSIGNED_ADDRESSES, (int)assignedAddresses.size());++k) { + nlohmann::json &addr = assignedAddresses[k]; + if ((arg2 == "ip4" && addr.get<std::string>().find(".") != std::string::npos) + || ((arg2.find("ip6") == 0) && addr.get<std::string>().find(":") != std::string::npos) + || (arg2 == "ip") + ) { + matchingIdxs[addressCountOfType++] = k; + } + } + for (int k=0; k<addressCountOfType; k++) { + nlohmann::json &addr = assignedAddresses[matchingIdxs[k]]; + if (!addr.is_string()) { + continue; + } + if (arg2.find("ip6p") == 0) { + if (arg2 == "ip6plane") { + if (addr.get<std::string>().find("fc") == 0) { + aa.append(addr.get<std::string>().substr(0,addr.get<std::string>().find("/"))); + if (k < addressCountOfType-1) aa.append("\n"); + } + } + if (arg2 == "ip6prefix") { + if (addr.get<std::string>().find("fc") == 0) { + aa.append(addr.get<std::string>().substr(0,addr.get<std::string>().find("/")).substr(0,24)); + if (k < addressCountOfType-1) aa.append("\n"); + } + } + } + else { + aa.append(addr.get<std::string>().substr(0,addr.get<std::string>().find("/"))); + if (k < addressCountOfType-1) aa.append("\n"); + } + } + } + printf("%s\n",aa.c_str()); } } } @@ -1356,12 +1483,14 @@ int main(int argc,char **argv) #ifdef __UNIX_LIKE__ signal(SIGHUP,&_sighandlerHup); signal(SIGPIPE,SIG_IGN); + signal(SIGIO,SIG_IGN); signal(SIGUSR1,SIG_IGN); signal(SIGUSR2,SIG_IGN); signal(SIGALRM,SIG_IGN); signal(SIGINT,&_sighandlerQuit); signal(SIGTERM,&_sighandlerQuit); signal(SIGQUIT,&_sighandlerQuit); + signal(SIGINT,&_sighandlerQuit); /* Ensure that there are no inherited file descriptors open from a previous * incarnation. This is a hack to ensure that GitHub issue #61 or variants diff --git a/osdep/Arp.cpp b/osdep/Arp.cpp index cc4e920a..86f047a8 100644 --- a/osdep/Arp.cpp +++ b/osdep/Arp.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/Arp.hpp b/osdep/Arp.hpp index 27e92fdb..1a21a207 100644 --- a/osdep/Arp.hpp +++ b/osdep/Arp.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index 053df21d..a1e7a945 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/BSDEthernetTap.hpp b/osdep/BSDEthernetTap.hpp index 3d91dbbb..deefd5c7 100644 --- a/osdep/BSDEthernetTap.hpp +++ b/osdep/BSDEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index 691e5281..bf7aef28 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -373,7 +373,7 @@ public: tcps = phy.tcpListen(reinterpret_cast<const struct sockaddr *>(&(ii->first)),(void *)0); if ((udps)&&(tcps)) { #ifdef __LINUX__ - // Bind Linux sockets to their device so routes tha we manage do not override physical routes (wish all platforms had this!) + // Bind Linux sockets to their device so routes that we manage do not override physical routes (wish all platforms had this!) if (ii->second.length() > 0) { char tmp[256]; Utils::scopy(tmp,sizeof(tmp),ii->second.c_str()); @@ -389,6 +389,7 @@ public: _bindings[_bindingCount].udpSock = udps; _bindings[_bindingCount].tcpListenSock = tcps; _bindings[_bindingCount].address = ii->first; + phy.setIfName(udps,(char*)ii->second.c_str(),(int)ii->second.length()); ++_bindingCount; } } else { diff --git a/osdep/BlockingQueue.hpp b/osdep/BlockingQueue.hpp index 351a095a..f8b79342 100644 --- a/osdep/BlockingQueue.hpp +++ b/osdep/BlockingQueue.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -32,6 +32,8 @@ #include <condition_variable> #include <chrono> +#include "Thread.hpp" + namespace ZeroTier { /** @@ -52,11 +54,27 @@ public: c.notify_one(); } + inline void postLimit(T t,const unsigned long limit) + { + std::unique_lock<std::mutex> lock(m); + for(;;) { + if (q.size() < limit) { + q.push(t); + c.notify_one(); + break; + } + if (!r) + break; + gc.wait(lock); + } + } + inline void stop(void) { std::lock_guard<std::mutex> lock(m); r = false; c.notify_all(); + gc.notify_all(); } inline bool get(T &value) @@ -65,10 +83,14 @@ public: if (!r) return false; while (q.empty()) { c.wait(lock); - if (!r) return false; + if (!r) { + gc.notify_all(); + return false; + } } value = q.front(); q.pop(); + gc.notify_all(); return true; } @@ -98,8 +120,8 @@ public: private: volatile bool r; std::queue<T> q; - std::mutex m; - std::condition_variable c; + mutable std::mutex m; + mutable std::condition_variable c,gc; }; } // namespace ZeroTier diff --git a/osdep/Http.cpp b/osdep/Http.cpp index 16785c96..1ab58836 100644 --- a/osdep/Http.cpp +++ b/osdep/Http.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/Http.hpp b/osdep/Http.hpp index b717c5c9..a277bc4e 100644 --- a/osdep/Http.hpp +++ b/osdep/Http.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index 06bbbada..2ea93dd1 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -56,6 +56,7 @@ #include "../node/Dictionary.hpp" #include "OSUtils.hpp" #include "LinuxEthernetTap.hpp" +#include "LinuxNetLink.hpp" // ff:ff:ff:ff:ff:ff with no ADI static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0); @@ -97,6 +98,9 @@ LinuxEthernetTap::LinuxEthernetTap( char procpath[128],nwids[32]; struct stat sbuf; + // ensure netlink connection is started + (void)LinuxNetLink::getInstance(); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally @@ -263,18 +267,8 @@ bool LinuxEthernetTap::enabled() const static bool ___removeIp(const std::string &_dev,const InetAddress &ip) { - long cpid = (long)vfork(); - if (cpid == 0) { - OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); - setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1); - char iptmp[128]; - ::execlp("ip","ip","addr","del",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0); - ::_exit(-1); - } else { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } + LinuxNetLink::getInstance().removeAddress(ip, _dev.c_str()); + return true; } #ifdef __SYNOLOGY__ @@ -285,49 +279,32 @@ bool LinuxEthernetTap::addIpSyn(std::vector<InetAddress> ips) std::string cfg_contents = "DEVICE="+_dev+"\nBOOTPROTO=static"; int ip4=0,ip6=0,ip4_tot=0,ip6_tot=0; - long cpid = (long)vfork(); - if (cpid == 0) { - OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); - setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1); - // We must know if there is at least (one) of each protocol version so we - // can properly enumerate address/netmask combinations in the ifcfg-dev file - for(int i=0; i<(int)ips.size(); i++) { - if (ips[i].isV4()) - ip4_tot++; - else - ip6_tot++; - } - // Assemble and write contents of ifcfg-dev file - for(int i=0; i<(int)ips.size(); i++) { - if (ips[i].isV4()) { - char iptmp[64],iptmp2[64]; - std::string numstr4 = ip4_tot > 1 ? std::to_string(ip4) : ""; - cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString(iptmp) - + "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; - ip4++; - } - else { - char iptmp[64],iptmp2[64]; - std::string numstr6 = ip6_tot > 1 ? std::to_string(ip6) : ""; - cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString(iptmp) - + "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; - ip6++; - } - } - OSUtils::writeFile(filepath.c_str(), cfg_contents.c_str(), cfg_contents.length()); - // Finaly, add IPs - for(int i=0; i<(int)ips.size(); i++){ - char iptmp[128],iptmp2[128]; - if (ips[i].isV4()) - ::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"broadcast",ips[i].broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0); - else - ::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"dev",_dev.c_str(),(const char *)0); + for(int i=0; i<(int)ips.size(); i++) { + if (ips[i].isV4()) + ip4_tot++; + else + ip6_tot++; + } + // Assemble and write contents of ifcfg-dev file + for(int i=0; i<(int)ips.size(); i++) { + if (ips[i].isV4()) { + char iptmp[64],iptmp2[64]; + std::string numstr4 = ip4_tot > 1 ? std::to_string(ip4) : ""; + cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString(iptmp) + + "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; + ip4++; + } else { + char iptmp[64],iptmp2[64]; + std::string numstr6 = ip6_tot > 1 ? std::to_string(ip6) : ""; + cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString(iptmp) + + "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; + ip6++; } - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - return (exitcode == 0); + } + OSUtils::writeFile(filepath.c_str(), cfg_contents.c_str(), cfg_contents.length()); + // Finaly, add IPs + for(int i=0; i<(int)ips.size(); i++){ + LinuxNetLink::getInstance().addAddress(ips[i], _dev.c_str()); } return true; } @@ -348,24 +325,9 @@ bool LinuxEthernetTap::addIp(const InetAddress &ip) ___removeIp(_dev,*i); } - long cpid = (long)vfork(); - if (cpid == 0) { - OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); - setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1); - char iptmp[128],iptmp2[128]; - if (ip.isV4()) { - ::execlp("ip","ip","addr","add",ip.toString(iptmp),"broadcast",ip.broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0); - } else { - ::execlp("ip","ip","addr","add",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0); - } - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } + LinuxNetLink::getInstance().addAddress(ip, _dev.c_str()); - return false; + return true; } bool LinuxEthernetTap::removeIp(const InetAddress &ip) diff --git a/osdep/LinuxEthernetTap.hpp b/osdep/LinuxEthernetTap.hpp index 5142eec1..050bec34 100644 --- a/osdep/LinuxEthernetTap.hpp +++ b/osdep/LinuxEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/LinuxNetLink.cpp b/osdep/LinuxNetLink.cpp new file mode 100644 index 00000000..da75ccdf --- /dev/null +++ b/osdep/LinuxNetLink.cpp @@ -0,0 +1,1077 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#include "LinuxNetLink.hpp" + +#include <unistd.h> +#include <linux/if_tun.h> + +namespace ZeroTier { + +struct nl_route_req { + struct nlmsghdr nl; + struct rtmsg rt; + char buf[8192]; +}; + +struct nl_if_req { + struct nlmsghdr nl; + struct ifinfomsg ifa; + char buf[8192]; +}; + +struct nl_adr_req { + struct nlmsghdr nl; + struct ifaddrmsg ifa; + char buf[8192]; +}; + +LinuxNetLink::LinuxNetLink() + : _t() + , _running(false) + , _routes_ipv4() + , _rv4_m() + , _routes_ipv6() + , _rv6_m() + , _seq(0) + , _interfaces() + , _if_m() + , _fd(socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE)) + , _la({0}) +{ + // set socket timeout to 1 sec so we're not permablocking recv() calls + _setSocketTimeout(_fd, 1); + + _la.nl_family = AF_NETLINK; + _la.nl_pid = getpid()+1; + _la.nl_groups = RTMGRP_LINK|RTMGRP_IPV4_IFADDR|RTMGRP_IPV6_IFADDR|RTMGRP_IPV4_ROUTE|RTMGRP_IPV6_ROUTE|RTMGRP_NOTIFY; + if (bind(_fd, (struct sockaddr*)&_la, sizeof(_la))) { + fprintf(stderr, "Error connecting to RTNETLINK: %s\n", strerror(errno)); + ::exit(1); + } + + _requestIPv4Routes(); + _requestIPv6Routes(); + _requestInterfaceList(); + + _running = true; + _t = Thread::start(this); +} + +LinuxNetLink::~LinuxNetLink() +{ + _running = false; + Thread::join(_t); + ::close(_fd); +} + +void LinuxNetLink::_setSocketTimeout(int fd, int seconds) +{ + struct timeval tv; + tv.tv_sec = seconds; + tv.tv_usec = 0; + if(setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv)) != 0) { +#ifdef ZT_TRACE + fprintf(stderr, "setsockopt failed: %s\n", strerror(errno)); +#endif + } +} + +#define ZT_NL_BUF_SIZE 16384 +int LinuxNetLink::_doRecv(int fd) +{ + char *const buf = (char *)valloc(ZT_NL_BUF_SIZE); + if (!buf) { + fprintf(stderr,"malloc failed!\n"); + ::exit(1); + } + + char *p = NULL; + struct nlmsghdr *nlp; + int nll = 0; + int rtn = 0; + p = buf; + + for(;;) { + rtn = recv(fd, p, ZT_NL_BUF_SIZE - nll, 0); + + if (rtn > 0) { + nlp = (struct nlmsghdr *)p; + + if(nlp->nlmsg_type == NLMSG_ERROR && (nlp->nlmsg_flags & NLM_F_ACK) != NLM_F_ACK) { + struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(nlp); + if (err->error != 0) { +#ifdef ZT_TRACE + //fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error))); +#endif + } + p = buf; + nll = 0; + break; + } + + if (nlp->nlmsg_type == NLMSG_NOOP) { + break; + } + + if( (nlp->nlmsg_flags & NLM_F_MULTI) == NLM_F_MULTI || (nlp->nlmsg_type == NLMSG_DONE)) + { + if (nlp->nlmsg_type == NLMSG_DONE) { + _processMessage(nlp, nll); + p = buf; + nll = 0; + break; + } + p += rtn; + nll += rtn; + } + + if (nlp->nlmsg_type == NLMSG_OVERRUN) { +//#ifdef ZT_TRACE + fprintf(stderr, "NLMSG_OVERRUN: Data lost\n"); +//#endif + p = buf; + nll = 0; + break; + } + + nll += rtn; + + _processMessage(nlp, nll); + + p = buf; + nll = 0; + break; + } else { + break; + } + } + + free(buf); + + return rtn; +} + +void LinuxNetLink::threadMain() throw() +{ + int rtn = 0; + + while(_running) { + rtn = _doRecv(_fd); + if (rtn <= 0) { + Thread::sleep(100); + continue; + } + } +} + +void LinuxNetLink::_processMessage(struct nlmsghdr *nlp, int nll) +{ + for(; NLMSG_OK(nlp, nll); nlp=NLMSG_NEXT(nlp, nll)) + { + switch(nlp->nlmsg_type) + { + case RTM_NEWLINK: + _linkAdded(nlp); + break; + case RTM_DELLINK: + _linkDeleted(nlp); + break; + case RTM_NEWADDR: + _ipAddressAdded(nlp); + break; + case RTM_DELADDR: + _ipAddressDeleted(nlp); + break; + case RTM_NEWROUTE: + _routeAdded(nlp); + break; + case RTM_DELROUTE: + _routeDeleted(nlp); + break; + default: + break; + } + } +} + +void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp) +{ + struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp); + struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap); + int ifal = IFA_PAYLOAD(nlp); + + char addr[40] = {0}; + char local[40] = {0}; + char label[40] = {0}; + char bcast[40] = {0}; + + for(;RTA_OK(rtap, ifal); rtap=RTA_NEXT(rtap,ifal)) + { + switch(rtap->rta_type) { + case IFA_ADDRESS: + inet_ntop(ifap->ifa_family, RTA_DATA(rtap), addr, 40); + break; + case IFA_LOCAL: + inet_ntop(ifap->ifa_family, RTA_DATA(rtap), local, 40); + break; + case IFA_LABEL: + memcpy(label, RTA_DATA(rtap), 40); + break; + case IFA_BROADCAST: + inet_ntop(ifap->ifa_family, RTA_DATA(rtap), bcast, 40); + break; + } + } + +#ifdef ZT_TRACE + //fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); +#endif +} + +void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp) +{ + struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp); + struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap); + int ifal = IFA_PAYLOAD(nlp); + + char addr[40] = {0}; + char local[40] = {0}; + char label[40] = {0}; + char bcast[40] = {0}; + + for(;RTA_OK(rtap, ifal); rtap=RTA_NEXT(rtap,ifal)) + { + switch(rtap->rta_type) { + case IFA_ADDRESS: + inet_ntop(ifap->ifa_family, RTA_DATA(rtap), addr, 40); + break; + case IFA_LOCAL: + inet_ntop(ifap->ifa_family, RTA_DATA(rtap), local, 40); + break; + case IFA_LABEL: + memcpy(label, RTA_DATA(rtap), 40); + break; + case IFA_BROADCAST: + inet_ntop(ifap->ifa_family, RTA_DATA(rtap), bcast, 40); + break; + } + } + +#ifdef ZT_TRACE + //fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); +#endif +} + +void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp) +{ + char dsts[40] = {0}; + char gws[40] = {0}; + char srcs[40] = {0}; + char ifs[16] = {0}; + char ms[24] = {0}; + + struct rtmsg *rtp = (struct rtmsg *)NLMSG_DATA(nlp); + struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp); + int rtl = RTM_PAYLOAD(nlp); + + for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl)) + { + switch(rtap->rta_type) + { + case RTA_DST: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40); + break; + case RTA_SRC: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24: 40); + break; + case RTA_GATEWAY: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40); + break; + case RTA_OIF: + sprintf(ifs, "%d", *((int*)RTA_DATA(rtap))); + break; + } + } + sprintf(ms, "%d", rtp->rtm_dst_len); + +#ifdef ZT_TRACE + //fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); +#endif +} + +void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp) +{ + char dsts[40] = {0}; + char gws[40] = {0}; + char srcs[40] = {0}; + char ifs[16] = {0}; + char ms[24] = {0}; + + struct rtmsg *rtp = (struct rtmsg *) NLMSG_DATA(nlp); + struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp); + int rtl = RTM_PAYLOAD(nlp); + + for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl)) + { + switch(rtap->rta_type) + { + case RTA_DST: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40); + break; + case RTA_SRC: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24 : 40); + break; + case RTA_GATEWAY: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40); + break; + case RTA_OIF: + sprintf(ifs, "%d", *((int*)RTA_DATA(rtap))); + break; + } + } + sprintf(ms, "%d", rtp->rtm_dst_len); + +#ifdef ZT_TRACE + //fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); +#endif +} + +void LinuxNetLink::_linkAdded(struct nlmsghdr *nlp) +{ + unsigned char mac_bin[6] = {0}; + unsigned int mtu = 0; + char ifname[IFNAMSIZ] = {0}; + + struct ifinfomsg *ifip = (struct ifinfomsg *)NLMSG_DATA(nlp); + struct rtattr *rtap = (struct rtattr *)IFLA_RTA(ifip); + int ifil = RTM_PAYLOAD(nlp); + + const char *ptr = (const char *)0; + for(;RTA_OK(rtap, ifil);rtap=RTA_NEXT(rtap, ifil)) + { + switch(rtap->rta_type) { + case IFLA_ADDRESS: + ptr = (const char *)RTA_DATA(rtap); + memcpy(mac_bin, ptr, 6); + break; + case IFLA_IFNAME: + ptr = (const char *)RTA_DATA(rtap); + memcpy(ifname, ptr, strlen(ptr)); + break; + case IFLA_MTU: + memcpy(&mtu, RTA_DATA(rtap), sizeof(unsigned int)); + break; + } + } + + { + Mutex::Lock l(_if_m); + struct iface_entry &entry = _interfaces[ifip->ifi_index]; + entry.index = ifip->ifi_index; + memcpy(entry.ifacename, ifname, sizeof(ifname)); + snprintf(entry.mac,sizeof(entry.mac),"%.02x:%.02x:%.02x:%.02x:%.02x:%.02x",(unsigned int)mac_bin[0],(unsigned int)mac_bin[1],(unsigned int)mac_bin[2],(unsigned int)mac_bin[3],(unsigned int)mac_bin[4],(unsigned int)mac_bin[5]); + memcpy(entry.mac_bin, mac_bin, 6); + entry.mtu = mtu; + } +} + +void LinuxNetLink::_linkDeleted(struct nlmsghdr *nlp) +{ + unsigned int mtu = 0; + char ifname[40] = {0}; + + struct ifinfomsg *ifip = (struct ifinfomsg *)NLMSG_DATA(nlp); + struct rtattr *rtap = (struct rtattr *)IFLA_RTA(ifip); + int ifil = RTM_PAYLOAD(nlp); + + const char *ptr = (const char *)0; + for(;RTA_OK(rtap, ifil);rtap=RTA_NEXT(rtap, ifil)) + { + switch(rtap->rta_type) { + case IFLA_IFNAME: + ptr = (const char*)RTA_DATA(rtap); + memcpy(ifname, ptr, strlen(ptr)); + break; + case IFLA_MTU: + memcpy(&mtu, RTA_DATA(rtap), sizeof(unsigned int)); + break; + } + } + + { + Mutex::Lock l(_if_m); + if(_interfaces.contains(ifip->ifi_index)) { + _interfaces.erase(ifip->ifi_index); + } + } +} + +void LinuxNetLink::_requestIPv4Routes() +{ + int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd == -1) { + fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno)); + return; + } + + _setSocketTimeout(fd); + + struct sockaddr_nl la; + la.nl_family = AF_NETLINK; + la.nl_pid = getpid(); + la.nl_groups = RTMGRP_IPV4_ROUTE; + if(bind(fd, (struct sockaddr*)&la, sizeof(la))) { + fprintf(stderr, "Error binding RTNETLINK (_requiestIPv4Routes #1): %s\n", strerror(errno)); + close(fd); + return; + } + + struct nl_route_req req; + bzero(&req, sizeof(req)); + req.nl.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; + req.nl.nlmsg_type = RTM_GETROUTE; + req.nl.nlmsg_pid = 0; + req.nl.nlmsg_seq = ++_seq; + req.rt.rtm_family = AF_INET; + req.rt.rtm_table = RT_TABLE_MAIN; + + struct sockaddr_nl pa; + bzero(&pa, sizeof(pa)); + pa.nl_family = AF_NETLINK; + + struct msghdr msg; + bzero(&msg, sizeof(msg)); + msg.msg_name = (void*)&pa; + msg.msg_namelen = sizeof(pa); + + struct iovec iov; + bzero(&iov, sizeof(iov)); + iov.iov_base = (void*)&req.nl; + iov.iov_len = req.nl.nlmsg_len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + sendmsg(fd, &msg, 0); + + _doRecv(fd); + + close(fd); +} + +void LinuxNetLink::_requestIPv6Routes() +{ + int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd == -1) { + fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno)); + return; + } + + _setSocketTimeout(fd); + + struct sockaddr_nl la; + la.nl_family = AF_NETLINK; + la.nl_pid = getpid(); + la.nl_groups = RTMGRP_IPV6_ROUTE; + if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) { + fprintf(stderr, "Error binding RTNETLINK (_requestIPv6Routes #1): %s\n", strerror(errno)); + close(fd); + return; + } + + struct nl_route_req req; + bzero(&req, sizeof(req)); + req.nl.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; + req.nl.nlmsg_type = RTM_GETROUTE; + req.nl.nlmsg_pid = 0; + req.nl.nlmsg_seq = ++_seq; + req.rt.rtm_family = AF_INET6; + req.rt.rtm_table = RT_TABLE_MAIN; + + struct sockaddr_nl pa; + bzero(&pa, sizeof(pa)); + pa.nl_family = AF_NETLINK; + + struct msghdr msg; + bzero(&msg, sizeof(msg)); + msg.msg_name = (void*)&pa; + msg.msg_namelen = sizeof(pa); + + struct iovec iov; + bzero(&iov, sizeof(iov)); + iov.iov_base = (void*)&req.nl; + iov.iov_len = req.nl.nlmsg_len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + sendmsg(fd, &msg, 0); + + _doRecv(fd); + + close(fd); +} + +void LinuxNetLink::_requestInterfaceList() +{ + int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd == -1) { + fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno)); + return; + } + + _setSocketTimeout(fd); + + struct sockaddr_nl la; + la.nl_family = AF_NETLINK; + la.nl_pid = getpid(); + la.nl_groups = RTMGRP_LINK; + if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) { + fprintf(stderr, "Error binding RTNETLINK (_requestInterfaceList #1): %s\n", strerror(errno)); + close(fd); + return; + } + + struct nl_if_req req; + bzero(&req, sizeof(req)); + req.nl.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)); + req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; + req.nl.nlmsg_type = RTM_GETLINK; + req.nl.nlmsg_pid = 0; + req.nl.nlmsg_seq = ++_seq; + req.ifa.ifi_family = AF_UNSPEC; + + struct sockaddr_nl pa; + bzero(&pa, sizeof(pa)); + pa.nl_family = AF_NETLINK; + + struct msghdr msg; + bzero(&msg, sizeof(msg)); + msg.msg_name = (void*)&pa; + msg.msg_namelen = sizeof(pa); + + struct iovec iov; + bzero(&iov, sizeof(iov)); + iov.iov_base = (void*)&req.nl; + iov.iov_len = req.nl.nlmsg_len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + sendmsg(fd, &msg, 0); + + _doRecv(fd); + + close(fd); +} + +void LinuxNetLink::addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName) +{ + if (!target) return; + + int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd == -1) { + fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno)); + return; + } + + _setSocketTimeout(fd); + + struct sockaddr_nl la; + bzero(&la, sizeof(la)); + la.nl_family = AF_NETLINK; + la.nl_pid = getpid(); + + if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) { + fprintf(stderr, "Error binding RTNETLINK (addRoute #1): %s\n", strerror(errno)); + close(fd); + return; + } + +#ifdef ZT_TRACE + //char tmp[64]; + //char tmp2[64]; + //char tmp3[64]; + //fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName); +#endif + + int rtl = sizeof(struct rtmsg); + struct nl_route_req req; + bzero(&req, sizeof(req)); + + struct rtattr *rtap = (struct rtattr *)req.buf; + rtap->rta_type = RTA_DST; + if (target.isV4()) { + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&target)->sin_addr, sizeof(struct in_addr)); + } else { + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&target)->sin6_addr, sizeof(struct in6_addr)); + } + rtl += rtap->rta_len; + + if(via) { + rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len); + rtap->rta_type = RTA_GATEWAY; + if(via.isV4()) { + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&via)->sin_addr, sizeof(struct in_addr)); + } else { + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&via)->sin6_addr, sizeof(struct in6_addr)); + } + rtl += rtap->rta_len; + } else if (src) { + rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len); + rtap->rta_type = RTA_SRC; + if(src.isV4()) { + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&src)->sin_addr, sizeof(struct in_addr)); + + } else { + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&src)->sin6_addr, sizeof(struct in6_addr)); + } + req.rt.rtm_src_len = src.netmaskBits(); + } + + if (ifaceName != NULL) { + int interface_index = _indexForInterface(ifaceName); + if (interface_index != -1) { + rtap = (struct rtattr *) (((char*)rtap) + rtap->rta_len); + rtap->rta_type = RTA_OIF; + rtap->rta_len = RTA_LENGTH(sizeof(int)); + memcpy(RTA_DATA(rtap), &interface_index, sizeof(int)); + rtl += rtap->rta_len; + } + } + + req.nl.nlmsg_len = NLMSG_LENGTH(rtl); + req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_EXCL | NLM_F_CREATE | NLM_F_ACK; + req.nl.nlmsg_type = RTM_NEWROUTE; + req.nl.nlmsg_pid = 0; + req.nl.nlmsg_seq = ++_seq; + req.rt.rtm_family = target.ss_family; + req.rt.rtm_table = RT_TABLE_MAIN; + req.rt.rtm_protocol = RTPROT_STATIC; + req.rt.rtm_scope = RT_SCOPE_UNIVERSE; + req.rt.rtm_type = RTN_UNICAST; + req.rt.rtm_dst_len = target.netmaskBits(); + req.rt.rtm_flags = 0; + + struct sockaddr_nl pa; + bzero(&pa, sizeof(pa)); + pa.nl_family = AF_NETLINK; + + struct msghdr msg; + bzero(&msg, sizeof(msg)); + msg.msg_name = (void*)&pa; + msg.msg_namelen = sizeof(pa); + + struct iovec iov; + bzero(&iov, sizeof(iov)); + iov.iov_base = (void*)&req.nl; + iov.iov_len = req.nl.nlmsg_len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + sendmsg(fd, &msg, 0); + + _doRecv(fd); + + close(fd); +} + +void LinuxNetLink::delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName) +{ + if (!target) return; + + int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd == -1) { + fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno)); + return; + } + + _setSocketTimeout(fd); + + struct sockaddr_nl la; + la.nl_family = AF_NETLINK; + la.nl_pid = getpid(); + + if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) { + fprintf(stderr, "Error binding RTNETLINK (delRoute #1): %s\n", strerror(errno)); + close(fd); + return; + } + +#ifdef ZT_TRACE + //char tmp[64]; + //char tmp2[64]; + //char tmp3[64]; + //fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName); +#endif + + int rtl = sizeof(struct rtmsg); + struct nl_route_req req; + bzero(&req, sizeof(req)); + + struct rtattr *rtap = (struct rtattr *)req.buf; + rtap->rta_type = RTA_DST; + if (target.isV4()) { + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&target)->sin_addr, sizeof(struct in_addr)); + } else { + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&target)->sin6_addr, sizeof(struct in6_addr)); + } + rtl += rtap->rta_len; + + if(via) { + rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len); + rtap->rta_type = RTA_GATEWAY; + if(via.isV4()) { + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&via)->sin_addr, sizeof(struct in_addr)); + } else { + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&via)->sin6_addr, sizeof(struct in6_addr)); + } + rtl += rtap->rta_len; + } else if (src) { + rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len); + rtap->rta_type = RTA_SRC; + if(src.isV4()) { + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&src)->sin_addr, sizeof(struct in_addr)); + + } else { + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&src)->sin6_addr, sizeof(struct in6_addr)); + } + req.rt.rtm_src_len = src.netmaskBits(); + } + + if (ifaceName != NULL) { + int interface_index = _indexForInterface(ifaceName); + if (interface_index != -1) { + rtap = (struct rtattr *) (((char*)rtap) + rtap->rta_len); + rtap->rta_type = RTA_OIF; + rtap->rta_len = RTA_LENGTH(sizeof(int)); + memcpy(RTA_DATA(rtap), &interface_index, sizeof(int)); + rtl += rtap->rta_len; + } + } + + req.nl.nlmsg_len = NLMSG_LENGTH(rtl); + req.nl.nlmsg_flags = NLM_F_REQUEST; + req.nl.nlmsg_type = RTM_DELROUTE; + req.nl.nlmsg_pid = 0; + req.nl.nlmsg_seq = ++_seq; + req.rt.rtm_family = target.ss_family; + req.rt.rtm_table = RT_TABLE_MAIN; + req.rt.rtm_protocol = RTPROT_STATIC; + req.rt.rtm_scope = RT_SCOPE_UNIVERSE; + req.rt.rtm_type = RTN_UNICAST; + req.rt.rtm_dst_len = target.netmaskBits(); + req.rt.rtm_flags = 0; + + struct sockaddr_nl pa; + bzero(&pa, sizeof(pa)); + pa.nl_family = AF_NETLINK; + + struct msghdr msg; + bzero(&msg, sizeof(msg)); + msg.msg_name = (void*)&pa; + msg.msg_namelen = sizeof(pa); + + struct iovec iov; + bzero(&iov, sizeof(iov)); + iov.iov_base = (void*)&req.nl; + iov.iov_len = req.nl.nlmsg_len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + sendmsg(fd, &msg, 0); + + _doRecv(fd); + + close(fd); +} + +void LinuxNetLink::addAddress(const InetAddress &addr, const char *iface) +{ + int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd == -1) { + fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno)); + return; + } + + _setSocketTimeout(fd); + + struct sockaddr_nl la; + memset(&la,0,sizeof(la)); + la.nl_family = AF_NETLINK; + la.nl_pid = getpid(); + if (addr.isV4()) { + la.nl_groups = RTMGRP_IPV4_IFADDR; + } else { + la.nl_groups = RTMGRP_IPV6_IFADDR; + } + + if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) { + fprintf(stderr, "Error binding RTNETLINK (addAddress #1): %s\n", strerror(errno)); + close(fd); + return; + } + +#ifdef ZT_TRACE + //char tmp[128]; + //fprintf(stderr, "Adding IP address %s to interface %s", addr.toString(tmp), iface); +#endif + + int interface_index = _indexForInterface(iface); + for (int reps = 0; interface_index == -1 && reps < 10; ++reps) { + Thread::sleep(100); + interface_index = _indexForInterface(iface); + } + + if (interface_index == -1) { + fprintf(stderr, "Unable to find index for interface %s\n", iface); + close(fd); + return; + } + + int rtl = sizeof(struct ifaddrmsg); + struct nl_adr_req req; + bzero(&req, sizeof(struct nl_adr_req)); + + struct rtattr *rtap = (struct rtattr *)req.buf;; + if(addr.isV4()) { + struct sockaddr_in *addr_v4 = (struct sockaddr_in*)&addr; + rtap->rta_type = IFA_ADDRESS; + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr)); + rtl += rtap->rta_len; + + rtap = (struct rtattr*)(((char*)rtap) + rtap->rta_len); + rtap->rta_type = IFA_LOCAL; + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr)); + rtl += rtap->rta_len; + + InetAddress broadcast = addr.broadcast(); + if(broadcast) { + rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len); + struct sockaddr_in *bcast = (struct sockaddr_in*)&broadcast; + rtap->rta_type = IFA_BROADCAST; + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &bcast->sin_addr, sizeof(struct in_addr)); + rtl += rtap->rta_len; + } + } else { //V6 + rtap->rta_type = IFA_ADDRESS; + struct sockaddr_in6 *addr_v6 = (struct sockaddr_in6*)&addr; + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &addr_v6->sin6_addr, sizeof(struct in6_addr)); + rtl += rtap->rta_len; + } + + if (iface) { + rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len); + rtap->rta_type = IFA_LABEL; + rtap->rta_len = RTA_LENGTH(strlen(iface)); + memcpy(RTA_DATA(rtap), iface, strlen(iface)); + rtl += rtap->rta_len; + } + + req.nl.nlmsg_len = NLMSG_LENGTH(rtl); + req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL; + req.nl.nlmsg_type = RTM_NEWADDR; + req.nl.nlmsg_pid = 0; + req.nl.nlmsg_seq = ++_seq; + req.ifa.ifa_family = addr.ss_family; + req.ifa.ifa_prefixlen = addr.port(); + req.ifa.ifa_flags = IFA_F_PERMANENT; + req.ifa.ifa_scope = 0; + req.ifa.ifa_index = interface_index; + + struct sockaddr_nl pa; + bzero(&pa, sizeof(sockaddr_nl)); + pa.nl_family = AF_NETLINK; + + struct msghdr msg; + bzero(&msg, sizeof(msg)); + msg.msg_name = (void*)&pa; + msg.msg_namelen = sizeof(pa); + + struct iovec iov; + iov.iov_base = (void*)&req.nl; + iov.iov_len = req.nl.nlmsg_len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + sendmsg(fd, &msg, 0); + + _doRecv(fd); + + close(fd); +} + +void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface) +{ + int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd == -1) { + fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno)); + return; + } + + _setSocketTimeout(fd); + + struct sockaddr_nl la; + la.nl_family = AF_NETLINK; + la.nl_pid = getpid(); + if (addr.isV4()) { + la.nl_groups = RTMGRP_IPV4_IFADDR; + } else { + la.nl_groups = RTMGRP_IPV6_IFADDR; + } + if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) { + fprintf(stderr, "Error binding RTNETLINK (removeAddress #1): %s\n", strerror(errno)); + close(fd); + return; + } + +#ifdef ZT_TRACE + //char tmp[128]; + //fprintf(stderr, "Removing IP address %s from interface %s", addr.toString(tmp), iface); +#endif + + int interface_index = _indexForInterface(iface); + + if (interface_index == -1) { + fprintf(stderr, "Unable to find index for interface %s\n", iface); + close(fd); + return; + } + + int rtl = sizeof(struct ifaddrmsg); + struct nl_adr_req req; + bzero(&req, sizeof(struct nl_adr_req)); + + struct rtattr *rtap = (struct rtattr *)req.buf; + if(addr.isV4()) { + struct sockaddr_in *addr_v4 = (struct sockaddr_in*)&addr; + rtap->rta_type = IFA_ADDRESS; + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr)); + rtl += rtap->rta_len; + + rtap = (struct rtattr*)(((char*)rtap) + rtap->rta_len); + rtap->rta_type = IFA_LOCAL; + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr)); + rtl += rtap->rta_len; + + InetAddress broadcast = addr.broadcast(); + if(broadcast) { + rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len); + struct sockaddr_in *bcast = (struct sockaddr_in*)&broadcast; + rtap->rta_type = IFA_BROADCAST; + rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr)); + memcpy(RTA_DATA(rtap), &bcast->sin_addr, sizeof(struct in_addr)); + rtl += rtap->rta_len; + } + } else { //V6 + rtap->rta_type = IFA_ADDRESS; + struct sockaddr_in6 *addr_v6 = (struct sockaddr_in6*)&addr; + rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr)); + memcpy(RTA_DATA(rtap), &addr_v6->sin6_addr, sizeof(struct in6_addr)); + rtl += rtap->rta_len; + } + + if (iface) { + rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len); + rtap->rta_type = IFA_LABEL; + rtap->rta_len = RTA_LENGTH(strlen(iface)); + memcpy(RTA_DATA(rtap), iface, strlen(iface)); + rtl += rtap->rta_len; + } + + req.nl.nlmsg_len = NLMSG_LENGTH(rtl); + req.nl.nlmsg_flags = NLM_F_REQUEST; + req.nl.nlmsg_type = RTM_DELADDR; + req.nl.nlmsg_pid = 0; + req.nl.nlmsg_seq = ++_seq; + req.ifa.ifa_family = addr.ss_family; + req.ifa.ifa_prefixlen = addr.port(); + req.ifa.ifa_flags = IFA_F_PERMANENT; + req.ifa.ifa_scope = 0; + req.ifa.ifa_index = interface_index; + + struct sockaddr_nl pa; + bzero(&pa, sizeof(sockaddr_nl)); + pa.nl_family = AF_NETLINK; + + struct msghdr msg; + bzero(&msg, sizeof(msg)); + msg.msg_name = (void*)&pa; + msg.msg_namelen = sizeof(pa); + + struct iovec iov; + iov.iov_base = (void*)&req.nl; + iov.iov_len = req.nl.nlmsg_len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + sendmsg(fd, &msg, 0); + + _doRecv(fd); + + close(fd); +} + +RouteList LinuxNetLink::getIPV4Routes() const +{ + return _routes_ipv4; +} + +RouteList LinuxNetLink::getIPV6Routes() const +{ + return _routes_ipv6; +} + +int LinuxNetLink::_indexForInterface(const char *iface) +{ + Mutex::Lock l(_if_m); + int interface_index = -1; + Hashtable<int, iface_entry>::Iterator iter(_interfaces); + int *k = NULL; + iface_entry *v = NULL; + while(iter.next(k,v)) { + if(strcmp(iface, v->ifacename) == 0) { + interface_index = v->index; + break; + } + } + return interface_index; +} + +} // namespace ZeroTier diff --git a/osdep/LinuxNetLink.hpp b/osdep/LinuxNetLink.hpp new file mode 100644 index 00000000..b0bf24b3 --- /dev/null +++ b/osdep/LinuxNetLink.hpp @@ -0,0 +1,129 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_LINUX_NETLINK_HPP +#define ZT_LINUX_NETLINK_HPP + +#include <vector> + +#include <sys/socket.h> +#include <asm/types.h> +#include <linux/rtnetlink.h> +#include <sys/socket.h> +#include <linux/if.h> + +#include "../node/InetAddress.hpp" +#include "../node/MAC.hpp" +#include "Thread.hpp" +#include "../node/Hashtable.hpp" +#include "../node/Mutex.hpp" + + +namespace ZeroTier { + +struct route_entry { + InetAddress target; + InetAddress via; + int if_index; + char iface[IFNAMSIZ]; +}; +typedef std::vector<route_entry> RouteList; + +/** + * Interface with Linux's RTNETLINK + */ +class LinuxNetLink +{ +private: + LinuxNetLink(); + ~LinuxNetLink(); + +public: + static LinuxNetLink& getInstance() + { + static LinuxNetLink instance; + return instance; + } + + LinuxNetLink(LinuxNetLink const&) = delete; + void operator=(LinuxNetLink const&) = delete; + + void addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName); + void delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName); + RouteList getIPV4Routes() const; + RouteList getIPV6Routes() const; + + void addAddress(const InetAddress &addr, const char *iface); + void removeAddress(const InetAddress &addr, const char *iface); + + void threadMain() throw(); +private: + int _doRecv(int fd); + + void _processMessage(struct nlmsghdr *nlp, int nll); + void _routeAdded(struct nlmsghdr *nlp); + void _routeDeleted(struct nlmsghdr *nlp); + void _linkAdded(struct nlmsghdr *nlp); + void _linkDeleted(struct nlmsghdr *nlp); + void _ipAddressAdded(struct nlmsghdr *nlp); + void _ipAddressDeleted(struct nlmsghdr *nlp); + + void _requestInterfaceList(); + void _requestIPv4Routes(); + void _requestIPv6Routes(); + + int _indexForInterface(const char *iface); + + void _setSocketTimeout(int fd, int seconds = 1); + + Thread _t; + bool _running; + + RouteList _routes_ipv4; + Mutex _rv4_m; + RouteList _routes_ipv6; + Mutex _rv6_m; + + uint32_t _seq; + + struct iface_entry { + int index; + char ifacename[IFNAMSIZ]; + char mac[18]; + char mac_bin[6]; + unsigned int mtu; + }; + Hashtable<int, iface_entry> _interfaces; + Mutex _if_m; + + // socket communication vars; + int _fd; + struct sockaddr_nl _la; +}; + +} + +#endif // ZT_LINUX_NETLINK_HPPS
\ No newline at end of file diff --git a/osdep/MacEthernetTap.cpp b/osdep/MacEthernetTap.cpp new file mode 100644 index 00000000..0f534299 --- /dev/null +++ b/osdep/MacEthernetTap.cpp @@ -0,0 +1,464 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +#include <unistd.h> +#include <signal.h> + +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/ioctl.h> +#include <sys/wait.h> +#include <sys/select.h> +#include <sys/cdefs.h> +#include <sys/uio.h> +#include <sys/param.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <net/route.h> +#include <net/if.h> +#include <net/if_dl.h> +#include <sys/sysctl.h> +#include <ifaddrs.h> + +#include <string> +#include <map> +#include <set> +#include <algorithm> + +#include "../node/Constants.hpp" +#include "../node/Utils.hpp" +#include "../node/Mutex.hpp" +#include "../node/Dictionary.hpp" +#include "OSUtils.hpp" +#include "MacEthernetTap.hpp" +#include "MacEthernetTapAgent.h" + +static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0); + +namespace ZeroTier { + +static Mutex globalTapCreateLock; +static bool globalTapInitialized = false; + +MacEthernetTap::MacEthernetTap( + const char *homePath, + const MAC &mac, + unsigned int mtu, + unsigned int metric, + uint64_t nwid, + const char *friendlyName, + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), + void *arg) : + _handler(handler), + _arg(arg), + _nwid(nwid), + _homePath(homePath), + _mtu(mtu), + _metric(metric), + _agentStdin(-1), + _agentStdout(-1), + _agentStderr(-1), + _agentStdin2(-1), + _agentStdout2(-1), + _agentStderr2(-1), + _agentPid(-1), + _enabled(true) +{ + char ethaddr[64],mtustr[16],devnostr[16],devstr[16],metricstr[16]; + OSUtils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + OSUtils::ztsnprintf(mtustr,sizeof(mtustr),"%u",mtu); + OSUtils::ztsnprintf(metricstr,sizeof(metricstr),"%u",metric); + + std::string agentPath(homePath); + agentPath.push_back(ZT_PATH_SEPARATOR); + agentPath.append("MacEthernetTapAgent"); + if (!OSUtils::fileExists(agentPath.c_str())) + throw std::runtime_error("MacEthernetTapAgent not installed in ZeroTier home"); + + Mutex::Lock _gl(globalTapCreateLock); // only make one at a time + + // Destroy all feth devices on first tap start in case ZeroTier did not exit cleanly last time. + // We leave interfaces less than feth100 alone in case something else is messing with feth devices. + if (!globalTapInitialized) { + globalTapInitialized = true; + struct ifaddrs *ifa = (struct ifaddrs *)0; + std::set<std::string> deleted; + if (!getifaddrs(&ifa)) { + struct ifaddrs *p = ifa; + while (p) { + if ((!strncmp(p->ifa_name,"feth",4))&&(strlen(p->ifa_name) >= 7)&&(deleted.count(std::string(p->ifa_name)) == 0)) { + deleted.insert(std::string(p->ifa_name)); + const char *args[4]; + args[0] = "/sbin/ifconfig"; + args[1] = p->ifa_name; + args[2] = "destroy"; + args[3] = (char *)0; + const pid_t pid = vfork(); + if (pid == 0) { + execv(args[0],const_cast<char **>(args)); + _exit(-1); + } else if (pid > 0) { + int rv = 0; + waitpid(pid,&rv,0); + } + } + p = p->ifa_next; + } + freeifaddrs(ifa); + } + } + + unsigned int devNo = 100 + ((nwid ^ (nwid >> 32) ^ (nwid >> 48)) % 4900); + for(;;) { + OSUtils::ztsnprintf(devnostr,sizeof(devnostr),"%u",devNo); + OSUtils::ztsnprintf(devstr,sizeof(devstr),"feth%u",devNo); + bool duplicate = false; + struct ifaddrs *ifa = (struct ifaddrs *)0; + if (!getifaddrs(&ifa)) { + struct ifaddrs *p = ifa; + while (p) { + if (!strcmp(p->ifa_name,devstr)) { + duplicate = true; + break; + } + p = p->ifa_next; + } + freeifaddrs(ifa); + } + if (duplicate) { + devNo = (devNo + 1) % 5000; + if (devNo < 100) + devNo = 100; + } else { + break; + } + } + + if (::pipe(_shutdownSignalPipe)) + throw std::runtime_error("pipe creation failed"); + + int agentStdin[2]; + int agentStdout[2]; + int agentStderr[2]; + if (::pipe(agentStdin)) + throw std::runtime_error("pipe creation failed"); + if (::pipe(agentStdout)) + throw std::runtime_error("pipe creation failed"); + if (::pipe(agentStderr)) + throw std::runtime_error("pipe creation failed"); + _agentStdin = agentStdin[1]; + _agentStdout = agentStdout[0]; + _agentStderr = agentStderr[0]; + _agentStdin2 = agentStdin[0]; + _agentStdout2 = agentStdout[1]; + _agentStderr2 = agentStderr[1]; + long apid = (long)fork(); + if (apid < 0) { + throw std::runtime_error("fork failed"); + } else if (apid == 0) { + ::dup2(agentStdin[0],STDIN_FILENO); + ::dup2(agentStdout[1],STDOUT_FILENO); + ::dup2(agentStderr[1],STDERR_FILENO); + ::close(agentStdin[0]); + ::close(agentStdin[1]); + ::close(agentStdout[0]); + ::close(agentStdout[1]); + ::close(agentStderr[0]); + ::close(agentStderr[1]); + ::execl(agentPath.c_str(),agentPath.c_str(),devnostr,ethaddr,mtustr,metricstr,(char *)0); + ::_exit(-1); + } else { + _agentPid = apid; + } + Thread::sleep(100); // this causes them to come up in a more user-friendly order on launch + + _thread = Thread::start(this); +} + +MacEthernetTap::~MacEthernetTap() +{ + Mutex::Lock _gl(globalTapCreateLock); + ::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit + Thread::join(_thread); + ::close(_shutdownSignalPipe[0]); + ::close(_shutdownSignalPipe[1]); + int ec = 0; + ::kill(_agentPid,SIGTERM); + ::waitpid(_agentPid,&ec,0); + ::close(_agentStdin); + ::close(_agentStdout); + ::close(_agentStderr); + ::close(_agentStdin2); + ::close(_agentStdout2); + ::close(_agentStderr2); +} + +void MacEthernetTap::setEnabled(bool en) { _enabled = en; } +bool MacEthernetTap::enabled() const { return _enabled; } + +bool MacEthernetTap::addIp(const InetAddress &ip) +{ + char tmp[128]; + + if (!ip) + return false; + + std::string cmd; + cmd.push_back((char)ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG); + cmd.append((ip.ss_family == AF_INET6) ? "inet6" : "inet"); + cmd.push_back(0); + cmd.append(ip.toString(tmp)); + cmd.push_back(0); + cmd.append("alias"); + cmd.push_back(0); + + uint16_t l = (uint16_t)cmd.length(); + _putLock.lock(); + write(_agentStdin,&l,2); + write(_agentStdin,cmd.data(),cmd.length()); + _putLock.unlock(); + + return true; +} + +bool MacEthernetTap::removeIp(const InetAddress &ip) +{ + char tmp[128]; + + if (!ip) + return false; + + std::string cmd; + cmd.push_back((char)ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG); + cmd.append((ip.ss_family == AF_INET6) ? "inet6" : "inet"); + cmd.push_back(0); + cmd.append(ip.toString(tmp)); + cmd.push_back(0); + cmd.append("-alias"); + cmd.push_back(0); + + uint16_t l = (uint16_t)cmd.length(); + _putLock.lock(); + write(_agentStdin,&l,2); + write(_agentStdin,cmd.data(),cmd.length()); + _putLock.unlock(); + + return true; +} + +std::vector<InetAddress> MacEthernetTap::ips() const +{ + struct ifaddrs *ifa = (struct ifaddrs *)0; + std::vector<InetAddress> r; + if (!getifaddrs(&ifa)) { + struct ifaddrs *p = ifa; + while (p) { + if ((!strcmp(p->ifa_name,_dev.c_str()))&&(p->ifa_addr)&&(p->ifa_netmask)&&(p->ifa_addr->sa_family == p->ifa_netmask->sa_family)) { + switch(p->ifa_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *)p->ifa_addr; + struct sockaddr_in *nm = (struct sockaddr_in *)p->ifa_netmask; + r.push_back(InetAddress(&(sin->sin_addr.s_addr),4,Utils::countBits((uint32_t)nm->sin_addr.s_addr))); + } break; + case AF_INET6: { + struct sockaddr_in6 *sin = (struct sockaddr_in6 *)p->ifa_addr; + struct sockaddr_in6 *nm = (struct sockaddr_in6 *)p->ifa_netmask; + uint32_t b[4]; + memcpy(b,nm->sin6_addr.s6_addr,sizeof(b)); + r.push_back(InetAddress(sin->sin6_addr.s6_addr,16,Utils::countBits(b[0]) + Utils::countBits(b[1]) + Utils::countBits(b[2]) + Utils::countBits(b[3]))); + } break; + } + } + p = p->ifa_next; + } + freeifaddrs(ifa); + } + std::sort(r.begin(),r.end()); + r.erase(std::unique(r.begin(),r.end()),r.end()); + return r; +} + +void MacEthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) +{ + struct iovec iov[3]; + unsigned char hdr[15]; + uint16_t l; + if ((_agentStdin > 0)&&(len <= _mtu)&&(_enabled)) { + hdr[0] = ZT_MACETHERNETTAPAGENT_STDIN_CMD_PACKET; + to.copyTo(hdr + 1,6); + from.copyTo(hdr + 7,6); + hdr[13] = (unsigned char)((etherType >> 8) & 0xff); + hdr[14] = (unsigned char)(etherType & 0xff); + l = (uint16_t)(len + 15); + iov[0].iov_base = &l; + iov[0].iov_len = 2; + iov[1].iov_base = hdr; + iov[1].iov_len = 15; + iov[2].iov_base = const_cast<void *>(data); + iov[2].iov_len = len; + _putLock.lock(); + writev(_agentStdin,iov,3); + _putLock.unlock(); + } +} + +std::string MacEthernetTap::deviceName() const { return _dev; } +void MacEthernetTap::setFriendlyName(const char *friendlyName) {} + +void MacEthernetTap::scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed) +{ + std::vector<MulticastGroup> newGroups; + + struct ifmaddrs *ifmap = (struct ifmaddrs *)0; + if (!getifmaddrs(&ifmap)) { + struct ifmaddrs *p = ifmap; + while (p) { + if (p->ifma_addr->sa_family == AF_LINK) { + struct sockaddr_dl *in = (struct sockaddr_dl *)p->ifma_name; + struct sockaddr_dl *la = (struct sockaddr_dl *)p->ifma_addr; + if ((la->sdl_alen == 6)&&(in->sdl_nlen <= _dev.length())&&(!memcmp(_dev.data(),in->sdl_data,in->sdl_nlen))) + newGroups.push_back(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen,6),0)); + } + p = p->ifma_next; + } + freeifmaddrs(ifmap); + } + + std::vector<InetAddress> allIps(ips()); + for(std::vector<InetAddress>::iterator ip(allIps.begin());ip!=allIps.end();++ip) + newGroups.push_back(MulticastGroup::deriveMulticastGroupForAddressResolution(*ip)); + + std::sort(newGroups.begin(),newGroups.end()); + std::unique(newGroups.begin(),newGroups.end()); + + for(std::vector<MulticastGroup>::iterator m(newGroups.begin());m!=newGroups.end();++m) { + if (!std::binary_search(_multicastGroups.begin(),_multicastGroups.end(),*m)) + added.push_back(*m); + } + for(std::vector<MulticastGroup>::iterator m(_multicastGroups.begin());m!=_multicastGroups.end();++m) { + if (!std::binary_search(newGroups.begin(),newGroups.end(),*m)) + removed.push_back(*m); + } + + _multicastGroups.swap(newGroups); +} + +void MacEthernetTap::setMtu(unsigned int mtu) +{ + if (_mtu != mtu) { + char tmp[16]; + std::string cmd; + cmd.push_back((char)ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG); + cmd.append("mtu"); + cmd.push_back(0); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); + cmd.append(tmp); + cmd.push_back(0); + uint16_t l = (uint16_t)cmd.length(); + _putLock.lock(); + write(_agentStdin,&l,2); + write(_agentStdin,cmd.data(),cmd.length()); + _putLock.unlock(); + _mtu = mtu; + } +} + +#define ZT_MACETHERNETTAP_AGENT_READ_BUF_SIZE 131072 + +void MacEthernetTap::threadMain() + throw() +{ + char agentReadBuf[ZT_MACETHERNETTAP_AGENT_READ_BUF_SIZE]; + char agentStderrBuf[256]; + fd_set readfds,nullfds; + MAC to,from; + + Thread::sleep(250); + + const int nfds = std::max(std::max(_shutdownSignalPipe[0],_agentStdout),_agentStderr) + 1; + long agentReadPtr = 0; + fcntl(_agentStdout,F_SETFL,fcntl(_agentStdout,F_GETFL)|O_NONBLOCK); + fcntl(_agentStderr,F_SETFL,fcntl(_agentStderr,F_GETFL)|O_NONBLOCK); + + FD_ZERO(&readfds); + FD_ZERO(&nullfds); + for(;;) { + FD_SET(_shutdownSignalPipe[0],&readfds); + FD_SET(_agentStdout,&readfds); + FD_SET(_agentStderr,&readfds); + select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0); + + if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) + break; + + if (FD_ISSET(_agentStdout,&readfds)) { + long n = (long)read(_agentStdout,agentReadBuf + agentReadPtr,ZT_MACETHERNETTAP_AGENT_READ_BUF_SIZE - agentReadPtr); + if (n > 0) { + agentReadPtr += n; + while (agentReadPtr >= 2) { + long len = *((uint16_t *)agentReadBuf); + if (agentReadPtr >= (len + 2)) { + char *msg = agentReadBuf + 2; + + if ((len > 14)&&(_enabled)) { + to.setTo(msg,6); + from.setTo(msg + 6,6); + _handler(_arg,(void *)0,_nwid,from,to,ntohs(((const uint16_t *)msg)[6]),0,(const void *)(msg + 14),(unsigned int)len - 14); + } + + if (agentReadPtr > (len + 2)) { + memmove(agentReadBuf,agentReadBuf + len + 2,agentReadPtr -= (len + 2)); + } else { + agentReadPtr = 0; + } + } else { + break; + } + } + } + } + + if (FD_ISSET(_agentStderr,&readfds)) { + read(_agentStderr,agentStderrBuf,sizeof(agentStderrBuf)); + /* + const ssize_t n = read(_agentStderr,agentStderrBuf,sizeof(agentStderrBuf)); + if (n > 0) + write(STDERR_FILENO,agentStderrBuf,(size_t)n); + */ + } + } +} + +} // namespace ZeroTier diff --git a/osdep/OSXEthernetTap.hpp b/osdep/MacEthernetTap.hpp index fe402901..fb5bc6db 100644 --- a/osdep/OSXEthernetTap.hpp +++ b/osdep/MacEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -38,18 +38,16 @@ #include "../node/MAC.hpp" #include "../node/InetAddress.hpp" #include "../node/MulticastGroup.hpp" +#include "../node/Mutex.hpp" #include "Thread.hpp" namespace ZeroTier { -/** - * OSX Ethernet tap using ZeroTier kernel extension zt# devices - */ -class OSXEthernetTap +class MacEthernetTap { public: - OSXEthernetTap( + MacEthernetTap( const char *homePath, const MAC &mac, unsigned int mtu, @@ -59,7 +57,7 @@ public: void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); - ~OSXEthernetTap(); + ~MacEthernetTap(); void setEnabled(bool en); bool enabled() const; @@ -83,10 +81,12 @@ private: std::string _homePath; std::string _dev; std::vector<MulticastGroup> _multicastGroups; + Mutex _putLock; unsigned int _mtu; unsigned int _metric; - int _fd; int _shutdownSignalPipe[2]; + int _agentStdin,_agentStdout,_agentStderr,_agentStdin2,_agentStdout2,_agentStderr2; + long _agentPid; volatile bool _enabled; }; diff --git a/osdep/MacEthernetTapAgent.c b/osdep/MacEthernetTapAgent.c new file mode 100644 index 00000000..b5ad1f62 --- /dev/null +++ b/osdep/MacEthernetTapAgent.c @@ -0,0 +1,438 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +/* + * This creates a pair of feth devices with the lower numbered device + * being the ZeroTier virtual interface and the other being the device + * used to actually read and write packets. The latter gets no IP config + * and is only used for I/O. The behavior of feth is similar to the + * veth pairs that exist on Linux. + * + * The feth device has only existed since MacOS Sierra, but that's fairly + * long ago in Mac terms. + * + * I/O with feth must be done using two different sockets. The BPF socket + * is used to receive packets, while an AF_NDRV (low-level network driver + * access) socket must be used to inject. AF_NDRV can't read IP frames + * since BSD doesn't forward packets out the NDRV tap if they've already + * been handled, and while BPF can inject its MTU for injected packets + * is limited to 2048. AF_NDRV packet injection is required to inject + * ZeroTier's large MTU frames. + * + * Benchmarks show that this performs similarly to the old tap.kext driver, + * and a kext is no longer required. Splitting it off into an agent will + * also make it easier to have zerotier-one itself drop permissions. + * + * All this stuff is basically undocumented. A lot of tracing through + * the Darwin/XNU kernel source was required to figure out how to make + * this actually work. + * + * See also: + * + * https://apple.stackexchange.com/questions/337715/fake-ethernet-interfaces-feth-if-fake-anyone-ever-seen-this + * https://opensource.apple.com/source/xnu/xnu-4570.41.2/bsd/net/if_fake.c.auto.html + * + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdarg.h> +#include <unistd.h> +#include <signal.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/signal.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/ioctl.h> +#include <sys/wait.h> +#include <sys/select.h> +#include <sys/cdefs.h> +#include <sys/uio.h> +#include <sys/param.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <sys/sysctl.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <net/bpf.h> +#include <net/route.h> +#include <net/if.h> +#include <net/if_arp.h> +#include <net/if_dl.h> +#include <net/if_media.h> +#include <net/ndrv.h> +#include <netinet/in_var.h> +#include <netinet/icmp6.h> +#include <netinet6/in6_var.h> +#include <netinet6/nd6.h> +#include <ifaddrs.h> + +#include "../version.h" +#include "MacEthernetTapAgent.h" + +#ifndef SIOCAUTOCONF_START +#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq) /* accept rtadvd on this interface */ +#endif +#ifndef SIOCAUTOCONF_STOP +#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq) /* stop accepting rtadv for this interface */ +#endif + +#define P_IFCONFIG "/sbin/ifconfig" + +static unsigned char s_pktReadBuf[131072] __attribute__ ((__aligned__(16))); +static unsigned char s_stdinReadBuf[131072] __attribute__ ((__aligned__(16))); +static char s_deviceName[IFNAMSIZ]; +static char s_peerDeviceName[IFNAMSIZ]; +static int s_bpffd = -1; +static int s_ndrvfd = -1; +static pid_t s_parentPid; + +static void configureIpv6Parameters(const char *ifname,int performNUD,int acceptRouterAdverts) +{ + struct in6_ndireq nd; + struct in6_ifreq ifr; + + int s = socket(AF_INET6,SOCK_DGRAM,0); + if (s <= 0) + return; + + memset(&nd,0,sizeof(nd)); + strncpy(nd.ifname,ifname,sizeof(nd.ifname)); + + if (ioctl(s,SIOCGIFINFO_IN6,&nd)) { + close(s); + return; + } + + unsigned long oldFlags = (unsigned long)nd.ndi.flags; + + if (performNUD) + nd.ndi.flags |= ND6_IFF_PERFORMNUD; + else nd.ndi.flags &= ~ND6_IFF_PERFORMNUD; + + if (oldFlags != (unsigned long)nd.ndi.flags) { + if (ioctl(s,SIOCSIFINFO_FLAGS,&nd)) { + close(s); + return; + } + } + + memset(&ifr,0,sizeof(ifr)); + strncpy(ifr.ifr_name,ifname,sizeof(ifr.ifr_name)); + if (ioctl(s,acceptRouterAdverts ? SIOCAUTOCONF_START : SIOCAUTOCONF_STOP,&ifr)) { + close(s); + return; + } + + close(s); +} + +static int run(const char *path,...) +{ + va_list ap; + char *args[16]; + int argNo = 1; + + va_start(ap,path); + args[0] = (char *)path; + for(;argNo<15;++argNo) { + args[argNo] = va_arg(ap,char *); + if (!args[argNo]) { + break; + } + } + args[argNo++] = (char *)0; + va_end(ap); + + pid_t pid = vfork(); + if (pid < 0) { + return -1; + } else if (pid == 0) { + dup2(STDERR_FILENO,STDOUT_FILENO); + execv(args[0],args); + _exit(-1); + } + int rv = 0; + waitpid(pid,&rv,0); + return rv; +} + +static void die() +{ + if (s_ndrvfd >= 0) + close(s_ndrvfd); + if (s_bpffd >= 0) + close(s_bpffd); + if (s_deviceName[0]) + run("/sbin/ifconfig",s_deviceName,"destroy",(char *)0); + if (s_peerDeviceName[0]) + run("/sbin/ifconfig",s_peerDeviceName,"destroy",(char *)0); +} + +int main(int argc,char **argv) +{ + char buf[128]; + struct ifreq ifr; + u_int fl; + fd_set rfds,wfds,efds; + struct iovec iov[2]; + + s_deviceName[0] = 0; + s_peerDeviceName[0] = 0; + s_parentPid = getppid(); + + atexit(&die); + signal(SIGIO,SIG_IGN); + signal(SIGCHLD,SIG_IGN); + signal(SIGPIPE,SIG_IGN); + signal(SIGUSR1,SIG_IGN); + signal(SIGUSR2,SIG_IGN); + signal(SIGALRM,SIG_IGN); + signal(SIGQUIT,&exit); + signal(SIGTERM,&exit); + signal(SIGKILL,&exit); + signal(SIGINT,&exit); + signal(SIGPIPE,&exit); + + if (getuid() != 0) { + if (setuid(0) != 0) { + fprintf(stderr,"E must be run as root or with root setuid bit on executable\n"); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_INVALID_REQUEST; + } + } + + if (argc < 5) { + fprintf(stderr,"E invalid or missing argument(s) (usage: MacEthernetTapAgent <0-4999> <mac> <mtu> <metric>)\n"); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_INVALID_REQUEST; + } + const int deviceNo = atoi(argv[1]); + if ((deviceNo < 0)||(deviceNo > 4999)) { + fprintf(stderr,"E invalid or missing argument(s) (usage: MacEthernetTapAgent <0-4999> <mac> <mtu> <metric>)\n"); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_INVALID_REQUEST; + } + const char *mac = argv[2]; + const char *mtu = argv[3]; + const char *metric = argv[4]; + + s_ndrvfd = socket(AF_NDRV,SOCK_RAW,0); + if (s_ndrvfd < 0) { + fprintf(stderr,"E unable to open AF_NDRV socket\n"); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + + snprintf(s_deviceName,sizeof(s_deviceName),"feth%d",deviceNo); + snprintf(s_peerDeviceName,sizeof(s_peerDeviceName),"feth%d",deviceNo+5000); + if (run(P_IFCONFIG,s_peerDeviceName,"create",(char *)0) != 0) { + fprintf(stderr,"E unable to create %s\n",s_deviceName); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + usleep(10); + if (run(P_IFCONFIG,s_deviceName,"create",(char *)0) != 0) { + fprintf(stderr,"E unable to create %s\n",s_deviceName); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + run(P_IFCONFIG,s_deviceName,"lladdr",mac,(char *)0); + usleep(10); + run(P_IFCONFIG,s_peerDeviceName,"peer",s_deviceName,(char *)0); + usleep(10); + run(P_IFCONFIG,s_peerDeviceName,"mtu","16370","up",(char *)0); /* 16370 is the largest MTU MacOS/Darwin seems to allow */ + usleep(10); + run(P_IFCONFIG,s_deviceName,"mtu",mtu,"metric",metric,"up",(char *)0); + usleep(10); + configureIpv6Parameters(s_deviceName,1,0); + usleep(10); + + struct sockaddr_ndrv nd; + nd.snd_len = sizeof(struct sockaddr_ndrv); + nd.snd_family = AF_NDRV; + memcpy(nd.snd_name,s_peerDeviceName,sizeof(nd.snd_name)); + if (bind(s_ndrvfd,(struct sockaddr *)&nd,sizeof(nd)) != 0) { + fprintf(stderr,"E unable to bind AF_NDRV socket\n"); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + if (connect(s_ndrvfd,(struct sockaddr *)&nd,sizeof(nd)) != 0) { + fprintf(stderr,"E unable to connect AF_NDRV socket\n"); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + + /* Start at /dev/bpf1 since some simple bpf-using net utilities hard-code /dev/bpf0. + * Things like libpcap are smart enough to search. */ + for(int bpfno=1;bpfno<5000;++bpfno) { + char tmp[32]; + snprintf(tmp,sizeof(tmp),"/dev/bpf%d",bpfno); + s_bpffd = open(tmp,O_RDWR); + if (s_bpffd >= 0) { + break; + } + } + if (s_bpffd < 0) { + fprintf(stderr,"E unable to open bpf device\n"); + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + + fl = sizeof(s_pktReadBuf); + if (ioctl(s_bpffd,BIOCSBLEN,&fl) != 0) { + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + const size_t readPktSize = (size_t)fl; + fl = 1; + if (ioctl(s_bpffd,BIOCIMMEDIATE,&fl) != 0) { + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + fl = 0; + if (ioctl(s_bpffd,BIOCSSEESENT,&fl) != 0) { + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + memset(&ifr,0,sizeof(ifr)); + memcpy(ifr.ifr_name,s_peerDeviceName,IFNAMSIZ); + if (ioctl(s_bpffd,BIOCSETIF,&ifr) != 0) { + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + fl = 1; + if (ioctl(s_bpffd,BIOCSHDRCMPLT,&fl) != 0) { + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + fl = 1; + if (ioctl(s_bpffd,BIOCPROMISC,&fl) != 0) { + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE; + } + + fprintf(stderr,"I %s %s %d.%d.%d.%d\n",s_deviceName,s_peerDeviceName,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,ZEROTIER_ONE_VERSION_BUILD); + + FD_ZERO(&rfds); + FD_ZERO(&wfds); + FD_ZERO(&efds); + long stdinReadPtr = 0; + for(;;) { + FD_SET(STDIN_FILENO,&rfds); + FD_SET(s_bpffd,&rfds); + if (select(s_bpffd+1,&rfds,&wfds,&efds,(struct timeval *)0) < 0) { + if ((errno == EAGAIN)||(errno == EINTR)) { + usleep(10); + continue; + } + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_READ_ERROR; + } + + if (FD_ISSET(s_bpffd,&rfds)) { + long n = (long)read(s_bpffd,s_pktReadBuf,readPktSize); + if (n > 0) { + for(unsigned char *p=s_pktReadBuf,*eof=p+n;p<eof;) { + struct bpf_hdr *h = (struct bpf_hdr *)p; + if ((h->bh_caplen > 0)&&((p + h->bh_hdrlen + h->bh_caplen) <= eof)) { + uint16_t len = (uint16_t)h->bh_caplen; + iov[0].iov_base = &len; + iov[0].iov_len = 2; + iov[1].iov_base = p + h->bh_hdrlen; + iov[1].iov_len = h->bh_caplen; + writev(STDOUT_FILENO,iov,2); + } + p += BPF_WORDALIGN(h->bh_hdrlen + h->bh_caplen); + } + } + } + + if (FD_ISSET(STDIN_FILENO,&rfds)) { + long n = (long)read(STDIN_FILENO,s_stdinReadBuf + stdinReadPtr,sizeof(s_stdinReadBuf) - stdinReadPtr); + if (n > 0) { + stdinReadPtr += n; + while (stdinReadPtr >= 2) { + long len = *((uint16_t *)s_stdinReadBuf); + if (stdinReadPtr >= (len + 2)) { + if (len > 0) { + unsigned char *msg = s_stdinReadBuf + 2; + + switch(msg[0]) { + case ZT_MACETHERNETTAPAGENT_STDIN_CMD_PACKET: + if (len > 1) { + if (write(s_ndrvfd,msg+1,len-1) < 0) { + fprintf(stderr,"E inject failed size==%ld errno==%d\n",len-1,errno); + } + } + break; + + case ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG: { + char *args[16]; + args[0] = P_IFCONFIG; + args[1] = s_deviceName; + int argNo = 2; + for(int argPtr=0,k=1,l=(int)len;k<l;++k) { + if (!msg[k]) { + if (argPtr > 0) { + argPtr = 0; + ++argNo; + if (argNo >= 15) { + break; + } + } + } else { + if (argPtr == 0) { + args[argNo] = (char *)(msg + k); + } + argPtr++; + } + } + args[argNo] = (char *)0; + if (argNo > 2) { + pid_t pid = fork(); + if (pid < 0) { + return -1; + } else if (pid == 0) { + dup2(STDERR_FILENO,STDOUT_FILENO); + execv(args[0],args); + _exit(-1); + } + int rv = 0; + waitpid(pid,&rv,0); + } + } break; + + case ZT_MACETHERNETTAPAGENT_STDIN_CMD_EXIT: + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_SUCCESS; + + default: + fprintf(stderr,"E unrecognized message type over pipe from host process: %d (length: %d)\n",(int)msg[0],(int)len); + break; + } + } + + if (stdinReadPtr > (len + 2)) { + memmove(s_stdinReadBuf,s_stdinReadBuf + len + 2,stdinReadPtr -= (len + 2)); + } else { + stdinReadPtr = 0; + } + } else { + break; + } + } + } + } + } + + return ZT_MACETHERNETTAPAGENT_EXIT_CODE_SUCCESS; +} diff --git a/osdep/MacEthernetTapAgent.h b/osdep/MacEthernetTapAgent.h new file mode 100644 index 00000000..f6459523 --- /dev/null +++ b/osdep/MacEthernetTapAgent.h @@ -0,0 +1,41 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_MACETHERNETTAPAGENT_H +#define ZT_MACETHERNETTAPAGENT_H + +#define ZT_MACETHERNETTAPAGENT_EXIT_CODE_SUCCESS 0 +#define ZT_MACETHERNETTAPAGENT_EXIT_CODE_INVALID_REQUEST -1 +#define ZT_MACETHERNETTAPAGENT_EXIT_CODE_UNABLE_TO_CREATE -2 +#define ZT_MACETHERNETTAPAGENT_EXIT_CODE_READ_ERROR -3 + +#define ZT_MACETHERNETTAPAGENT_STDIN_CMD_PACKET 0 +#define ZT_MACETHERNETTAPAGENT_STDIN_CMD_IFCONFIG 1 +#define ZT_MACETHERNETTAPAGENT_STDIN_CMD_EXIT 2 + +#define ZT_MACETHERNETTAPAGENT_DEFAULT_SYSTEM_PATH "/Library/Application Support/ZeroTier/One/MacEthernetTapAgent" + +#endif diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 324fada0..56e7b094 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -46,7 +46,9 @@ #include <sys/wait.h> #include <netinet/in.h> #include <arpa/inet.h> +#ifndef ZT_SDK #include <net/route.h> +#endif #include <net/if.h> #ifdef __BSD__ #include <net/if_dl.h> @@ -109,6 +111,7 @@ struct _RTE #ifdef __BSD__ // ------------------------------------------------------------ #define ZT_ROUTING_SUPPORT_FOUND 1 +#ifndef ZT_SDK static std::vector<_RTE> _getRTEs(const InetAddress &target,bool contains) { std::vector<_RTE> rtes; @@ -243,6 +246,7 @@ static std::vector<_RTE> _getRTEs(const InetAddress &target,bool contains) return rtes; } +#endif static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *ifscope,const char *localInterface) { @@ -409,6 +413,7 @@ static bool _winHasRoute(const NET_LUID &interfaceLuid, const NET_IFINDEX &inter * Linux default route override implies asymmetric routes, which then * trigger Linux's "martian packet" filter. */ +#ifndef ZT_SDK bool ManagedRoute::sync() { #ifdef __WINDOWS__ @@ -519,6 +524,7 @@ bool ManagedRoute::sync() return true; } +#endif void ManagedRoute::remove() { @@ -562,4 +568,4 @@ void ManagedRoute::remove() _applied.clear(); } -} // namespace ZeroTier +} // namespace ZeroTier
\ No newline at end of file diff --git a/osdep/ManagedRoute.hpp b/osdep/ManagedRoute.hpp index 779ad6a1..e1a43c26 100644 --- a/osdep/ManagedRoute.hpp +++ b/osdep/ManagedRoute.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -49,14 +49,20 @@ class ManagedRoute friend class SharedPtr<ManagedRoute>; public: - ManagedRoute(const InetAddress &target,const InetAddress &via,const char *device) + ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device) { _target = target; _via = via; + _src = src; if (via.ss_family == AF_INET) _via.setPort(32); else if (via.ss_family == AF_INET6) _via.setPort(128); + if (src.ss_family == AF_INET) { + _src.setPort(32); + } else if (src.ss_family == AF_INET6) { + _src.setPort(128); + } Utils::scopy(_device,sizeof(_device),device); _systemDevice[0] = (char)0; } @@ -87,6 +93,7 @@ public: inline const InetAddress &target() const { return _target; } inline const InetAddress &via() const { return _via; } + inline const InetAddress &src() const { return _src; } inline const char *device() const { return _device; } private: @@ -95,6 +102,7 @@ private: InetAddress _target; InetAddress _via; + InetAddress _src; InetAddress _systemVia; // for route overrides std::map<InetAddress,bool> _applied; // routes currently applied char _device[128]; diff --git a/osdep/NeighborDiscovery.cpp b/osdep/NeighborDiscovery.cpp index d9862f3d..134bd246 100644 --- a/osdep/NeighborDiscovery.cpp +++ b/osdep/NeighborDiscovery.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/NeighborDiscovery.hpp b/osdep/NeighborDiscovery.hpp index 59186289..b660068a 100644 --- a/osdep/NeighborDiscovery.hpp +++ b/osdep/NeighborDiscovery.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/NetBSDEthernetTap.cpp b/osdep/NetBSDEthernetTap.cpp index 5d1c9625..027f2ec3 100644 --- a/osdep/NetBSDEthernetTap.cpp +++ b/osdep/NetBSDEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include <stdint.h> diff --git a/osdep/NetBSDEthernetTap.hpp b/osdep/NetBSDEthernetTap.hpp index a555a4b1..32b6dfa6 100644 --- a/osdep/NetBSDEthernetTap.hpp +++ b/osdep/NetBSDEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,15 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NetBSDEthernetTap_HPP @@ -31,10 +39,6 @@ #include "../node/MAC.hpp" #include "Thread.hpp" - - - - namespace ZeroTier { class NetBSDEthernetTap diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index cadd4e6b..83c615de 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -29,6 +29,7 @@ #include <stdlib.h> #include <stdarg.h> #include <sys/stat.h> +#include <stdlib.h> #include "../node/Constants.hpp" #include "../node/Utils.hpp" @@ -366,7 +367,7 @@ std::vector<std::string> OSUtils::split(const char *s,const char *const sep,cons if (buf.size() > 0) { fields.push_back(buf); buf.clear(); - } // else skip runs of seperators + } // else skip runs of separators } else buf.push_back(*s); } ++s; @@ -398,6 +399,21 @@ std::string OSUtils::platformDefaultHomePath() return homeDir; #endif + // Check for user-defined environment variable before using defaults +#ifdef __WINDOWS__ + DWORD bufferSize = 65535; + std::string userDefinedPath; + bufferSize = GetEnvironmentVariable("ZEROTIER_HOME", &userDefinedPath[0], bufferSize); + if (bufferSize) { + return userDefinedPath; + } +#else + if(const char* userDefinedPath = getenv("ZEROTIER_HOME")) { + return std::string(userDefinedPath); + } +#endif + + // Finally, resort to using default paths if no user-defined path was provided #ifdef __UNIX_LIKE__ #ifdef __APPLE__ @@ -432,6 +448,7 @@ std::string OSUtils::platformDefaultHomePath() #endif // __UNIX_LIKE__ or not... } +#ifndef OMIT_JSON_SUPPORT // Inline these massive JSON operations in one place only to reduce binary footprint and compile time nlohmann::json OSUtils::jsonParse(const std::string &buf) { return nlohmann::json::parse(buf.c_str()); } std::string OSUtils::jsonDump(const nlohmann::json &j,int indentation) { return j.dump(indentation); } @@ -523,6 +540,8 @@ std::string OSUtils::jsonBinFromHex(const nlohmann::json &jv) return std::string(); } +#endif // OMIT_JSON_SUPPORT + // Used to convert HTTP header names to ASCII lower case const unsigned char OSUtils::TOLOWER_TABLE[256] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ' ', '!', '"', '#', '$', '%', '&', 0x27, '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 324b7923..afdb1bf1 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -55,7 +55,9 @@ #endif #endif +#ifndef OMIT_JSON_SUPPORT #include "../ext/json/json.hpp" +#endif namespace ZeroTier { @@ -284,6 +286,7 @@ public: */ static std::string platformDefaultHomePath(); +#ifndef OMIT_JSON_SUPPORT static nlohmann::json jsonParse(const std::string &buf); static std::string jsonDump(const nlohmann::json &j,int indentation = 1); static uint64_t jsonInt(const nlohmann::json &jv,const uint64_t dfl); @@ -291,6 +294,7 @@ public: static bool jsonBool(const nlohmann::json &jv,const bool dfl); static std::string jsonString(const nlohmann::json &jv,const char *dfl); static std::string jsonBinFromHex(const nlohmann::json &jv); +#endif // OMIT_JSON_SUPPORT private: static const unsigned char TOLOWER_TABLE[256]; diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp deleted file mode 100644 index dbff6200..00000000 --- a/osdep/OSXEthernetTap.cpp +++ /dev/null @@ -1,703 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <errno.h> - -#include <unistd.h> -#include <signal.h> - -#include <fcntl.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/ioctl.h> -#include <sys/wait.h> -#include <sys/select.h> -#include <sys/cdefs.h> -#include <sys/uio.h> -#include <sys/param.h> -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <net/route.h> -#include <net/if.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> -#include <sys/sysctl.h> -#include <netinet6/in6_var.h> -#include <netinet/in_var.h> -#include <netinet/icmp6.h> - -// OSX compile fix... in6_var defines this in a struct which namespaces it for C++ ... why?!? -struct prf_ra { - u_char onlink : 1; - u_char autonomous : 1; - u_char reserved : 6; -} prf_ra; - -#include <netinet6/nd6.h> -#include <ifaddrs.h> - -// These are KERNEL_PRIVATE... why? -#ifndef SIOCAUTOCONF_START -#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq) /* accept rtadvd on this interface */ -#endif -#ifndef SIOCAUTOCONF_STOP -#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq) /* stop accepting rtadv for this interface */ -#endif - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- -// This source is from: -// http://www.opensource.apple.com/source/Libinfo/Libinfo-406.17/gen.subproj/getifmaddrs.c?txt -// It's here because OSX 10.6 does not have this convenience function. - -#define SALIGN (sizeof(uint32_t) - 1) -#define SA_RLEN(sa) ((sa)->sa_len ? (((sa)->sa_len + SALIGN) & ~SALIGN) : \ -(SALIGN + 1)) -#define MAX_SYSCTL_TRY 5 -#define RTA_MASKS (RTA_GATEWAY | RTA_IFP | RTA_IFA) - -/* FreeBSD uses NET_RT_IFMALIST and RTM_NEWMADDR from <sys/socket.h> */ -/* We can use NET_RT_IFLIST2 and RTM_NEWMADDR2 on Darwin */ -//#define DARWIN_COMPAT - -//#ifdef DARWIN_COMPAT -#define GIM_SYSCTL_MIB NET_RT_IFLIST2 -#define GIM_RTM_ADDR RTM_NEWMADDR2 -//#else -//#define GIM_SYSCTL_MIB NET_RT_IFMALIST -//#define GIM_RTM_ADDR RTM_NEWMADDR -//#endif - -// Not in 10.6 includes so use our own -struct _intl_ifmaddrs { - struct _intl_ifmaddrs *ifma_next; - struct sockaddr *ifma_name; - struct sockaddr *ifma_addr; - struct sockaddr *ifma_lladdr; -}; - -static inline int _intl_getifmaddrs(struct _intl_ifmaddrs **pif) -{ - int icnt = 1; - int dcnt = 0; - int ntry = 0; - size_t len; - size_t needed; - int mib[6]; - int i; - char *buf; - char *data; - char *next; - char *p; - struct ifma_msghdr2 *ifmam; - struct _intl_ifmaddrs *ifa, *ift; - struct rt_msghdr *rtm; - struct sockaddr *sa; - - mib[0] = CTL_NET; - mib[1] = PF_ROUTE; - mib[2] = 0; /* protocol */ - mib[3] = 0; /* wildcard address family */ - mib[4] = GIM_SYSCTL_MIB; - mib[5] = 0; /* no flags */ - do { - if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0) - return (-1); - if ((buf = (char *)malloc(needed)) == NULL) - return (-1); - if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) { - if (errno != ENOMEM || ++ntry >= MAX_SYSCTL_TRY) { - free(buf); - return (-1); - } - free(buf); - buf = NULL; - } - } while (buf == NULL); - - for (next = buf; next < buf + needed; next += rtm->rtm_msglen) { - rtm = (struct rt_msghdr *)(void *)next; - if (rtm->rtm_version != RTM_VERSION) - continue; - switch (rtm->rtm_type) { - case GIM_RTM_ADDR: - ifmam = (struct ifma_msghdr2 *)(void *)rtm; - if ((ifmam->ifmam_addrs & RTA_IFA) == 0) - break; - icnt++; - p = (char *)(ifmam + 1); - for (i = 0; i < RTAX_MAX; i++) { - if ((RTA_MASKS & ifmam->ifmam_addrs & - (1 << i)) == 0) - continue; - sa = (struct sockaddr *)(void *)p; - len = SA_RLEN(sa); - dcnt += len; - p += len; - } - break; - } - } - - data = (char *)malloc(sizeof(struct _intl_ifmaddrs) * icnt + dcnt); - if (data == NULL) { - free(buf); - return (-1); - } - - ifa = (struct _intl_ifmaddrs *)(void *)data; - data += sizeof(struct _intl_ifmaddrs) * icnt; - - memset(ifa, 0, sizeof(struct _intl_ifmaddrs) * icnt); - ift = ifa; - - for (next = buf; next < buf + needed; next += rtm->rtm_msglen) { - rtm = (struct rt_msghdr *)(void *)next; - if (rtm->rtm_version != RTM_VERSION) - continue; - - switch (rtm->rtm_type) { - case GIM_RTM_ADDR: - ifmam = (struct ifma_msghdr2 *)(void *)rtm; - if ((ifmam->ifmam_addrs & RTA_IFA) == 0) - break; - - p = (char *)(ifmam + 1); - for (i = 0; i < RTAX_MAX; i++) { - if ((RTA_MASKS & ifmam->ifmam_addrs & - (1 << i)) == 0) - continue; - sa = (struct sockaddr *)(void *)p; - len = SA_RLEN(sa); - switch (i) { - case RTAX_GATEWAY: - ift->ifma_lladdr = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - case RTAX_IFP: - ift->ifma_name = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - case RTAX_IFA: - ift->ifma_addr = - (struct sockaddr *)(void *)data; - memcpy(data, p, len); - data += len; - break; - - default: - data += len; - break; - } - p += len; - } - ift->ifma_next = ift + 1; - ift = ift->ifma_next; - break; - } - } - - free(buf); - - if (ift > ifa) { - ift--; - ift->ifma_next = NULL; - *pif = ifa; - } else { - *pif = NULL; - free(ifa); - } - return (0); -} - -static inline void _intl_freeifmaddrs(struct _intl_ifmaddrs *ifmp) -{ - free(ifmp); -} - -// -------------------------------------------------------------------------- -// -------------------------------------------------------------------------- - -#include <string> -#include <map> -#include <set> -#include <algorithm> - -#include "../node/Constants.hpp" -#include "../node/Utils.hpp" -#include "../node/Mutex.hpp" -#include "../node/Dictionary.hpp" -#include "OSUtils.hpp" -#include "OSXEthernetTap.hpp" - -// ff:ff:ff:ff:ff:ff with no ADI -static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0); - -static inline bool _setIpv6Stuff(const char *ifname,bool performNUD,bool acceptRouterAdverts) -{ - struct in6_ndireq nd; - struct in6_ifreq ifr; - - int s = socket(AF_INET6,SOCK_DGRAM,0); - if (s <= 0) - return false; - - memset(&nd,0,sizeof(nd)); - strncpy(nd.ifname,ifname,sizeof(nd.ifname)); - - if (ioctl(s,SIOCGIFINFO_IN6,&nd)) { - close(s); - return false; - } - - unsigned long oldFlags = (unsigned long)nd.ndi.flags; - - if (performNUD) - nd.ndi.flags |= ND6_IFF_PERFORMNUD; - else nd.ndi.flags &= ~ND6_IFF_PERFORMNUD; - - if (oldFlags != (unsigned long)nd.ndi.flags) { - if (ioctl(s,SIOCSIFINFO_FLAGS,&nd)) { - close(s); - return false; - } - } - - memset(&ifr,0,sizeof(ifr)); - strncpy(ifr.ifr_name,ifname,sizeof(ifr.ifr_name)); - if (ioctl(s,acceptRouterAdverts ? SIOCAUTOCONF_START : SIOCAUTOCONF_STOP,&ifr)) { - close(s); - return false; - } - - close(s); - return true; -} - -namespace ZeroTier { - -static long globalTapsRunning = 0; -static Mutex globalTapCreateLock; - -OSXEthernetTap::OSXEthernetTap( - const char *homePath, - const MAC &mac, - unsigned int mtu, - unsigned int metric, - uint64_t nwid, - const char *friendlyName, - void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), - void *arg) : - _handler(handler), - _arg(arg), - _nwid(nwid), - _homePath(homePath), - _mtu(mtu), - _metric(metric), - _fd(0), - _enabled(true) -{ - char devpath[64],ethaddr[64],mtustr[32],metstr[32],nwids[32]; - struct stat stattmp; - - OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); - - Mutex::Lock _gl(globalTapCreateLock); - - if (::stat("/dev/zt0",&stattmp)) { - long kextpid = (long)vfork(); - if (kextpid == 0) { - ::chdir(homePath); - OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); - ::execl("/sbin/kextload","/sbin/kextload","-q","-repository",homePath,"tap.kext",(const char *)0); - ::_exit(-1); - } else if (kextpid > 0) { - int exitcode = -1; - ::waitpid(kextpid,&exitcode,0); - } - ::usleep(500); // give tap device driver time to start up and try again - if (::stat("/dev/zt0",&stattmp)) - throw std::runtime_error("/dev/zt# tap devices do not exist and cannot load tap.kext"); - } - - // Try to reopen the last device we had, if we had one and it's still unused. - std::map<std::string,std::string> globalDeviceMap; - FILE *devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"r"); - if (devmapf) { - char buf[256]; - while (fgets(buf,sizeof(buf),devmapf)) { - char *x = (char *)0; - char *y = (char *)0; - char *saveptr = (char *)0; - for(char *f=Utils::stok(buf,"\r\n=",&saveptr);(f);f=Utils::stok((char *)0,"\r\n=",&saveptr)) { - if (!x) x = f; - else if (!y) y = f; - else break; - } - if ((x)&&(y)&&(x[0])&&(y[0])) - globalDeviceMap[x] = y; - } - fclose(devmapf); - } - bool recalledDevice = false; - std::map<std::string,std::string>::const_iterator gdmEntry = globalDeviceMap.find(nwids); - if (gdmEntry != globalDeviceMap.end()) { - std::string devpath("/dev/"); devpath.append(gdmEntry->second); - if (stat(devpath.c_str(),&stattmp) == 0) { - _fd = ::open(devpath.c_str(),O_RDWR); - if (_fd > 0) { - _dev = gdmEntry->second; - recalledDevice = true; - } - } - } - - // Open the first unused tap device if we didn't recall a previous one. - if (!recalledDevice) { - for(int i=0;i<64;++i) { - OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i); - if (stat(devpath,&stattmp)) - throw std::runtime_error("no more TAP devices available"); - _fd = ::open(devpath,O_RDWR); - if (_fd > 0) { - char foo[16]; - OSUtils::ztsnprintf(foo,sizeof(foo),"zt%d",i); - _dev = foo; - break; - } - } - } - - if (_fd <= 0) - throw std::runtime_error("unable to open TAP device or no more devices available"); - - if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) { - ::close(_fd); - throw std::runtime_error("unable to set flags on file descriptor for TAP device"); - } - - // Configure MAC address and MTU, bring interface up - OSUtils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - OSUtils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); - OSUtils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); - long cpid = (long)vfork(); - if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - if (exitcode) { - ::close(_fd); - throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface"); - } - } - - _setIpv6Stuff(_dev.c_str(),true,false); - - // Set close-on-exec so that devices cannot persist if we fork/exec for update - fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC); - - ::pipe(_shutdownSignalPipe); - - ++globalTapsRunning; - - globalDeviceMap[nwids] = _dev; - devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"w"); - if (devmapf) { - gdmEntry = globalDeviceMap.begin(); - while (gdmEntry != globalDeviceMap.end()) { - fprintf(devmapf,"%s=%s\n",gdmEntry->first.c_str(),gdmEntry->second.c_str()); - ++gdmEntry; - } - fclose(devmapf); - } - - _thread = Thread::start(this); -} - -OSXEthernetTap::~OSXEthernetTap() -{ - ::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit - Thread::join(_thread); - - ::close(_fd); - ::close(_shutdownSignalPipe[0]); - ::close(_shutdownSignalPipe[1]); - - { - Mutex::Lock _gl(globalTapCreateLock); - if (--globalTapsRunning <= 0) { - globalTapsRunning = 0; // sanity check -- should not be possible - - char tmp[16384]; - sprintf(tmp,"%s/%s",_homePath.c_str(),"tap.kext"); - long kextpid = (long)vfork(); - if (kextpid == 0) { - OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); - ::execl("/sbin/kextunload","/sbin/kextunload",tmp,(const char *)0); - ::_exit(-1); - } else if (kextpid > 0) { - int exitcode = -1; - ::waitpid(kextpid,&exitcode,0); - } - } - } -} - -void OSXEthernetTap::setEnabled(bool en) -{ - _enabled = en; - // TODO: interface status change -} - -bool OSXEthernetTap::enabled() const -{ - return _enabled; -} - -bool OSXEthernetTap::addIp(const InetAddress &ip) -{ - if (!ip) - return false; - - long cpid = (long)vfork(); - if (cpid == 0) { - char tmp[128]; - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toString(tmp),"alias",(const char *)0); - ::_exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - ::waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } // else return false... - - return false; -} - -bool OSXEthernetTap::removeIp(const InetAddress &ip) -{ - if (!ip) - return true; - std::vector<InetAddress> allIps(ips()); - for(std::vector<InetAddress>::iterator i(allIps.begin());i!=allIps.end();++i) { - if (*i == ip) { - long cpid = (long)vfork(); - if (cpid == 0) { - char tmp[128]; - execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toIpString(tmp),"-alias",(const char *)0); - _exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - waitpid(cpid,&exitcode,0); - return (exitcode == 0); - } - } - } - return false; -} - -std::vector<InetAddress> OSXEthernetTap::ips() const -{ - struct ifaddrs *ifa = (struct ifaddrs *)0; - if (getifaddrs(&ifa)) - return std::vector<InetAddress>(); - - std::vector<InetAddress> r; - - struct ifaddrs *p = ifa; - while (p) { - if ((!strcmp(p->ifa_name,_dev.c_str()))&&(p->ifa_addr)&&(p->ifa_netmask)&&(p->ifa_addr->sa_family == p->ifa_netmask->sa_family)) { - switch(p->ifa_addr->sa_family) { - case AF_INET: { - struct sockaddr_in *sin = (struct sockaddr_in *)p->ifa_addr; - struct sockaddr_in *nm = (struct sockaddr_in *)p->ifa_netmask; - r.push_back(InetAddress(&(sin->sin_addr.s_addr),4,Utils::countBits((uint32_t)nm->sin_addr.s_addr))); - } break; - case AF_INET6: { - struct sockaddr_in6 *sin = (struct sockaddr_in6 *)p->ifa_addr; - struct sockaddr_in6 *nm = (struct sockaddr_in6 *)p->ifa_netmask; - uint32_t b[4]; - memcpy(b,nm->sin6_addr.s6_addr,sizeof(b)); - r.push_back(InetAddress(sin->sin6_addr.s6_addr,16,Utils::countBits(b[0]) + Utils::countBits(b[1]) + Utils::countBits(b[2]) + Utils::countBits(b[3]))); - } break; - } - } - p = p->ifa_next; - } - - if (ifa) - freeifaddrs(ifa); - - std::sort(r.begin(),r.end()); - r.erase(std::unique(r.begin(),r.end()),r.end()); - - return r; -} - -void OSXEthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) -{ - char putBuf[ZT_MAX_MTU + 64]; - if ((_fd > 0)&&(len <= _mtu)&&(_enabled)) { - to.copyTo(putBuf,6); - from.copyTo(putBuf + 6,6); - *((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType); - memcpy(putBuf + 14,data,len); - len += 14; - ::write(_fd,putBuf,len); - } -} - -std::string OSXEthernetTap::deviceName() const -{ - return _dev; -} - -void OSXEthernetTap::setFriendlyName(const char *friendlyName) -{ -} - -void OSXEthernetTap::scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed) -{ - std::vector<MulticastGroup> newGroups; - - struct _intl_ifmaddrs *ifmap = (struct _intl_ifmaddrs *)0; - if (!_intl_getifmaddrs(&ifmap)) { - struct _intl_ifmaddrs *p = ifmap; - while (p) { - if (p->ifma_addr->sa_family == AF_LINK) { - struct sockaddr_dl *in = (struct sockaddr_dl *)p->ifma_name; - struct sockaddr_dl *la = (struct sockaddr_dl *)p->ifma_addr; - if ((la->sdl_alen == 6)&&(in->sdl_nlen <= _dev.length())&&(!memcmp(_dev.data(),in->sdl_data,in->sdl_nlen))) - newGroups.push_back(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen,6),0)); - } - p = p->ifma_next; - } - _intl_freeifmaddrs(ifmap); - } - - std::vector<InetAddress> allIps(ips()); - for(std::vector<InetAddress>::iterator ip(allIps.begin());ip!=allIps.end();++ip) - newGroups.push_back(MulticastGroup::deriveMulticastGroupForAddressResolution(*ip)); - - std::sort(newGroups.begin(),newGroups.end()); - std::unique(newGroups.begin(),newGroups.end()); - - for(std::vector<MulticastGroup>::iterator m(newGroups.begin());m!=newGroups.end();++m) { - if (!std::binary_search(_multicastGroups.begin(),_multicastGroups.end(),*m)) - added.push_back(*m); - } - for(std::vector<MulticastGroup>::iterator m(_multicastGroups.begin());m!=_multicastGroups.end();++m) { - if (!std::binary_search(newGroups.begin(),newGroups.end(),*m)) - removed.push_back(*m); - } - - _multicastGroups.swap(newGroups); -} - -void OSXEthernetTap::setMtu(unsigned int mtu) -{ - if (mtu != _mtu) { - _mtu = mtu; - long cpid = (long)vfork(); - if (cpid == 0) { - char tmp[64]; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); - execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); - _exit(-1); - } else if (cpid > 0) { - int exitcode = -1; - waitpid(cpid,&exitcode,0); - } - } -} - -void OSXEthernetTap::threadMain() - throw() -{ - fd_set readfds,nullfds; - MAC to,from; - int n,nfds,r; - char getBuf[ZT_MAX_MTU + 64]; - - Thread::sleep(500); - - FD_ZERO(&readfds); - FD_ZERO(&nullfds); - nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1; - - r = 0; - for(;;) { - FD_SET(_shutdownSignalPipe[0],&readfds); - FD_SET(_fd,&readfds); - select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0); - - if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread - break; - - if (FD_ISSET(_fd,&readfds)) { - n = (int)::read(_fd,getBuf + r,sizeof(getBuf) - r); - if (n < 0) { - if ((errno != EINTR)&&(errno != ETIMEDOUT)) - break; - } else { - // Some tap drivers like to send the ethernet frame and the - // payload in two chunks, so handle that by accumulating - // data until we have at least a frame. - r += n; - if (r > 14) { - if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms - r = _mtu + 14; - - if (_enabled) { - to.setTo(getBuf,6); - from.setTo(getBuf + 6,6); - unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); - // TODO: VLAN support - _handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); - } - - r = 0; - } - } - } - } -} - -} // namespace ZeroTier diff --git a/osdep/Phy.hpp b/osdep/Phy.hpp index e359ccdd..333e5c24 100644 --- a/osdep/Phy.hpp +++ b/osdep/Phy.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -152,12 +152,13 @@ private: ZT_PHY_SOCKET_UNIX_LISTEN = 0x08 }; - struct PhySocketImpl - { + struct PhySocketImpl { + PhySocketImpl() { memset(ifname, 0, sizeof(ifname)); } PhySocketType type; ZT_PHY_SOCKFD_TYPE sock; void *uptr; // user-settable pointer ZT_PHY_SOCKADDR_STORAGE_TYPE saddr; // remote for TCP_OUT and TCP_IN, local for TCP_LISTEN, RAW, and UDP + char ifname[16]; }; std::list<PhySocketImpl> _socks; @@ -250,6 +251,70 @@ public: static inline void** getuptr(PhySocket *s) throw() { return &(reinterpret_cast<PhySocketImpl *>(s)->uptr); } /** + * @param s Socket object + * @param nameBuf Buffer to store name of interface which this Socket object is bound to + * @param buflen Length of buffer to copy name into + */ + static inline void getIfName(PhySocket *s, char *nameBuf, int buflen) + { + if (s) { + memcpy(nameBuf, reinterpret_cast<PhySocketImpl *>(s)->ifname, buflen); + } + } + + /** + * @param s Socket object + * @param ifname Buffer containing name of interface that this Socket object is bound to + * @param len Length of name of interface + */ + static inline void setIfName(PhySocket *s, char *ifname, int len) + { + if (s) { + memcpy(&(reinterpret_cast<PhySocketImpl *>(s)->ifname), ifname, len); + } + } + + /** + * Whether or not the socket object is in a closed state + * + * @param s Socket object + * @return true if socket is closed, false if otherwise + */ + inline bool isClosed(PhySocket *s) + { + PhySocketImpl *sws = (reinterpret_cast<PhySocketImpl *>(s)); + return sws->type == ZT_PHY_SOCKET_CLOSED; + } + + /** + * Get state of socket object + * + * @param s Socket object + * @return State of socket + */ + inline int getState(PhySocket *s) + { + PhySocketImpl *sws = (reinterpret_cast<PhySocketImpl *>(s)); + return sws->type; + } + + /** + * In the event that this socket is erased, we need a way to convey to the multipath logic + * that this path is no longer valid. + * + * @param s Socket object + * @return Whether the state of this socket is within an acceptable range of values + */ + inline bool isValidState(PhySocket *s) + { + if (s) { + PhySocketImpl *sws = (reinterpret_cast<PhySocketImpl *>(s)); + return sws->type >= ZT_PHY_SOCKET_CLOSED && sws->type <= ZT_PHY_SOCKET_UNIX_LISTEN; + } + return false; + } + + /** * Cause poll() to stop waiting immediately * * This can be used to reset the polling loop after changes that require @@ -985,7 +1050,7 @@ public: ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { try { - _handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr),false); + _handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr)); } catch ( ... ) {} } if (FD_ISSET(sock,&rfds)) { diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index a7dd3046..6b2409b6 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/PortMapper.hpp b/osdep/PortMapper.hpp index fa3cdc31..a075b774 100644 --- a/osdep/PortMapper.hpp +++ b/osdep/PortMapper.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/TestEthernetTap.hpp b/osdep/TestEthernetTap.hpp index 6ccf92f3..6b44d48e 100644 --- a/osdep/TestEthernetTap.hpp +++ b/osdep/TestEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/osdep/Thread.hpp b/osdep/Thread.hpp index 35ea5035..4d32ce86 100644 --- a/osdep/Thread.hpp +++ b/osdep/Thread.hpp @@ -1,6 +1,6 @@ /*
* ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/
+ * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@ * GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* --
*
diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index aa96d33a..773c56c1 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -58,6 +58,8 @@ #include "..\windows\TapDriver6\tap-windows.h" +#include <netcon.h> + // Create a fake unused default route to force detection of network type on networks without gateways #define ZT_WINDOWS_CREATE_FAKE_DEFAULT_ROUTE @@ -824,6 +826,61 @@ void WindowsEthernetTap::setFriendlyName(const char *dn) RegSetKeyValueA(ifp,"Connection","Name",REG_SZ,(LPCVOID)dn,(DWORD)(strlen(dn)+1)); RegCloseKey(ifp); } + + HRESULT hr = CoInitialize(nullptr); + if (hr != S_OK) return; + CoInitializeSecurity(NULL, -1, NULL, NULL, + RPC_C_AUTHN_LEVEL_PKT, + RPC_C_IMP_LEVEL_IMPERSONATE, + NULL, EOAC_NONE, NULL); + if (hr != S_OK) return; + + INetSharingManager *nsm; + hr = CoCreateInstance(__uuidof(NetSharingManager), NULL, CLSCTX_ALL, __uuidof(INetSharingManager), (void**)&nsm); + if (hr != S_OK) return; + + bool found = false; + INetSharingEveryConnectionCollection *nsecc = nullptr; + hr = nsm->get_EnumEveryConnection(&nsecc); + if (!nsecc) { + fprintf(stderr, "Failed to get NSM connections"); + return; + } + + IEnumVARIANT *ev = nullptr; + IUnknown *unk = nullptr; + hr = nsecc->get__NewEnum(&unk); + if (unk) { + hr = unk->QueryInterface(__uuidof(IEnumVARIANT), (void**)&ev); + unk->Release(); + } + if (ev) { + VARIANT v; + VariantInit(&v); + + while ((S_OK == ev->Next(1, &v, NULL)) && found == FALSE) { + if (V_VT(&v) == VT_UNKNOWN) { + INetConnection *nc = nullptr; + V_UNKNOWN(&v)->QueryInterface(__uuidof(INetConnection), (void**)&nc); + if (nc) { + NETCON_PROPERTIES *ncp = nullptr; + nc->GetProperties(&ncp); + + GUID curId = ncp->guidId; + if (curId == _deviceGuid) { + wchar_t wtext[255]; + mbstowcs(wtext, dn, strlen(dn)+1); + nc->Rename(wtext); + found = true; + } + nc->Release(); + } + } + VariantClear(&v); + } + ev->Release(); + } + nsecc->Release(); } void WindowsEthernetTap::scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed) diff --git a/osdep/WindowsEthernetTap.hpp b/osdep/WindowsEthernetTap.hpp index 1e36bdd8..78a95672 100644 --- a/osdep/WindowsEthernetTap.hpp +++ b/osdep/WindowsEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -71,7 +71,7 @@ public: static std::string destroyAllPersistentTapDevices(); /** - * Uninstall a specific persistent tap device by instance ID + * Uninstalls a specific persistent tap device by instance ID * * @param instanceId Device instance ID * @return Empty string on success, otherwise an error message diff --git a/osdep/freebsd_getifmaddrs.h b/osdep/freebsd_getifmaddrs.h index 166b6d23..963c422d 100644 --- a/osdep/freebsd_getifmaddrs.h +++ b/osdep/freebsd_getifmaddrs.h @@ -53,12 +53,12 @@ struct ifmaddrs { /* * Message format for use in obtaining information about multicast addresses - * from the routing socket + * from the routing socket. */ struct ifma_msghdr { int ifmam_msglen; /* to skip over non-understood messages */ int ifmam_version; /* future binary compatibility */ - int ifmam_type; /* message type */ + int ifmam_type; /* message type */ int ifmam_addrs; /* like rtm_addrs */ int ifmam_flags; /* value of ifa_flags */ int ifmam_index; /* index for associated ifp */ diff --git a/rule-compiler/rule-compiler.js b/rule-compiler/rule-compiler.js index 9cc4f76a..6e21c2dc 100644 --- a/rule-compiler/rule-compiler.js +++ b/rule-compiler/rule-compiler.js @@ -65,7 +65,8 @@ const OPEN_BLOCK_KEYWORDS = { 'tee': true, 'watch': true, 'redirect': true, - 'break': true + 'break': true, + 'priority': true }; // Reserved words that can't be used as tag, capability, or rule set names @@ -81,6 +82,7 @@ const RESERVED_WORDS = { 'watch': true, 'redirect': true, 'break': true, + 'priority': true, 'ztsrc': true, 'ztdest': true, @@ -131,6 +133,7 @@ const KEYWORD_TO_API_MAP = { 'watch': 'ACTION_WATCH', 'redirect': 'ACTION_REDIRECT', 'break': 'ACTION_BREAK', + 'priority': 'ACTION_PRIORITY', 'ztsrc': 'MATCH_SOURCE_ZEROTIER_ADDRESS', 'ztdest': 'MATCH_DEST_ZEROTIER_ADDRESS', diff --git a/selftest.cpp b/selftest.cpp index 4008cd2a..77c06cc0 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -670,52 +670,6 @@ static int testOther() std::cout << std::endl; #if 0 - std::cout << "[other] Benchmarking memcpy... "; std::cout.flush(); - { - unsigned char *bb = (unsigned char *)::malloc(1234567); - unsigned char *cc = (unsigned char *)::malloc(1234567); - for(unsigned int i=0;i<1234567;++i) - bb[i] = (unsigned char)i; - double bytes = 0.0; - uint64_t start = OSUtils::now(); - for(unsigned int i=0;i<20000;++i) { - ++bb[i]; - ++bb[i+1]; - memcpy(cc,bb,1234567); - bytes += 1234567.0; - } - if (cc[0] != bb[0]) - abort(); - uint64_t end = OSUtils::now(); - std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second" << std::endl; - ::free((void *)bb); - ::free((void *)cc); - } -#endif - - std::cout << "[other] Benchmarking ZT_FAST_MEMCPY... "; std::cout.flush(); - { - unsigned char *bb = (unsigned char *)::malloc(1234567); - unsigned char *cc = (unsigned char *)::malloc(1234567); - for(unsigned int i=0;i<1234567;++i) - bb[i] = (unsigned char)i; - double bytes = 0.0; - uint64_t start = OSUtils::now(); - for(unsigned int i=0;i<20000;++i) { - ++bb[0]; - ++bb[1234566]; - ZT_FAST_MEMCPY(cc,bb,1234567); - bytes += 1234567.0; - } - if (cc[0] != bb[0]) - abort(); - uint64_t end = OSUtils::now(); - std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second" << std::endl; - ::free((void *)bb); - ::free((void *)cc); - } - -#if 0 std::cout << "[other] Testing Hashtable... "; std::cout.flush(); { Hashtable<uint64_t,std::string> ht; @@ -999,7 +953,7 @@ struct TestPhyHandlers inline void phyOnUnixAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN) {} inline void phyOnUnixClose(PhySocket *sock,void **uptr) {} inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {} - inline void phyOnUnixWritable(PhySocket *sock,void **uptr,bool b) {} + inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {} #endif // __UNIX_LIKE__ inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {} diff --git a/service/OneService.cpp b/service/OneService.cpp index 389cdc91..40ffce2f 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -34,6 +34,9 @@ #include <vector> #include <algorithm> #include <list> +#include <thread> +#include <mutex> +#include <condition_variable> #include "../version.h" #include "../include/ZeroTierOne.h" @@ -57,6 +60,7 @@ #include "../osdep/PortMapper.hpp" #include "../osdep/Binder.hpp" #include "../osdep/ManagedRoute.hpp" +#include "../osdep/BlockingQueue.hpp" #include "OneService.hpp" #include "SoftwareUpdater.hpp" @@ -67,9 +71,12 @@ #include <ShlObj.h> #include <netioapi.h> #include <iphlpapi.h> +//#include <unistd.h> +#define stat _stat #else #include <sys/types.h> #include <sys/socket.h> +#include <sys/stat.h> #include <sys/wait.h> #include <unistd.h> #include <ifaddrs.h> @@ -81,11 +88,18 @@ #include "../ext/http-parser/http_parser.h" #endif +#if ZT_VAULT_SUPPORT +extern "C" { +#include <curl/curl.h> +} +#endif + #include "../ext/json/json.hpp" using json = nlohmann::json; #include "../controller/EmbeddedNetworkController.hpp" +#include "../controller/RabbitMQ.hpp" #ifdef ZT_USE_TEST_TAP @@ -99,14 +113,14 @@ namespace ZeroTier { typedef TestEthernetTap EthernetTap; } #include "../controller/EmbeddedNetworkController.hpp" #include "../node/Node.hpp" // Use the virtual netcon endpoint instead of a tun/tap port driver -#include "../include/VirtualTap.h" +#include "../include/VirtualTap.hpp" namespace ZeroTier { typedef VirtualTap EthernetTap; } #else #ifdef __APPLE__ -#include "../osdep/OSXEthernetTap.hpp" -namespace ZeroTier { typedef OSXEthernetTap EthernetTap; } +#include "../osdep/MacEthernetTap.hpp" +namespace ZeroTier { typedef MacEthernetTap EthernetTap; } #endif // __APPLE__ #ifdef __LINUX__ #include "../osdep/LinuxEthernetTap.hpp" @@ -129,10 +143,14 @@ namespace ZeroTier { typedef NetBSDEthernetTap EthernetTap; } namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } #endif // __OpenBSD__ -#endif // ZT_SERVICE_NETCON +#endif // ZT_SDK #endif // ZT_USE_TEST_TAP +#ifndef ZT_SOFTWARE_UPDATE_DEFAULT +#define ZT_SOFTWARE_UPDATE_DEFAULT "disable" +#endif + // Sanity limits for HTTP #define ZT_MAX_HTTP_MESSAGE_SIZE (1024 * 1024 * 64) #define ZT_MAX_HTTP_CONNECTIONS 65536 @@ -145,7 +163,9 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } #define ZT_TAP_CHECK_MULTICAST_INTERVAL 5000 // TCP fallback relay (run by ZeroTier, Inc. -- this will eventually go away) +#ifndef ZT_SDK #define ZT_TCP_FALLBACK_RELAY "204.80.128.1/443" +#endif // Frequency at which we re-resolve the TCP fallback relay #define ZT_TCP_FALLBACK_RERESOLVE_DELAY 86400000 @@ -162,10 +182,20 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } // TCP activity timeout #define ZT_TCP_ACTIVITY_TIMEOUT 60000 +#if ZT_VAULT_SUPPORT +size_t curlResponseWrite(void *ptr, size_t size, size_t nmemb, std::string *data) +{ + data->append((char*)ptr, size * nmemb); + return size * nmemb; +} +#endif + namespace ZeroTier { namespace { +static const InetAddress NULL_INET_ADDR; + // Fake TLS hello for TCP tunnel outgoing connections (TUNNELED mode) static const char ZT_TCP_TUNNEL_HELLO[9] = { 0x17,0x03,0x03,0x00,0x04,(char)ZEROTIER_ONE_VERSION_MAJOR,(char)ZEROTIER_ONE_VERSION_MINOR,(char)((ZEROTIER_ONE_VERSION_REVISION >> 8) & 0xff),(char)(ZEROTIER_ONE_VERSION_REVISION & 0xff) }; @@ -283,6 +313,39 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) pj["paths"] = pa; } +static void _peerAggregateLinkToJson(nlohmann::json &pj,const ZT_Peer *peer) +{ + char tmp[256]; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); + pj["aggregateLinkLatency"] = peer->latency; + + nlohmann::json pa = nlohmann::json::array(); + for(unsigned int i=0;i<peer->pathCount;++i) { + int64_t lastSend = peer->paths[i].lastSend; + int64_t lastReceive = peer->paths[i].lastReceive; + nlohmann::json j; + j["address"] = reinterpret_cast<const InetAddress *>(&(peer->paths[i].address))->toString(tmp); + j["lastSend"] = (lastSend < 0) ? 0 : lastSend; + j["lastReceive"] = (lastReceive < 0) ? 0 : lastReceive; + //j["trustedPathId"] = peer->paths[i].trustedPathId; + //j["active"] = (bool)(peer->paths[i].expired == 0); + //j["expired"] = (bool)(peer->paths[i].expired != 0); + //j["preferred"] = (bool)(peer->paths[i].preferred != 0); + j["latency"] = peer->paths[i].latency; + j["pdv"] = peer->paths[i].packetDelayVariance; + //j["throughputDisturbCoeff"] = peer->paths[i].throughputDisturbCoeff; + //j["packetErrorRatio"] = peer->paths[i].packetErrorRatio; + //j["packetLossRatio"] = peer->paths[i].packetLossRatio; + j["stability"] = peer->paths[i].stability; + j["throughput"] = peer->paths[i].throughput; + //j["maxThroughput"] = peer->paths[i].maxThroughput; + j["allocation"] = peer->paths[i].allocation; + j["ifname"] = peer->paths[i].ifname; + pa.push_back(j); + } + pj["paths"] = pa; +} + static void _moonToJson(nlohmann::json &mj,const World &world) { char tmp[4096]; @@ -384,6 +447,15 @@ struct TcpConnection Mutex writeq_m; }; +struct OneServiceIncomingPacket +{ + uint64_t now; + int64_t sock; + struct sockaddr_storage from; + unsigned int size; + uint8_t data[ZT_MAX_MTU]; +}; + class OneServiceImpl : public OneService { public: @@ -403,7 +475,11 @@ public: PhySocket *_localControlSocket6; bool _updateAutoApply; bool _allowTcpFallbackRelay; + bool _allowSecondaryPort; + unsigned int _multipathMode; unsigned int _primaryPort; + unsigned int _secondaryPort; + unsigned int _tertiaryPort; volatile unsigned int _udpPortPickerCounter; // Local configuration and memo-ized information from it @@ -418,11 +494,13 @@ public: std::vector< std::string > _interfacePrefixBlacklist; Mutex _localConfig_m; + std::vector<InetAddress> explicitBind; + /* * To attempt to handle NAT/gateway craziness we use three local UDP ports: * * [0] is the normal/default port, usually 9993 - * [1] is a port dervied from our ZeroTier address + * [1] is a port derived from our ZeroTier address * [2] is a port computed from the normal/default for use with uPnP/NAT-PMP mappings * * [2] exists because on some gateways trying to do regular NAT-t interferes @@ -481,10 +559,20 @@ public: PortMapper *_portMapper; #endif + // HashiCorp Vault Settings +#if ZT_VAULT_SUPPORT + bool _vaultEnabled; + std::string _vaultURL; + std::string _vaultToken; + std::string _vaultPath; // defaults to cubbyhole/zerotier/identity.secret for per-access key storage +#endif + // Set to false to force service to stop volatile bool _run; Mutex _run_m; + MQConfig *_mqc; + // end member variables ---------------------------------------------------- OneServiceImpl(const char *hp,unsigned int port) : @@ -513,11 +601,22 @@ public: #ifdef ZT_USE_MINIUPNPC ,_portMapper((PortMapper *)0) #endif +#ifdef ZT_VAULT_SUPPORT + ,_vaultEnabled(false) + ,_vaultURL() + ,_vaultToken() + ,_vaultPath("cubbyhole/zerotier") +#endif ,_run(true) + ,_mqc(NULL) { _ports[0] = 0; _ports[1] = 0; _ports[2] = 0; + +#if ZT_VAULT_SUPPORT + curl_global_init(CURL_GLOBAL_DEFAULT); +#endif } virtual ~OneServiceImpl() @@ -525,10 +624,16 @@ public: _binder.closeAll(_phy); _phy.close(_localControlSocket4); _phy.close(_localControlSocket6); + +#if ZT_VAULT_SUPPORT + curl_global_cleanup(); +#endif + #ifdef ZT_USE_MINIUPNPC delete _portMapper; #endif delete _controller; + delete _mqc; } virtual ReasonForTermination run() @@ -568,98 +673,8 @@ public: _node = new Node(this,(void *)0,&cb,OSUtils::now()); } - // Read local configuration - std::vector<InetAddress> explicitBind; - { - std::map<InetAddress,ZT_PhysicalPathConfiguration> ppc; - - // LEGACY: support old "trustedpaths" flat file - FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r"); - if (trustpaths) { - fprintf(stderr,"WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in local.conf" ZT_EOL_S); - char buf[1024]; - while (fgets(buf,sizeof(buf),trustpaths)) { - int fno = 0; - char *saveptr = (char *)0; - uint64_t trustedPathId = 0; - InetAddress trustedPathNetwork; - for(char *f=Utils::stok(buf,"=\r\n \t",&saveptr);(f);f=Utils::stok((char *)0,"=\r\n \t",&saveptr)) { - if (fno == 0) { - trustedPathId = Utils::hexStrToU64(f); - } else if (fno == 1) { - trustedPathNetwork = InetAddress(f); - } else break; - ++fno; - } - if ( (trustedPathId != 0) && ((trustedPathNetwork.ss_family == AF_INET)||(trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathNetwork.netmaskBits() > 0) ) { - ppc[trustedPathNetwork].trustedPathId = trustedPathId; - ppc[trustedPathNetwork].mtu = 0; // use default - } - } - fclose(trustpaths); - } - - // Read local config file - Mutex::Lock _l2(_localConfig_m); - std::string lcbuf; - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(),lcbuf)) { - try { - _localConfig = OSUtils::jsonParse(lcbuf); - if (!_localConfig.is_object()) { - fprintf(stderr,"WARNING: unable to parse local.conf (root element is not a JSON object)" ZT_EOL_S); - } - } catch ( ... ) { - fprintf(stderr,"WARNING: unable to parse local.conf (invalid JSON)" ZT_EOL_S); - } - } - - // Get any trusted paths in local.conf (we'll parse the rest of physical[] elsewhere) - json &physical = _localConfig["physical"]; - if (physical.is_object()) { - for(json::iterator phy(physical.begin());phy!=physical.end();++phy) { - InetAddress net(OSUtils::jsonString(phy.key(),"").c_str()); - if (net) { - if (phy.value().is_object()) { - uint64_t tpid; - if ((tpid = OSUtils::jsonInt(phy.value()["trustedPathId"],0ULL)) != 0ULL) { - if ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) - ppc[net].trustedPathId = tpid; - } - ppc[net].mtu = (int)OSUtils::jsonInt(phy.value()["mtu"],0ULL); // 0 means use default - } - } - } - } - - json &settings = _localConfig["settings"]; - if (settings.is_object()) { - // Allow controller DB path to be put somewhere else - const std::string cdbp(OSUtils::jsonString(settings["controllerDbPath"],"")); - if (cdbp.length() > 0) - _controllerDbPath = cdbp; - - // Bind to wildcard instead of to specific interfaces (disables full tunnel capability) - json &bind = settings["bind"]; - if (bind.is_array()) { - for(unsigned long i=0;i<bind.size();++i) { - const std::string ips(OSUtils::jsonString(bind[i],"")); - if (ips.length() > 0) { - InetAddress ip(ips.c_str()); - if ((ip.ss_family == AF_INET)||(ip.ss_family == AF_INET6)) - explicitBind.push_back(ip); - } - } - } - } - - // Set trusted paths if there are any - if (ppc.size() > 0) { - for(std::map<InetAddress,ZT_PhysicalPathConfiguration>::iterator i(ppc.begin());i!=ppc.end();++i) - _node->setPhysicalPathConfiguration(reinterpret_cast<const struct sockaddr_storage *>(&(i->first)),&(i->second)); - } - } - - // Apply other runtime configuration from local.conf + // local.conf + readLocalSettings(); applyLocalConfig(); // Make sure we can use the primary port, and hunt for one if configured to do so @@ -708,16 +723,18 @@ public: // This exists because there are buggy NATs out there that fail if more // than one device behind the same NAT tries to use the same internal // private address port number. Buggy NATs are a running theme. - _ports[1] = 20000 + ((unsigned int)_node->address() % 45500); - for(int i=0;;++i) { - if (i > 1000) { - _ports[1] = 0; - break; - } else if (++_ports[1] >= 65536) { - _ports[1] = 20000; + if (_allowSecondaryPort) { + _ports[1] = (_secondaryPort == 0) ? 20000 + ((unsigned int)_node->address() % 45500) : _secondaryPort; + for(int i=0;;++i) { + if (i > 1000) { + _ports[1] = 0; + break; + } else if (++_ports[1] >= 65536) { + _ports[1] = 20000; + } + if (_trialBind(_ports[1])) + break; } - if (_trialBind(_ports[1])) - break; } #ifdef ZT_USE_MINIUPNPC @@ -726,7 +743,7 @@ public: // use the other two ports for that because some NATs do really funky // stuff with ports that are explicitly mapped that breaks things. if (_ports[1]) { - _ports[2] = _ports[1]; + _ports[2] = (_tertiaryPort == 0) ? _ports[1] : _tertiaryPort; for(int i=0;;++i) { if (i > 1000) { _ports[2] = 0; @@ -750,7 +767,7 @@ public: OSUtils::rmDashRf((_homePath + ZT_PATH_SEPARATOR_S "iddb.d").c_str()); // Network controller is now enabled by default for desktop and server - _controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str()); + _controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str(),_ports[0], _mqc); _node->setNetconfMaster((void *)_controller); // Join existing networks in networks.d @@ -780,8 +797,10 @@ public: int64_t lastTapMulticastGroupCheck = 0; int64_t lastBindRefresh = 0; int64_t lastUpdateCheck = clockShouldBe; + int64_t lastMultipathModeUpdate = 0; int64_t lastCleanedPeersDb = 0; int64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle + int64_t lastLocalConfFileCheck = OSUtils::now(); for(;;) { _run_m.lock(); if (!_run) { @@ -810,8 +829,21 @@ public: _updater->apply(); } + // Reload local.conf if anything changed recently + if ((now - lastLocalConfFileCheck) >= ZT_LOCAL_CONF_FILE_CHECK_INTERVAL) { + lastLocalConfFileCheck = now; + struct stat result; + if(stat((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(), &result)==0) { + int64_t mod_time = result.st_mtime * 1000; + if ((now - mod_time) <= ZT_LOCAL_CONF_FILE_CHECK_INTERVAL) { + readLocalSettings(); + applyLocalConfig(); + } + } + } + // Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow routes (e.g. shadow default) - if (((now - lastBindRefresh) >= ZT_BINDER_REFRESH_PERIOD)||(restarted)) { + if (((now - lastBindRefresh) >= (_multipathMode ? ZT_BINDER_REFRESH_PERIOD / 8 : ZT_BINDER_REFRESH_PERIOD))||(restarted)) { lastBindRefresh = now; unsigned int p[3]; unsigned int pc = 0; @@ -828,6 +860,11 @@ public: } } } + // Update multipath mode (if needed) + if (((now - lastMultipathModeUpdate) >= ZT_BINDER_REFRESH_PERIOD / 8)||(restarted)) { + lastMultipathModeUpdate = now; + _node->setMultipathMode(_multipathMode); + } // Run background task processor in core if it's time to do so int64_t dl = _nextBackgroundTaskDeadline; @@ -863,7 +900,7 @@ public: } // Sync information about physical network interfaces - if ((now - lastLocalInterfaceAddressCheck) >= ZT_LOCAL_INTERFACE_CHECK_INTERVAL) { + if ((now - lastLocalInterfaceAddressCheck) >= (_multipathMode ? ZT_LOCAL_INTERFACE_CHECK_INTERVAL / 8 : ZT_LOCAL_INTERFACE_CHECK_INTERVAL)) { lastLocalInterfaceAddressCheck = now; _node->clearLocalInterfaceAddresses(); @@ -922,6 +959,109 @@ public: return _termReason; } + void readLocalSettings() + { + // Read local configuration + std::map<InetAddress,ZT_PhysicalPathConfiguration> ppc; + + // LEGACY: support old "trustedpaths" flat file + FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r"); + if (trustpaths) { + fprintf(stderr,"WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in local.conf" ZT_EOL_S); + char buf[1024]; + while (fgets(buf,sizeof(buf),trustpaths)) { + int fno = 0; + char *saveptr = (char *)0; + uint64_t trustedPathId = 0; + InetAddress trustedPathNetwork; + for(char *f=Utils::stok(buf,"=\r\n \t",&saveptr);(f);f=Utils::stok((char *)0,"=\r\n \t",&saveptr)) { + if (fno == 0) { + trustedPathId = Utils::hexStrToU64(f); + } else if (fno == 1) { + trustedPathNetwork = InetAddress(f); + } else break; + ++fno; + } + if ( (trustedPathId != 0) && ((trustedPathNetwork.ss_family == AF_INET)||(trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathNetwork.netmaskBits() > 0) ) { + ppc[trustedPathNetwork].trustedPathId = trustedPathId; + ppc[trustedPathNetwork].mtu = 0; // use default + } + } + fclose(trustpaths); + } + + // Read local config file + Mutex::Lock _l2(_localConfig_m); + std::string lcbuf; + if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(),lcbuf)) { + try { + _localConfig = OSUtils::jsonParse(lcbuf); + if (!_localConfig.is_object()) { + fprintf(stderr,"ERROR: unable to parse local.conf (root element is not a JSON object)" ZT_EOL_S); + exit(1); + } + } catch ( ... ) { + fprintf(stderr,"ERROR: unable to parse local.conf (invalid JSON)" ZT_EOL_S); + exit(1); + } + } + + // Get any trusted paths in local.conf (we'll parse the rest of physical[] elsewhere) + json &physical = _localConfig["physical"]; + if (physical.is_object()) { + for(json::iterator phy(physical.begin());phy!=physical.end();++phy) { + InetAddress net(OSUtils::jsonString(phy.key(),"").c_str()); + if (net) { + if (phy.value().is_object()) { + uint64_t tpid; + if ((tpid = OSUtils::jsonInt(phy.value()["trustedPathId"],0ULL)) != 0ULL) { + if ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) + ppc[net].trustedPathId = tpid; + } + ppc[net].mtu = (int)OSUtils::jsonInt(phy.value()["mtu"],0ULL); // 0 means use default + } + } + } + } + + json &settings = _localConfig["settings"]; + if (settings.is_object()) { + // Allow controller DB path to be put somewhere else + const std::string cdbp(OSUtils::jsonString(settings["controllerDbPath"],"")); + if (cdbp.length() > 0) + _controllerDbPath = cdbp; + + json &rmq = settings["rabbitmq"]; + if (rmq.is_object() && _mqc == NULL) { + fprintf(stderr, "Reading RabbitMQ Config\n"); + _mqc = new MQConfig; + _mqc->port = rmq["port"]; + _mqc->host = OSUtils::jsonString(rmq["host"], "").c_str(); + _mqc->username = OSUtils::jsonString(rmq["username"], "").c_str(); + _mqc->password = OSUtils::jsonString(rmq["password"], "").c_str(); + } + + // Bind to wildcard instead of to specific interfaces (disables full tunnel capability) + json &bind = settings["bind"]; + if (bind.is_array()) { + for(unsigned long i=0;i<bind.size();++i) { + const std::string ips(OSUtils::jsonString(bind[i],"")); + if (ips.length() > 0) { + InetAddress ip(ips.c_str()); + if ((ip.ss_family == AF_INET)||(ip.ss_family == AF_INET6)) + explicitBind.push_back(ip); + } + } + } + } + + // Set trusted paths if there are any + if (ppc.size() > 0) { + for(std::map<InetAddress,ZT_PhysicalPathConfiguration>::iterator i(ppc.begin());i!=ppc.end();++i) + _node->setPhysicalPathConfiguration(reinterpret_cast<const struct sockaddr_storage *>(&(i->first)),&(i->second)); + } + } + virtual ReasonForTermination reasonForTermination() const { Mutex::Lock _l(_termReason_m); @@ -944,44 +1084,26 @@ public: } #ifdef ZT_SDK - virtual void leave(const uint64_t hp) - { - _node->leave(hp, NULL, NULL); - } - - virtual void join(const uint64_t hp) - { - _node->join(hp, NULL, NULL); - } - virtual std::string givenHomePath() { return _homePath; } - std::vector<ZT_VirtualNetworkRoute> *getRoutes(uint64_t nwid) + void getRoutes(uint64_t nwid, void *routeArray, unsigned int *numRoutes) { Mutex::Lock _l(_nets_m); NetworkState &n = _nets[nwid]; - std::vector<ZT_VirtualNetworkRoute> *routes = new std::vector<ZT_VirtualNetworkRoute>(); - for(int i=0; i<ZT_MAX_NETWORK_ROUTES; i++) { - routes->push_back(n.config.routes[i]); + *numRoutes = *numRoutes < n.config.routeCount ? *numRoutes : n.config.routeCount; + for(unsigned int i=0; i<*numRoutes; i++) { + ZT_VirtualNetworkRoute *vnr = (ZT_VirtualNetworkRoute*)routeArray; + memcpy(&vnr[i], &(n.config.routes[i]), sizeof(ZT_VirtualNetworkRoute)); } - return routes; } virtual Node *getNode() { return _node; } - - virtual void removeNets() - { - Mutex::Lock _l(_nets_m); - std::map<uint64_t,NetworkState>::iterator i; - for(i = _nets.begin(); i != _nets.end(); i++) - delete i->second.tap; - } #endif // ZT_SDK virtual void terminate() @@ -1082,16 +1204,6 @@ public: #ifdef __SYNOLOGY__ // Authenticate via Synology's built-in cgi script if (!isAuth) { - /* - fprintf(stderr, "path = %s\n", path.c_str()); - fprintf(stderr, "headers.size=%d\n", headers.size()); - std::map<std::string, std::string>::const_iterator it(headers.begin()); - while(it != headers.end()) { - fprintf(stderr,"header[%s] = %s\n", (it->first).c_str(), (it->second).c_str()); - it++; - } - */ - // parse out url args int synotoken_pos = path.find("SynoToken"); int argpos = path.find("?"); if(synotoken_pos != std::string::npos && argpos != std::string::npos) { @@ -1104,10 +1216,6 @@ public: setenv("HTTP_COOKIE", cookie_val.c_str(), true); setenv("HTTP_X_SYNO_TOKEN", synotoken_val.c_str(), true); setenv("REMOTE_ADDR", ah2->second.c_str(),true); - //fprintf(stderr, "HTTP_COOKIE: %s\n",std::getenv ("HTTP_COOKIE")); - //fprintf(stderr, "HTTP_X_SYNO_TOKEN: %s\n",std::getenv ("HTTP_X_SYNO_TOKEN")); - //fprintf(stderr, "REMOTE_ADDR: %s\n",std::getenv ("REMOTE_ADDR")); - // check synology web auth char user[256], buf[1024]; FILE *fp = NULL; bzero(user, 256); @@ -1153,6 +1261,23 @@ public: json &settings = res["config"]["settings"]; settings["primaryPort"] = OSUtils::jsonInt(settings["primaryPort"],(uint64_t)_primaryPort) & 0xffff; settings["allowTcpFallbackRelay"] = OSUtils::jsonBool(settings["allowTcpFallbackRelay"],_allowTcpFallbackRelay); + + if (_multipathMode) { + json &multipathConfig = res["multipath"]; + ZT_PeerList *pl = _node->peers(); + char peerAddrStr[256]; + if (pl) { + for(unsigned long i=0;i<pl->peerCount;++i) { + if (pl->peers[i].hadAggregateLink) { + nlohmann::json pj; + _peerAggregateLinkToJson(pj,&(pl->peers[i])); + OSUtils::ztsnprintf(peerAddrStr,sizeof(peerAddrStr),"%.10llx",pl->peers[i].address); + multipathConfig[peerAddrStr] = (pj); + } + } + } + } + #ifdef ZT_USE_MINIUPNPC settings["portMappingEnabled"] = OSUtils::jsonBool(settings["portMappingEnabled"],true); #else @@ -1481,6 +1606,17 @@ public: _primaryPort = (unsigned int)OSUtils::jsonInt(settings["primaryPort"],(uint64_t)_primaryPort) & 0xffff; _allowTcpFallbackRelay = OSUtils::jsonBool(settings["allowTcpFallbackRelay"],true); + _allowSecondaryPort = OSUtils::jsonBool(settings["allowSecondaryPort"],true); + _secondaryPort = (unsigned int)OSUtils::jsonInt(settings["secondaryPort"],0); + _tertiaryPort = (unsigned int)OSUtils::jsonInt(settings["tertiaryPort"],0); + if (_secondaryPort != 0 || _tertiaryPort != 0) { + fprintf(stderr,"WARNING: using manually-specified ports. This can cause NAT issues." ZT_EOL_S); + } + _multipathMode = (unsigned int)OSUtils::jsonInt(settings["multipathMode"],0); + if (_multipathMode != 0 && _allowTcpFallbackRelay) { + fprintf(stderr,"WARNING: multipathMode cannot be used with allowTcpFallbackRelay. Disabling allowTcpFallbackRelay" ZT_EOL_S); + _allowTcpFallbackRelay = false; + } _portMappingEnabled = OSUtils::jsonBool(settings["portMappingEnabled"],true); #ifndef ZT_SDK @@ -1518,6 +1654,47 @@ public: } } +#if ZT_VAULT_SUPPORT + json &vault = settings["vault"]; + if (vault.is_object()) { + const std::string url(OSUtils::jsonString(vault["vaultURL"], "").c_str()); + if (!url.empty()) { + _vaultURL = url; + } + + const std::string token(OSUtils::jsonString(vault["vaultToken"], "").c_str()); + if (!token.empty()) { + _vaultToken = token; + } + + const std::string path(OSUtils::jsonString(vault["vaultPath"], "").c_str()); + if (!path.empty()) { + _vaultPath = path; + } + } + + // also check environment variables for values. Environment variables + // will override local.conf variables + const std::string envURL(getenv("VAULT_ADDR")); + if (!envURL.empty()) { + _vaultURL = envURL; + } + + const std::string envToken(getenv("VAULT_TOKEN")); + if (!envToken.empty()) { + _vaultToken = envToken; + } + + const std::string envPath(getenv("VAULT_PATH")); + if (!envPath.empty()) { + _vaultPath = envPath; + } + + if (!_vaultURL.empty() && !_vaultToken.empty()) { + _vaultEnabled = true; + } +#endif + // Checks if a managed IP or route target is allowed bool checkIfManagedIsAllowed(const NetworkState &n,const InetAddress &target) { @@ -1632,21 +1809,37 @@ public: const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target)); const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via)); + const InetAddress *src = NULL; + for (unsigned int j=0; j<n.config.assignedAddressCount; ++j) { + const InetAddress *const tmp = reinterpret_cast<const InetAddress *>(&(n.config.assignedAddresses[j])); + if (target->isV4() && tmp->isV4()) { + src = reinterpret_cast<InetAddress *>(&(n.config.assignedAddresses[j])); + break; + } else if (target->isV6() && tmp->isV6()) { + src = reinterpret_cast<InetAddress *>(&(n.config.assignedAddresses[j])); + break; + } + } + if (!src) + src = &NULL_INET_ADDR; + if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) ) continue; bool haveRoute = false; // Ignore routes implied by local managed IPs since adding the IP adds the route +#ifndef __APPLE__ for(std::vector<InetAddress>::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) { haveRoute = true; break; } } +#endif if (haveRoute) continue; - +#ifndef ZT_SDK // If we've already applied this route, just sync it and continue for(std::list< SharedPtr<ManagedRoute> >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();++mr) { if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (tapdev == (*mr)->device()) ) ) { @@ -1659,9 +1852,10 @@ public: continue; // Add and apply new routes - n.managedRoutes.push_back(SharedPtr<ManagedRoute>(new ManagedRoute(*target,*via,tapdev))); + n.managedRoutes.push_back(SharedPtr<ManagedRoute>(new ManagedRoute(*target,*via,*src,tapdev))); if (!n.managedRoutes.back()->sync()) n.managedRoutes.pop_back(); +#endif } } } @@ -1672,16 +1866,10 @@ public: inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) { + const uint64_t now = OSUtils::now(); if ((len >= 16)&&(reinterpret_cast<const InetAddress *>(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) - _lastDirectReceiveFromGlobal = OSUtils::now(); - const ZT_ResultCode rc = _node->processWirePacket( - (void *)0, - OSUtils::now(), - reinterpret_cast<int64_t>(sock), - reinterpret_cast<const struct sockaddr_storage *>(from), // Phy<> uses sockaddr_storage, so it'll always be that big - data, - len, - &_nextBackgroundTaskDeadline); + _lastDirectReceiveFromGlobal = now; + const ZT_ResultCode rc = _node->processWirePacket(nullptr,now,reinterpret_cast<int64_t>(sock),reinterpret_cast<const struct sockaddr_storage *>(from),data,len,&_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; OSUtils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); @@ -1722,6 +1910,13 @@ public: _phy.close(sockN,false); return; } else { +#ifdef ZT_SDK + // Immediately close new local connections. The intention is to prevent the backplane from being accessed when operating as libzt + if (!allowHttpBackplaneManagement && ((InetAddress*)from)->ipScope() == InetAddress::IP_SCOPE_LOOPBACK) { + _phy.close(sockN,false); + return; + } +#endif TcpConnection *tc = new TcpConnection(); { Mutex::Lock _l(_tcpConnections_m); @@ -1921,7 +2116,7 @@ public: inline void phyOnUnixAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN) {} inline void phyOnUnixClose(PhySocket *sock,void **uptr) {} inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {} - inline void phyOnUnixWritable(PhySocket *sock,void **uptr,bool lwip_invoked) {} + inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {} inline int nodeVirtualNetworkConfigFunction(uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwc) { @@ -1999,7 +2194,7 @@ public: // After setting up tap, fall through to CONFIG_UPDATE since we also want to do this... case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE: - ZT_FAST_MEMCPY(&(n.config),nwc,sizeof(ZT_VirtualNetworkConfig)); + memcpy(&(n.config),nwc,sizeof(ZT_VirtualNetworkConfig)); if (n.tap) { // sanity check #if defined(__WINDOWS__) && !defined(ZT_SDK) // wait for up to 5 seconds for the WindowsEthernetTap to actually be initialized @@ -2081,8 +2276,88 @@ public: } } +#if ZT_VAULT_SUPPORT + inline bool nodeVaultPutIdentity(enum ZT_StateObjectType type, const void *data, int len) + { + bool retval = false; + if (type != ZT_STATE_OBJECT_IDENTITY_PUBLIC && type != ZT_STATE_OBJECT_IDENTITY_SECRET) { + return retval; + } + + CURL *curl = curl_easy_init(); + if (curl) { + char token[512] = { 0 }; + snprintf(token, sizeof(token), "X-Vault-Token: %s", _vaultToken.c_str()); + + struct curl_slist *chunk = NULL; + chunk = curl_slist_append(chunk, token); + + + char content_type[512] = { 0 }; + snprintf(content_type, sizeof(content_type), "Content-Type: application/json"); + + chunk = curl_slist_append(chunk, content_type); + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); + + char url[2048] = { 0 }; + snprintf(url, sizeof(url), "%s/v1/%s", _vaultURL.c_str(), _vaultPath.c_str()); + + curl_easy_setopt(curl, CURLOPT_URL, url); + + json d = json::object(); + if (type == ZT_STATE_OBJECT_IDENTITY_PUBLIC) { + std::string key((const char*)data, len); + d["public"] = key; + } + else if (type == ZT_STATE_OBJECT_IDENTITY_SECRET) { + std::string key((const char*)data, len); + d["secret"] = key; + } + + if (!d.empty()) { + std::string post = d.dump(); + + if (!post.empty()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post.length()); + +#ifndef NDEBUG + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); +#endif + + CURLcode res = curl_easy_perform(curl); + if (res == CURLE_OK) { + long response_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (response_code == 200 || response_code == 204) { + retval = true; + } + } + } + } + + curl_easy_cleanup(curl); + curl = NULL; + curl_slist_free_all(chunk); + chunk = NULL; + } + + return retval; + } +#endif + inline void nodeStatePutFunction(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) { +#if ZT_VAULT_SUPPORT + if (_vaultEnabled && (type == ZT_STATE_OBJECT_IDENTITY_SECRET || type == ZT_STATE_OBJECT_IDENTITY_PUBLIC)) { + if (nodeVaultPutIdentity(type, data, len)) { + // value successfully written to Vault + return; + } + // else fallback to disk + } +#endif char p[1024]; FILE *f; bool secure = false; @@ -2117,17 +2392,22 @@ public: return; } - if (len >= 0) { + if ((len >= 0)&&(data)) { // Check to see if we've already written this first. This reduces // redundant writes and I/O overhead on most platforms and has // little effect on others. f = fopen(p,"rb"); if (f) { - char buf[65535]; - long l = (long)fread(buf,1,sizeof(buf),f); - fclose(f); - if ((l == (long)len)&&(memcmp(data,buf,l) == 0)) - return; + char *const buf = (char *)malloc(len*4); + if (buf) { + long l = (long)fread(buf,1,(size_t)(len*4),f); + fclose(f); + if ((l == (long)len)&&(memcmp(data,buf,l) == 0)) { + free(buf); + return; + } + free(buf); + } } f = fopen(p,"wb"); @@ -2149,8 +2429,93 @@ public: } } +#if ZT_VAULT_SUPPORT + inline int nodeVaultGetIdentity(enum ZT_StateObjectType type, void *data, unsigned int maxlen) + { + if (type != ZT_STATE_OBJECT_IDENTITY_SECRET && type != ZT_STATE_OBJECT_IDENTITY_PUBLIC) { + return -1; + } + + int ret = -1; + CURL *curl = curl_easy_init(); + if (curl) { + char token[512] = { 0 }; + snprintf(token, sizeof(token), "X-Vault-Token: %s", _vaultToken.c_str()); + + struct curl_slist *chunk = NULL; + chunk = curl_slist_append(chunk, token); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); + + char url[2048] = { 0 }; + snprintf(url, sizeof(url), "%s/v1/%s", _vaultURL.c_str(), _vaultPath.c_str()); + + curl_easy_setopt(curl, CURLOPT_URL, url); + + std::string response; + std::string res_headers; + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &curlResponseWrite); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &res_headers); + +#ifndef NDEBUG + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); +#endif + + CURLcode res = curl_easy_perform(curl); + + if (res == CURLE_OK) { + long response_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (response_code == 200) { + try { + json payload = json::parse(response); + if (!payload["data"].is_null()) { + json &d = payload["data"]; + if (type == ZT_STATE_OBJECT_IDENTITY_SECRET) { + std::string secret = OSUtils::jsonString(d["secret"],""); + + if (!secret.empty()) { + ret = (int)secret.length(); + memcpy(data, secret.c_str(), ret); + } + } + else if (type == ZT_STATE_OBJECT_IDENTITY_PUBLIC) { + std::string pub = OSUtils::jsonString(d["public"],""); + + if (!pub.empty()) { + ret = (int)pub.length(); + memcpy(data, pub.c_str(), ret); + } + } + } + } + catch (...) { + ret = -1; + } + } + } + + curl_easy_cleanup(curl); + curl = NULL; + curl_slist_free_all(chunk); + chunk = NULL; + } + return ret; + } +#endif + inline int nodeStateGetFunction(enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen) { +#if ZT_VAULT_SUPPORT + if (_vaultEnabled && (type == ZT_STATE_OBJECT_IDENTITY_SECRET || type == ZT_STATE_OBJECT_IDENTITY_PUBLIC) ) { + int retval = nodeVaultGetIdentity(type, data, maxlen); + if (retval >= 0) + return retval; + + // else continue file based lookup + } +#endif char p[4096]; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: @@ -2178,6 +2543,17 @@ public: if (f) { int n = (int)fread(data,1,maxlen,f); fclose(f); +#if ZT_VAULT_SUPPORT + if (_vaultEnabled && (type == ZT_STATE_OBJECT_IDENTITY_SECRET || type == ZT_STATE_OBJECT_IDENTITY_PUBLIC)) { + // If we've gotten here while Vault is enabled, Vault does not know the key and it's been + // read from disk instead. + // + // We should put the value in Vault and remove the local file. + if (nodeVaultPutIdentity(type, data, n)) { + unlink(p); + } + } +#endif if (n >= 0) return n; } @@ -2330,7 +2706,7 @@ public: else return 0; const std::vector<InetAddress> *l = lh->get(ztaddr); if ((l)&&(l->size() > 0)) { - ZT_FAST_MEMCPY(result,&((*l)[(unsigned long)_node->prng() % l->size()]),sizeof(struct sockaddr_storage)); + memcpy(result,&((*l)[(unsigned long)_node->prng() % l->size()]),sizeof(struct sockaddr_storage)); return 1; } else return 0; } @@ -2403,6 +2779,7 @@ public: #endif #ifdef __APPLE__ + if ((ifname[0] == 'f')&&(ifname[1] == 'e')&&(ifname[2] == 't')&&(ifname[3] == 'h')) return false; // ... as is feth# if ((ifname[0] == 'l')&&(ifname[1] == 'o')) return false; // loopback if ((ifname[0] == 'z')&&(ifname[1] == 't')) return false; // sanity check: zt# if ((ifname[0] == 't')&&(ifname[1] == 'u')&&(ifname[2] == 'n')) return false; // tun# is probably an OpenVPN tunnel or similar diff --git a/service/OneService.hpp b/service/OneService.hpp index 3b670e4a..e5125676 100644 --- a/service/OneService.hpp +++ b/service/OneService.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -30,16 +30,17 @@ #include <string> #include <vector> -#include "../node/InetAddress.hpp" +namespace ZeroTier { #ifdef ZT_SDK -#include "../node/Node.hpp" -// Use the virtual netcon endpoint instead of a tun/tap port driver -#include "../include/VirtualTap.h" +class VirtualTap; +// Use the virtual libzt endpoint instead of a tun/tap port driver namespace ZeroTier { typedef VirtualTap EthernetTap; } #endif -namespace ZeroTier { +// Forward declaration so we can avoid dragging everything in +struct InetAddress; +class Node; /** * Local service for ZeroTier One as system VPN/NFV provider @@ -147,12 +148,18 @@ public: virtual std::string portDeviceName(uint64_t nwid) const = 0; #ifdef ZT_SDK - virtual void leave(const uint64_t hp) = 0; - virtual void join(const uint64_t hp) = 0; - virtual std::string givenHomePath() = 0; + /** + * Whether we allow access to the service via local HTTP requests (disabled by default in libzt) + */ + bool allowHttpBackplaneManagement = false; + /** + * @return Reference to the Node + */ virtual Node * getNode() = 0; - virtual void removeNets() = 0; - virtual std::vector<ZT_VirtualNetworkRoute> *getRoutes(uint64_t nwid) = 0; + /** + * Fills out a structure with network-specific route information + */ + virtual void getRoutes(uint64_t nwid, void *routeArray, unsigned int *numRoutes) = 0; #endif /** diff --git a/service/README.md b/service/README.md index 59d414aa..c77ee511 100644 --- a/service/README.md +++ b/service/README.md @@ -25,7 +25,9 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al } }, "settings": { /* Other global settings */ - "primaryPort": 0-65535, /* If set, override default port of 9993 and any command line port */ + "primaryPort": 1-65535, /* If set, override default port of 9993 and any command line port */ + "secondaryPort": 1-65535, /* If set, override default random secondary port */ + "tertiaryPort": 1-65535, /* If set, override default random tertiary port */ "portMappingEnabled": true|false, /* If true (the default), try to use uPnP or NAT-PMP to map ports */ "softwareUpdate": "apply"|"download"|"disable", /* Automatically apply updates, just download, or disable built-in software updates */ "softwareUpdateChannel": "release"|"beta", /* Software update channel */ @@ -33,7 +35,8 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al "interfacePrefixBlacklist": [ "XXX",... ], /* Array of interface name prefixes (e.g. eth for eth#) to blacklist for ZT traffic */ "allowManagementFrom": [ "NETWORK/bits", ...] |null, /* If non-NULL, allow JSON/HTTP management from this IP network. Default is 127.0.0.1 only. */ "bind": [ "ip",... ], /* If present and non-null, bind to these IPs instead of to each interface (wildcard IP allowed) */ - "allowTcpFallbackRelay": true|false /* Allow or disallow establishment of TCP relay connections (true by default) */ + "allowTcpFallbackRelay": true|false, /* Allow or disallow establishment of TCP relay connections (true by default) */ + "multipathMode": 0|1|2 /* multipath mode: none (0), random (1), proportional (2) */ } } ``` diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index d12861d4..df3d89c2 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index 7f73eba2..1fab7740 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * diff --git a/update_controllers.sh b/update_controllers.sh new file mode 100755 index 00000000..ad3f95a3 --- /dev/null +++ b/update_controllers.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +if [ -z "$1" ] +then + echo "Usage: $0 <docker_tag> <k8s_namespace>" + echo " k8s_namesapce is set to default if not specified" + exit 1 +fi + +if [ -z "$2" ] +then + NAMESPACE=default +else + NAMESPACE=$2 +fi + +TAG=$1 + +CONTROLLERS=`kubectl get pods -o=name --namespace=${NAMESPACE} | grep controller | sed "s/^.\{4\}//" | cut -d '-' -f 2` + +for c in ${CONTROLLERS[@]} +do + kubectl set image deployment --namespace=${NAMESPACE} controller-${c} ztcentral-controller=docker.zerotier.com/zerotier-central/ztcentral-controller:${TAG} + echo "Sleeping for 1 minute" + sleep 60 +done @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2018 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2019 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * along with this program. If not, see <http://www.gnu.org/licenses/>. * * -- * @@ -35,12 +35,12 @@ /** * Minor version */ -#define ZEROTIER_ONE_VERSION_MINOR 2 +#define ZEROTIER_ONE_VERSION_MINOR 4 /** * Revision */ -#define ZEROTIER_ONE_VERSION_REVISION 12 +#define ZEROTIER_ONE_VERSION_REVISION 0 /** * Build version diff --git a/windows/WinUI/APIHandler.cs b/windows/WinUI/APIHandler.cs index 015415c3..6a944f0f 100644 --- a/windows/WinUI/APIHandler.cs +++ b/windows/WinUI/APIHandler.cs @@ -12,7 +12,7 @@ using System.Windows.Threading; namespace WinUI { - + public class APIHandler { @@ -58,19 +58,19 @@ namespace WinUI String authToken = ""; Int32 port = 9993; - if (resetToken) - { - instance = null; - if (File.Exists(localZtDir + "\\authtoken.secret")) - { - File.Delete(localZtDir + "\\authtoken.secret"); - } + if (resetToken) + { + instance = null; + if (File.Exists(localZtDir + "\\authtoken.secret")) + { + File.Delete(localZtDir + "\\authtoken.secret"); + } - if (File.Exists(localZtDir + "\\zerotier-one.port")) - { - File.Delete(localZtDir + "\\zerotier-one.port"); - } - } + if (File.Exists(localZtDir + "\\zerotier-one.port")) + { + File.Delete(localZtDir + "\\zerotier-one.port"); + } + } if (!File.Exists(localZtDir + "\\authtoken.secret") || !File.Exists(localZtDir + "\\zerotier-one.port")) { @@ -78,7 +78,7 @@ namespace WinUI String curPath = System.Reflection.Assembly.GetEntryAssembly().Location; int index = curPath.LastIndexOf("\\"); curPath = curPath.Substring(0, index); - ProcessStartInfo startInfo = new ProcessStartInfo(curPath + "\\copyutil.exe", "\""+globalZtDir+"\"" + " " + "\""+localZtDir+"\""); + ProcessStartInfo startInfo = new ProcessStartInfo(curPath + "\\copyutil.exe", "\"" + globalZtDir + "\"" + " " + "\"" + localZtDir + "\""); startInfo.Verb = "runas"; @@ -148,7 +148,7 @@ namespace WinUI this.authtoken = authtoken; } - + public void GetStatus(StatusCallback cb) { @@ -162,33 +162,33 @@ namespace WinUI try { var httpResponse = (HttpWebResponse)request.GetResponse(); - if (httpResponse.StatusCode == HttpStatusCode.OK) - { - using (var streamReader = new StreamReader(httpResponse.GetResponseStream())) - { - var responseText = streamReader.ReadToEnd(); + if (httpResponse.StatusCode == HttpStatusCode.OK) + { + using (var streamReader = new StreamReader(httpResponse.GetResponseStream())) + { + var responseText = streamReader.ReadToEnd(); - ZeroTierStatus status = null; - try - { - status = JsonConvert.DeserializeObject<ZeroTierStatus>(responseText); + ZeroTierStatus status = null; + try + { + status = JsonConvert.DeserializeObject<ZeroTierStatus>(responseText); if (ZeroTierAddress != status.Address) { ZeroTierAddress = status.Address; } - } - catch (JsonReaderException e) - { - Console.WriteLine(e.ToString()); - } - cb(status); - } - } - else if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } + } + catch (JsonReaderException e) + { + Console.WriteLine(e.ToString()); + } + cb(status); + } + } + else if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } } catch (System.Net.Sockets.SocketException) { @@ -196,19 +196,19 @@ namespace WinUI } catch (System.Net.WebException e) { - HttpWebResponse res = (HttpWebResponse)e.Response; - if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } - else - { - cb(null); - } + HttpWebResponse res = (HttpWebResponse)e.Response; + if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } + else + { + cb(null); + } } } - + public void GetNetworks(NetworkListCallback cb) { @@ -226,33 +226,33 @@ namespace WinUI { var httpResponse = (HttpWebResponse)request.GetResponse(); - if (httpResponse.StatusCode == HttpStatusCode.OK) - { - using (var streamReader = new StreamReader(httpResponse.GetResponseStream())) - { - var responseText = streamReader.ReadToEnd(); - - List<ZeroTierNetwork> networkList = null; - try - { - networkList = JsonConvert.DeserializeObject<List<ZeroTierNetwork>>(responseText); - foreach (ZeroTierNetwork n in networkList) - { - // all networks received via JSON are connected by definition - n.IsConnected = true; - } - } - catch (JsonReaderException e) - { - Console.WriteLine(e.ToString()); - } - cb(networkList); - } - } - else if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } + if (httpResponse.StatusCode == HttpStatusCode.OK) + { + using (var streamReader = new StreamReader(httpResponse.GetResponseStream())) + { + var responseText = streamReader.ReadToEnd(); + + List<ZeroTierNetwork> networkList = null; + try + { + networkList = JsonConvert.DeserializeObject<List<ZeroTierNetwork>>(responseText); + foreach (ZeroTierNetwork n in networkList) + { + // all networks received via JSON are connected by definition + n.IsConnected = true; + } + } + catch (JsonReaderException e) + { + Console.WriteLine(e.ToString()); + } + cb(networkList); + } + } + else if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } } catch (System.Net.Sockets.SocketException) { @@ -260,137 +260,137 @@ namespace WinUI } catch (System.Net.WebException e) { - HttpWebResponse res = (HttpWebResponse)e.Response; - if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } - else - { - cb(null); - } + HttpWebResponse res = (HttpWebResponse)e.Response; + if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } + else + { + cb(null); + } } } public void JoinNetwork(Dispatcher d, string nwid, bool allowManaged = true, bool allowGlobal = false, bool allowDefault = false) { - Task.Factory.StartNew(() => - { - var request = WebRequest.Create(url + "/network/" + nwid + "?auth=" + authtoken) as HttpWebRequest; - if (request == null) - { - return; - } - - request.Method = "POST"; - request.ContentType = "applicaiton/json"; - request.Timeout = 30000; - try - { - using (var streamWriter = new StreamWriter(((HttpWebRequest)request).GetRequestStream())) - { - string json = "{\"allowManaged\":" + (allowManaged ? "true" : "false") + "," + - "\"allowGlobal\":" + (allowGlobal ? "true" : "false") + "," + - "\"allowDefault\":" + (allowDefault ? "true" : "false") + "}"; - streamWriter.Write(json); - streamWriter.Flush(); - streamWriter.Close(); - } - } - catch (System.Net.WebException) - { - d.BeginInvoke(DispatcherPriority.Normal, new Action(() => - { - MessageBox.Show("Error Joining Network: Cannot connect to ZeroTier service."); - })); - return; - } - - try - { - var httpResponse = (HttpWebResponse)request.GetResponse(); - - if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } - else if (httpResponse.StatusCode != HttpStatusCode.OK) - { - Console.WriteLine("Error sending join network message"); - } - } - catch (System.Net.Sockets.SocketException) - { - d.BeginInvoke(DispatcherPriority.Normal, new Action(() => - { - MessageBox.Show("Error Joining Network: Cannot connect to ZeroTier service."); - })); - } - catch (System.Net.WebException e) - { - HttpWebResponse res = (HttpWebResponse)e.Response; - if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } - d.BeginInvoke(DispatcherPriority.Normal, new Action(() => - { - MessageBox.Show("Error Joining Network: Cannot connect to ZeroTier service."); - })); - } - }); + Task.Factory.StartNew(() => + { + var request = WebRequest.Create(url + "/network/" + nwid + "?auth=" + authtoken) as HttpWebRequest; + if (request == null) + { + return; + } + + request.Method = "POST"; + request.ContentType = "applicaiton/json"; + request.Timeout = 30000; + try + { + using (var streamWriter = new StreamWriter(((HttpWebRequest)request).GetRequestStream())) + { + string json = "{\"allowManaged\":" + (allowManaged ? "true" : "false") + "," + + "\"allowGlobal\":" + (allowGlobal ? "true" : "false") + "," + + "\"allowDefault\":" + (allowDefault ? "true" : "false") + "}"; + streamWriter.Write(json); + streamWriter.Flush(); + streamWriter.Close(); + } + } + catch (System.Net.WebException) + { + d.BeginInvoke(DispatcherPriority.Normal, new Action(() => + { + MessageBox.Show("Error Joining Network: Cannot connect to ZeroTier service."); + })); + return; + } + + try + { + var httpResponse = (HttpWebResponse)request.GetResponse(); + + if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } + else if (httpResponse.StatusCode != HttpStatusCode.OK) + { + Console.WriteLine("Error sending join network message"); + } + } + catch (System.Net.Sockets.SocketException) + { + d.BeginInvoke(DispatcherPriority.Normal, new Action(() => + { + MessageBox.Show("Error Joining Network: Cannot connect to ZeroTier service."); + })); + } + catch (System.Net.WebException e) + { + HttpWebResponse res = (HttpWebResponse)e.Response; + if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } + d.BeginInvoke(DispatcherPriority.Normal, new Action(() => + { + MessageBox.Show("Error Joining Network: Cannot connect to ZeroTier service."); + })); + } + }); } public void LeaveNetwork(Dispatcher d, string nwid) { - Task.Factory.StartNew(() => - { - var request = WebRequest.Create(url + "/network/" + nwid + "?auth=" + authtoken) as HttpWebRequest; - if (request == null) - { - return; - } - - request.Method = "DELETE"; - request.Timeout = 30000; - - try - { - var httpResponse = (HttpWebResponse)request.GetResponse(); - - if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } - else if (httpResponse.StatusCode != HttpStatusCode.OK) - { - Console.WriteLine("Error sending leave network message"); - } - } - catch (System.Net.Sockets.SocketException) - { - d.BeginInvoke(DispatcherPriority.Normal, new Action(() => - { - MessageBox.Show("Error Leaving Network: Cannot connect to ZeroTier service."); - })); - } - catch (System.Net.WebException e) - { - HttpWebResponse res = (HttpWebResponse)e.Response; - if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } - d.BeginInvoke(DispatcherPriority.Normal, new Action(() => - { - MessageBox.Show("Error Leaving Network: Cannot connect to ZeroTier service."); - })); - } - catch - { - Console.WriteLine("Error leaving network: Unknown error"); - } - }); + Task.Factory.StartNew(() => + { + var request = WebRequest.Create(url + "/network/" + nwid + "?auth=" + authtoken) as HttpWebRequest; + if (request == null) + { + return; + } + + request.Method = "DELETE"; + request.Timeout = 30000; + + try + { + var httpResponse = (HttpWebResponse)request.GetResponse(); + + if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } + else if (httpResponse.StatusCode != HttpStatusCode.OK) + { + Console.WriteLine("Error sending leave network message"); + } + } + catch (System.Net.Sockets.SocketException) + { + d.BeginInvoke(DispatcherPriority.Normal, new Action(() => + { + MessageBox.Show("Error Leaving Network: Cannot connect to ZeroTier service."); + })); + } + catch (System.Net.WebException e) + { + HttpWebResponse res = (HttpWebResponse)e.Response; + if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } + d.BeginInvoke(DispatcherPriority.Normal, new Action(() => + { + MessageBox.Show("Error Leaving Network: Cannot connect to ZeroTier service."); + })); + } + catch + { + Console.WriteLine("Error leaving network: Unknown error"); + } + }); } public delegate void PeersCallback(List<ZeroTierPeer> peers); @@ -409,28 +409,28 @@ namespace WinUI try { var httpResponse = (HttpWebResponse)request.GetResponse(); - if (httpResponse.StatusCode == HttpStatusCode.OK) - { - using (var streamReader = new StreamReader(httpResponse.GetResponseStream())) - { - var responseText = streamReader.ReadToEnd(); - //Console.WriteLine(responseText); - List<ZeroTierPeer> peerList = null; - try - { - peerList = JsonConvert.DeserializeObject<List<ZeroTierPeer>>(responseText); - } - catch (JsonReaderException e) - { - Console.WriteLine(e.ToString()); - } - cb(peerList); - } - } - else if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } + if (httpResponse.StatusCode == HttpStatusCode.OK) + { + using (var streamReader = new StreamReader(httpResponse.GetResponseStream())) + { + var responseText = streamReader.ReadToEnd(); + //Console.WriteLine(responseText); + List<ZeroTierPeer> peerList = null; + try + { + peerList = JsonConvert.DeserializeObject<List<ZeroTierPeer>>(responseText); + } + catch (JsonReaderException e) + { + Console.WriteLine(e.ToString()); + } + cb(peerList); + } + } + else if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } } catch (System.Net.Sockets.SocketException) { @@ -438,15 +438,15 @@ namespace WinUI } catch (System.Net.WebException e) { - HttpWebResponse res = (HttpWebResponse)e.Response; - if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) - { - APIHandler.initHandler(true); - } - else - { - cb(null); - } + HttpWebResponse res = (HttpWebResponse)e.Response; + if (res != null && res.StatusCode == HttpStatusCode.Unauthorized) + { + APIHandler.initHandler(true); + } + else + { + cb(null); + } } } diff --git a/windows/WinUI/AboutView.xaml b/windows/WinUI/AboutView.xaml index 118a61b1..8d715fd2 100644 --- a/windows/WinUI/AboutView.xaml +++ b/windows/WinUI/AboutView.xaml @@ -19,9 +19,9 @@ <Run Text="ZeroTier One"/> </Paragraph> <Paragraph TextAlignment="Center"> - <Run FontSize="14" Text="Version 1.2.12"/> + <Run FontSize="14" Text="Version 1.2.99 (1.4.0pre)"/> <LineBreak/> - <Run FontSize="14" Text="(c) 2011-2018 ZeroTier, Inc."/> + <Run FontSize="14" Text="(c) 2011-2019 ZeroTier, Inc."/> <LineBreak/> <Run FontSize="14" Text="www.zerotier.com"/> </Paragraph> diff --git a/windows/WinUI/CentralAPI.cs b/windows/WinUI/CentralAPI.cs index 8c36f455..b0b4a267 100644 --- a/windows/WinUI/CentralAPI.cs +++ b/windows/WinUI/CentralAPI.cs @@ -67,7 +67,7 @@ namespace WinUI UseCookies = true, CookieContainer = cookieContainer }; - + client = new HttpClient(clientHandler); string centralConfigPath = CentralConfigFile(); diff --git a/windows/WinUI/NetworkMonitor.cs b/windows/WinUI/NetworkMonitor.cs index ce722e45..927f804c 100644 --- a/windows/WinUI/NetworkMonitor.cs +++ b/windows/WinUI/NetworkMonitor.cs @@ -156,7 +156,7 @@ namespace WinUI { Console.WriteLine("Monitor Thread Exception: " + "\n" + e.StackTrace); } - Console.WriteLine("Monitor Thread Ended"); + Console.WriteLine("Monitor Thread Ended"); } public void SubscribeStatusUpdates(StatusCallback cb) diff --git a/windows/WinUI/OnboardProcess/CreateAccount.xaml b/windows/WinUI/OnboardProcess/CreateAccount.xaml deleted file mode 100644 index ddf50252..00000000 --- a/windows/WinUI/OnboardProcess/CreateAccount.xaml +++ /dev/null @@ -1,38 +0,0 @@ -<UserControl x:Class="WinUI.OnboardProcess.CreateAccount" - xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" - xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:d="http://schemas.microsoft.com/expression/blend/2008" - xmlns:local="clr-namespace:WinUI.OnboardProcess" - mc:Ignorable="d" - d:DesignHeight="300" d:DesignWidth="300"> - <Grid HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Margin="10"> - <Grid.RowDefinitions> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="*"/> - <RowDefinition Height="Auto"/> - </Grid.RowDefinitions> - <Grid.ColumnDefinitions> - <ColumnDefinition Width="Auto"/> - <ColumnDefinition Width="Auto"/> - </Grid.ColumnDefinitions> - - <Label Grid.Column="0" Grid.Row="0" HorizontalAlignment="Right" Margin="10">Email Address:</Label> - <TextBox x:Name="EmailAddressTextBox" Grid.Column="1" Grid.Row="0" Width="150" Margin="10"></TextBox> - - <Label Grid.Column="0" Grid.Row="1" HorizontalAlignment="Right" Margin="10">Password:</Label> - <PasswordBox x:Name="PasswordTextBox1" Grid.Column="1" Grid.Row="1" PasswordChar="*" Margin="10"/> - <Label Grid.Column="0" Grid.Row="2" HorizontalAlignment="Right" Margin="10">Repeat Password:</Label> - <PasswordBox x:Name="PasswordTextBox2" Grid.Column="1" Grid.Row="2" PasswordChar="*" Margin="10"/> - - <Button Grid.Column="1" Grid.Row="3" Click="CreateAccount_Click" Margin="10" Content="Create Account" Background="#FFFFB354" Width="90" HorizontalAlignment="Right"/> - - <Label x:Name="ErrorText" Grid.Row="4" Grid.ColumnSpan="2" HorizontalAlignment="Center"></Label> - <Label Grid.Row="5"></Label> - <Button Grid.Column="0" Grid.Row="6" Background="#FFFFB354" Click="BackButton_Click">Go Back</Button> - </Grid> -</UserControl> diff --git a/windows/WinUI/OnboardProcess/CreateAccount.xaml.cs b/windows/WinUI/OnboardProcess/CreateAccount.xaml.cs deleted file mode 100644 index 72ba2182..00000000 --- a/windows/WinUI/OnboardProcess/CreateAccount.xaml.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; - -namespace WinUI.OnboardProcess -{ - /// <summary> - /// Interaction logic for CreateAccount.xaml - /// </summary> - public partial class CreateAccount : UserControl, ISwitchable - { - public CreateAccount() - { - InitializeComponent(); - } - - public void UtilizeState(object state) - { - throw new NotImplementedException(); - } - - public void CreateAccount_Click(object sender, RoutedEventArgs e) - { - DoCreateAccount(); - } - - public void BackButton_Click(object sender, RoutedEventArgs e) - { - Switcher.Switch(new RegisterOrLogIn()); - } - - public async void DoCreateAccount() - { - if (PasswordTextBox1.Password.ToString() != PasswordTextBox2.Password.ToString()) - { - ErrorText.Content = "Passwords do not match!"; - } - else - { - CentralAPI api = CentralAPI.Instance; - bool accountCreated = await api.Login(EmailAddressTextBox.Text, - PasswordTextBox1.Password.ToString(), true); - - if (accountCreated) - { - Switcher.Switch(new CreateOrJoin()); - } - else - { - ErrorText.Content = "An error ocurred while creating your account."; - } - } - } - } -} diff --git a/windows/WinUI/OnboardProcess/CreateOrJoin.xaml b/windows/WinUI/OnboardProcess/CreateOrJoin.xaml deleted file mode 100644 index 21413e84..00000000 --- a/windows/WinUI/OnboardProcess/CreateOrJoin.xaml +++ /dev/null @@ -1,49 +0,0 @@ -<UserControl x:Class="WinUI.OnboardProcess.CreateOrJoin" - xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" - xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:d="http://schemas.microsoft.com/expression/blend/2008" - xmlns:local="clr-namespace:WinUI.OnboardProcess" - mc:Ignorable="d" - d:DesignHeight="300" d:DesignWidth="400"> - <Grid HorizontalAlignment="Stretch" Margin="15"> - <Grid.RowDefinitions> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="*"/> - <RowDefinition Height="Auto"/> - </Grid.RowDefinitions> - <Grid.ColumnDefinitions> - <ColumnDefinition Width="*"/> - <ColumnDefinition Width="Auto"/> - <ColumnDefinition Width="*"/> - </Grid.ColumnDefinitions> - - <Button Grid.Column="1" x:Name="CreateButton" Content="Create a Network" Background="#FFFFB354" Click="OnCreateButtonClick"/> - - <Label Grid.Column="1" Grid.Row="1" Content="Or" HorizontalAlignment="Center"/> - - <Label Grid.Column="1" Grid.Row="2" Content="Join a Network:" HorizontalAlignment="Center"/> - - <ListBox Grid.ColumnSpan="3" Grid.Column="0" Grid.Row="3" Name="listViewDataBinding" HorizontalContentAlignment="Stretch"> - <ListBox.ItemTemplate> - <DataTemplate> - <Grid HorizontalAlignment="Stretch"> - <Grid.ColumnDefinitions> - <ColumnDefinition Width="Auto"/> - <ColumnDefinition Width="*"/> - <ColumnDefinition Width="Auto"/> - </Grid.ColumnDefinitions> - <Grid.RowDefinitions> - <RowDefinition Height="Auto"/> - </Grid.RowDefinitions> - <TextBlock Grid.Column="0" Grid.Row="0" Margin="5" HorizontalAlignment="Left" Text="{Binding Id}"/> - <TextBlock Grid.Column="1" Grid.Row="0" Margin="5" HorizontalAlignment="Center" Text="{Binding Config.Name}"/> - <Button Grid.Column="2" Grid.Row="0" Margin="5" Content="Join" HorizontalAlignment="Right" Background="#FFFFB354" Tag="{Binding Id}" Click="OnJoinButtonClick"/> - </Grid> - </DataTemplate> - </ListBox.ItemTemplate> - </ListBox> - </Grid> -</UserControl> diff --git a/windows/WinUI/OnboardProcess/CreateOrJoin.xaml.cs b/windows/WinUI/OnboardProcess/CreateOrJoin.xaml.cs deleted file mode 100644 index 91330540..00000000 --- a/windows/WinUI/OnboardProcess/CreateOrJoin.xaml.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; - -namespace WinUI.OnboardProcess -{ - /// <summary> - /// Interaction logic for CreateOrJoin.xaml - /// </summary> - public partial class CreateOrJoin : UserControl, ISwitchable - { - private List<CentralNetwork> networkList = new List<CentralNetwork>(); - - public CreateOrJoin() - { - InitializeComponent(); - listViewDataBinding.ItemsSource = networkList; - - GetAvailableNetworks(); - } - - public void UtilizeState(object state) - { - throw new NotImplementedException(); - } - - private async void GetAvailableNetworks() - { - CentralAPI api = CentralAPI.Instance; - - List<CentralNetwork> networks = await api.GetNetworkList(); - - foreach (CentralNetwork n in networks) - { - networkList.Add(n); - } - - listViewDataBinding.Items.Refresh(); - } - - public void OnJoinButtonClick(object sender, RoutedEventArgs e) - { - Button button = sender as Button; - string networkId = button.Tag as string; - - APIHandler handler = APIHandler.Instance; - - handler.JoinNetwork(this.Dispatcher, networkId); - - AuthorizeNetworkMember(networkId); - } - - public void OnCreateButtonClick(object sender, RoutedEventArgs e) - { - CreateNewNetwork(); - } - - private async void CreateNewNetwork() - { - CentralAPI api = CentralAPI.Instance; - - CentralNetwork newNetwork = await api.CreateNewNetwork(); - - APIHandler handler = APIHandler.Instance; - - handler.JoinNetwork(this.Dispatcher, newNetwork.Id); - - AuthorizeNetworkMember(newNetwork.Id); - } - - private async void AuthorizeNetworkMember(string networkId) - { - string nodeId = APIHandler.Instance.NodeAddress(); - - bool authorized = await CentralAPI.Instance.AuthorizeNode(nodeId, networkId); - - if (authorized) - { - Switcher.Switch(new Finished()); - } - else - { - - } - } - } -} diff --git a/windows/WinUI/OnboardProcess/EnterToken.xaml b/windows/WinUI/OnboardProcess/EnterToken.xaml deleted file mode 100644 index 1880167d..00000000 --- a/windows/WinUI/OnboardProcess/EnterToken.xaml +++ /dev/null @@ -1,29 +0,0 @@ -<UserControl x:Class="WinUI.OnboardProcess.EnterToken" - xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" - xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:d="http://schemas.microsoft.com/expression/blend/2008" - xmlns:local="clr-namespace:WinUI.OnboardProcess" - mc:Ignorable="d" - d:DesignHeight="300" d:DesignWidth="300"> - <Grid VerticalAlignment="Stretch" HorizontalAlignment="Stretch" Margin="10"> - <Grid.RowDefinitions> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="*"/> - <RowDefinition Height="Auto"/> - </Grid.RowDefinitions> - <Grid.ColumnDefinitions> - <ColumnDefinition Width="Auto"/> - <ColumnDefinition Width="*"/> - </Grid.ColumnDefinitions> - <TextBlock Grid.Column="0" Grid.Row="0" Grid.ColumnSpan="2" Margin="10" TextWrapping="Wrap" HorizontalAlignment="Center" Text="Your API Token can be found or created on https://my.zerotier.com"/> - <TextBlock Grid.Column="0" Grid.Row="1" Text="API Token:" Margin="10"/> - <TextBox x:Name="APITokenInput" Grid.Column="1" Grid.Row="1" Margin="10"/> - - <Button Grid.Column="1" Grid.Row="2" Background="#FFFFB354" Content="Next" HorizontalAlignment="Right" Click="Next_Click" Margin="10"/> - <Label Grid.Row="3"/> - <Button Grid.Column="0" Grid.Row="4" Background="#FFFFB354" Content="Go Back" Click="BackButton_Click" /> - </Grid> -</UserControl> diff --git a/windows/WinUI/OnboardProcess/EnterToken.xaml.cs b/windows/WinUI/OnboardProcess/EnterToken.xaml.cs deleted file mode 100644 index 5d2cc515..00000000 --- a/windows/WinUI/OnboardProcess/EnterToken.xaml.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; - -namespace WinUI.OnboardProcess -{ - /// <summary> - /// Interaction logic for EnterToken.xaml - /// </summary> - public partial class EnterToken : UserControl, ISwitchable - { - public EnterToken() - { - InitializeComponent(); - - if (!string.IsNullOrEmpty(CentralAPI.Instance.Central.APIKey)) - { - APITokenInput.Text = CentralAPI.Instance.Central.APIKey; - } - } - - public void UtilizeState(object staqte) - { - - } - - private void Next_Click(object sender, RoutedEventArgs e) - { - CentralAPI api = CentralAPI.Instance; - - if (api.Central.APIKey != APITokenInput.Text) - { - CentralServer server = new CentralServer(); - server.APIKey = APITokenInput.Text; - api.Central = server; - } - - Switcher.Switch(new CreateOrJoin()); - } - - private void BackButton_Click(object sender, RoutedEventArgs e) - { - Switcher.Switch(new RegisterOrLogIn()); - } - } -} diff --git a/windows/WinUI/OnboardProcess/Finished.xaml b/windows/WinUI/OnboardProcess/Finished.xaml deleted file mode 100644 index d29f536c..00000000 --- a/windows/WinUI/OnboardProcess/Finished.xaml +++ /dev/null @@ -1,27 +0,0 @@ -<UserControl x:Class="WinUI.OnboardProcess.Finished" - xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" - xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:d="http://schemas.microsoft.com/expression/blend/2008" - xmlns:local="clr-namespace:WinUI.OnboardProcess" - mc:Ignorable="d" - d:DesignHeight="300" d:DesignWidth="300"> - <Grid VerticalAlignment="Center" HorizontalAlignment="Stretch" Margin="10"> - <Grid.RowDefinitions> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - </Grid.RowDefinitions> - <Grid.ColumnDefinitions> - <ColumnDefinition Width="Auto" /> - </Grid.ColumnDefinitions> - - <TextBlock Grid.Column="0" Grid.Row="0" Text="All Finished!" HorizontalAlignment="Center"/> - <TextBlock Grid.Column="0" Grid.Row="1" HorizontalAlignment="Center" Text=""/> - <TextBlock Grid.Column="0" Grid.ColumnSpan="1" Grid.Row="2" MaxWidth="270" HorizontalAlignment="Center" Margin="10" Text="You've now joined your first ZeroTier network. Now go to add your other machines!" TextWrapping="Wrap"/> - <TextBlock Grid.Column="0" Grid.Row="3" HorizontalAlignment="Center" Text=""/> - <Button Grid.Column="0" Grid.Row="4" MaxWidth="100" Click="DoneButton_Click" Background="#FFFFB354" Margin="10">Done!</Button> - </Grid> -</UserControl> diff --git a/windows/WinUI/OnboardProcess/Finished.xaml.cs b/windows/WinUI/OnboardProcess/Finished.xaml.cs deleted file mode 100644 index a34abfe2..00000000 --- a/windows/WinUI/OnboardProcess/Finished.xaml.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; - -namespace WinUI.OnboardProcess -{ - /// <summary> - /// Interaction logic for Finished.xaml - /// </summary> - public partial class Finished : UserControl, ISwitchable - { - public Finished() - { - InitializeComponent(); - } - public void UtilizeState(object state) - { - throw new NotImplementedException(); - } - - private void DoneButton_Click(object sender, RoutedEventArgs e) - { - Window.GetWindow(this).Close(); - } - } -} diff --git a/windows/WinUI/OnboardProcess/LogIn.xaml b/windows/WinUI/OnboardProcess/LogIn.xaml deleted file mode 100644 index 501f7e0f..00000000 --- a/windows/WinUI/OnboardProcess/LogIn.xaml +++ /dev/null @@ -1,35 +0,0 @@ -<UserControl x:Class="WinUI.OnboardProcess.LogIn" - xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" - xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:d="http://schemas.microsoft.com/expression/blend/2008" - xmlns:local="clr-namespace:WinUI.OnboardProcess" - mc:Ignorable="d" - d:DesignHeight="300" d:DesignWidth="300"> - <Grid HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Margin="10"> - <Grid.RowDefinitions> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="Auto"/> - <RowDefinition Height="*" /> - <RowDefinition Height="Auto"/> - </Grid.RowDefinitions> - <Grid.ColumnDefinitions> - <ColumnDefinition Width="Auto"/> - <ColumnDefinition Width="Auto"/> - </Grid.ColumnDefinitions> - - <Label Grid.Column="0" Grid.Row="0" HorizontalAlignment="Right" Margin="10">Email Address:</Label> - <TextBox x:Name="EmailAddressTextBox" Grid.Column="1" Grid.Row="0" MinWidth="150" Margin="10"></TextBox> - - <Label Grid.Column="0" Grid.Row="1" HorizontalAlignment="Right" Margin="10">Password:</Label> - <PasswordBox x:Name="PasswordTextBox" Grid.Column="1" Grid.Row="1" PasswordChar="*" Margin="10"/> - - <Button Grid.Column="1" Grid.Row="2" Click="LoginButton_Click" Content="LogIn" Background="#FFFFB354" Width="90" Margin="10" HorizontalAlignment="Right"/> - - <Label x:Name="ErrorText" Grid.Row="3" Grid.ColumnSpan="2" HorizontalAlignment="Center"></Label> - <Label Grid.Row="4"></Label> - <Button Grid.Column="0" Grid.Row="5" Background="#FFFFB354" Click="BackButton_Click">Go Back</Button> - </Grid> -</UserControl> diff --git a/windows/WinUI/OnboardProcess/LogIn.xaml.cs b/windows/WinUI/OnboardProcess/LogIn.xaml.cs deleted file mode 100644 index 8657b800..00000000 --- a/windows/WinUI/OnboardProcess/LogIn.xaml.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; - -namespace WinUI.OnboardProcess -{ - /// <summary> - /// Interaction logic for LogIn.xaml - /// </summary> - public partial class LogIn : UserControl, ISwitchable - { - public LogIn() - { - InitializeComponent(); - } - - public void UtilizeState(object state) - { - throw new NotImplementedException(); - } - - public void LoginButton_Click(object sender, RoutedEventArgs e) - { - DoLogin(); - } - - public void BackButton_Click(object sender, RoutedEventArgs e) - { - Switcher.Switch(new RegisterOrLogIn()); - } - - private async void DoLogin() - { - CentralAPI api = CentralAPI.Instance; - bool didLogIn = await api.Login(EmailAddressTextBox.Text, PasswordTextBox.Password.ToString(), false); - if (didLogIn) - { - Switcher.Switch(new CreateOrJoin()); - } - else - { - ErrorText.Content = "Invalid username or password"; - } - } - } -} diff --git a/windows/WinUI/OnboardProcess/RegisterOrLogIn.xaml b/windows/WinUI/OnboardProcess/RegisterOrLogIn.xaml deleted file mode 100644 index 01f3ba9d..00000000 --- a/windows/WinUI/OnboardProcess/RegisterOrLogIn.xaml +++ /dev/null @@ -1,27 +0,0 @@ -<UserControl x:Class="WinUI.OnboardProcess.RegisterOrLogIn" - xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" - xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:d="http://schemas.microsoft.com/expression/blend/2008" - xmlns:local="clr-namespace:WinUI.OnboardProcess" - mc:Ignorable="d" - d:DesignHeight="300" d:DesignWidth="300"> - <StackPanel> - <TextBlock HorizontalAlignment="Center" FontSize="20" FontWeight="Bold" FontFamily="Segoe UI">Welcome to ZeroTier</TextBlock> - <TextBlock HorizontalAlignment="Center" FontSize="16">Let's get started!</TextBlock> - <TextBlock HorizontalAlignment="Center"> </TextBlock> - <TextBlock HorizontalAlignment="Center" TextWrapping="Wrap">If you haven't yet created an account, click "Create Account" below. If you have an account, you can log in with your username/password, or simply enter an API key.</TextBlock> - <StackPanel Orientation="Horizontal" HorizontalAlignment="Center"> - <StackPanel.Resources> - <Style TargetType="{x:Type Button}"> - <Setter Property="Margin" Value="10,10,0,0"/> - </Style> - </StackPanel.Resources> - <Button x:Name="CreateAccountButton" Margin="10,5" Content="Create Account" Background="#FFFFB354" Click="CreateAccountButton_Click"/> - - <Button x:Name="LogInButton" Margin="10,5" Content="Log In" Background="#FFFFB354" Click="LogInButton_Click"/> - - <Button x:Name="TokenButton" Margin="10,5" Content="API Token" Background="#FFFFB354" Click="APIToken_Click"/> - </StackPanel> - </StackPanel> -</UserControl> diff --git a/windows/WinUI/OnboardProcess/RegisterOrLogIn.xaml.cs b/windows/WinUI/OnboardProcess/RegisterOrLogIn.xaml.cs deleted file mode 100644 index 19c578dd..00000000 --- a/windows/WinUI/OnboardProcess/RegisterOrLogIn.xaml.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; - -namespace WinUI.OnboardProcess -{ - /// <summary> - /// Interaction logic for RegisterOrLogIn.xaml - /// </summary> - public partial class RegisterOrLogIn : UserControl, ISwitchable - { - public RegisterOrLogIn() - { - InitializeComponent(); - } - - public void UtilizeState(object state) - { - throw new NotImplementedException(); - } - - public void CreateAccountButton_Click(object sender, System.Windows.RoutedEventArgs e) - { - Switcher.Switch(new CreateAccount()); - } - - private void LogInButton_Click(object sender, RoutedEventArgs e) - { - Switcher.Switch(new LogIn()); - } - - public void APIToken_Click(object sender, RoutedEventArgs e) - { - Switcher.Switch(new EnterToken()); - } - } -} diff --git a/windows/WinUI/PageSwitcher.xaml b/windows/WinUI/PageSwitcher.xaml deleted file mode 100644 index 2e463e7a..00000000 --- a/windows/WinUI/PageSwitcher.xaml +++ /dev/null @@ -1,13 +0,0 @@ -<Window x:Class="WinUI.PageSwitcher" - xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" - xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:d="http://schemas.microsoft.com/expression/blend/2008" - xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:local="clr-namespace:WinUI" - mc:Ignorable="d" - Icon="ZeroTierIcon.ico" - Title="ZeroTier One" Height="300" Width="400"> - <Grid> - - </Grid> -</Window> diff --git a/windows/WinUI/PageSwitcher.xaml.cs b/windows/WinUI/PageSwitcher.xaml.cs deleted file mode 100644 index c9aa659f..00000000 --- a/windows/WinUI/PageSwitcher.xaml.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Shapes; - -namespace WinUI -{ - /// <summary> - /// Interaction logic for PageSwitcher.xaml - /// </summary> - public partial class PageSwitcher : Window - { - public PageSwitcher() - { - InitializeComponent(); - Switcher.pageSwitcher = this; - - CentralAPI api = CentralAPI.Instance; - - if (api.HasAccessToken()) - { - Switcher.Switch(new OnboardProcess.CreateOrJoin()); - } - else - { - Switcher.Switch(new OnboardProcess.RegisterOrLogIn()); - } - } - - public void Navigate(UserControl nextPage) - { - this.Content = nextPage; - } - - public void Navigate(UserControl nextPage, object state) - { - this.Content = nextPage; - ISwitchable s = nextPage as ISwitchable; - - if (s != null) - s.UtilizeState(state); - else - throw new ArgumentException("NextPage is not ISwitchable! " - + nextPage.Name.ToString()); - } - } -} diff --git a/windows/WinUI/Switcher.cs b/windows/WinUI/Switcher.cs deleted file mode 100644 index f48835df..00000000 --- a/windows/WinUI/Switcher.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Windows.Controls; -using System.Threading.Tasks; - -namespace WinUI -{ - public static class Switcher - { - public static PageSwitcher pageSwitcher; - - public static void Switch(UserControl newPage) - { - pageSwitcher.Navigate(newPage); - } - - public static void Switch(UserControl newPage, object state) - { - pageSwitcher.Navigate(newPage, state); - } - } -} diff --git a/windows/WinUI/ToolbarItem.xaml.cs b/windows/WinUI/ToolbarItem.xaml.cs index 2a0d717b..e38f9802 100644 --- a/windows/WinUI/ToolbarItem.xaml.cs +++ b/windows/WinUI/ToolbarItem.xaml.cs @@ -45,10 +45,6 @@ namespace WinUI private ObservableCollection<MenuItem> _networkCollection = new ObservableCollection<MenuItem>(); - private static Boolean shouldShowOnboardProcess = true; -#if DEBUG - private static bool isFirstRun = true; -#endif public ObservableCollection<MenuItem> NetworkCollection { @@ -85,23 +81,6 @@ namespace WinUI { if (networks != null) { - if (networks.Count > 0) - { -#if DEBUG - if (isFirstRun) - { - shouldShowOnboardProcess = true; - isFirstRun = false; - } - else - { - shouldShowOnboardProcess = false; - } -#else - shouldShowOnboardProcess = false; -#endif - } - Dispatcher.BeginInvoke(DispatcherPriority.Normal, new Action(() => { NetworkCollection.Clear(); @@ -116,25 +95,9 @@ namespace WinUI NetworkCollection.Add(item); } })); - - if (shouldShowOnboardProcess) - { - // TODO: Show onboarding process window (on main thread - showOnboardProcess(); - - shouldShowOnboardProcess = false; - } } } - private void showOnboardProcess() - { - Dispatcher.BeginInvoke(DispatcherPriority.Normal, new Action(() => - { - PageSwitcher ps = new PageSwitcher(); - ps.Show(); - })); - } private void updateStatus(ZeroTierStatus status) { if (status != null) @@ -144,16 +107,16 @@ namespace WinUI nodeIdMenuItem.Header = "Node ID: " + status.Address; nodeIdMenuItem.IsEnabled = true; nodeId = status.Address; - })); - } - if (CentralAPI.Instance.HasAccessToken()) - { - newNetworkItem.IsEnabled = true; - } - else - { - newNetworkItem.IsEnabled = false; + if (CentralAPI.Instance.HasAccessToken()) + { + newNetworkItem.IsEnabled = true; + } + else + { + newNetworkItem.IsEnabled = false; + } + })); } } @@ -356,10 +319,6 @@ namespace WinUI string nodeId = APIHandler.Instance.NodeAddress(); bool authorized = await CentralAPI.Instance.AuthorizeNode(nodeId, newNetwork.Id); - } - else - { - showOnboardProcess(); } } diff --git a/windows/WinUI/WinUI.csproj b/windows/WinUI/WinUI.csproj index 6bbc1cd7..7a2a1e2b 100644 --- a/windows/WinUI/WinUI.csproj +++ b/windows/WinUI/WinUI.csproj @@ -121,34 +121,12 @@ <Compile Include="NetworksPage.xaml.cs"> <DependentUpon>NetworksPage.xaml</DependentUpon> </Compile> - <Compile Include="OnboardProcess\CreateAccount.xaml.cs"> - <DependentUpon>CreateAccount.xaml</DependentUpon> - </Compile> - <Compile Include="OnboardProcess\CreateOrJoin.xaml.cs"> - <DependentUpon>CreateOrJoin.xaml</DependentUpon> - </Compile> - <Compile Include="OnboardProcess\EnterToken.xaml.cs"> - <DependentUpon>EnterToken.xaml</DependentUpon> - </Compile> - <Compile Include="OnboardProcess\Finished.xaml.cs"> - <DependentUpon>Finished.xaml</DependentUpon> - </Compile> - <Compile Include="OnboardProcess\LogIn.xaml.cs"> - <DependentUpon>LogIn.xaml</DependentUpon> - </Compile> - <Compile Include="OnboardProcess\RegisterOrLogIn.xaml.cs"> - <DependentUpon>RegisterOrLogIn.xaml</DependentUpon> - </Compile> - <Compile Include="PageSwitcher.xaml.cs"> - <DependentUpon>PageSwitcher.xaml</DependentUpon> - </Compile> <Compile Include="PeersPage.xaml.cs"> <DependentUpon>PeersPage.xaml</DependentUpon> </Compile> <Compile Include="PreferencesView.xaml.cs"> <DependentUpon>PreferencesView.xaml</DependentUpon> </Compile> - <Compile Include="Switcher.cs" /> <Compile Include="ToolbarItem.xaml.cs"> <DependentUpon>ToolbarItem.xaml</DependentUpon> </Compile> @@ -185,34 +163,6 @@ <SubType>Designer</SubType> <Generator>MSBuild:Compile</Generator> </Page> - <Page Include="OnboardProcess\CreateAccount.xaml"> - <SubType>Designer</SubType> - <Generator>MSBuild:Compile</Generator> - </Page> - <Page Include="OnboardProcess\CreateOrJoin.xaml"> - <SubType>Designer</SubType> - <Generator>MSBuild:Compile</Generator> - </Page> - <Page Include="OnboardProcess\EnterToken.xaml"> - <SubType>Designer</SubType> - <Generator>MSBuild:Compile</Generator> - </Page> - <Page Include="OnboardProcess\Finished.xaml"> - <SubType>Designer</SubType> - <Generator>MSBuild:Compile</Generator> - </Page> - <Page Include="OnboardProcess\LogIn.xaml"> - <SubType>Designer</SubType> - <Generator>MSBuild:Compile</Generator> - </Page> - <Page Include="OnboardProcess\RegisterOrLogIn.xaml"> - <SubType>Designer</SubType> - <Generator>MSBuild:Compile</Generator> - </Page> - <Page Include="PageSwitcher.xaml"> - <SubType>Designer</SubType> - <Generator>MSBuild:Compile</Generator> - </Page> <Page Include="PeersPage.xaml"> <SubType>Designer</SubType> <Generator>MSBuild:Compile</Generator> diff --git a/windows/ZeroTierOne.sln b/windows/ZeroTierOne.sln index 05f498fb..76d579a8 100644 --- a/windows/ZeroTierOne.sln +++ b/windows/ZeroTierOne.sln @@ -1,7 +1,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.25420.1 +# Visual Studio 15 +VisualStudioVersion = 15.0.28010.2050 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZeroTierOne", "ZeroTierOne\ZeroTierOne.vcxproj", "{B00A4957-5977-4AC1-9EF4-571DC27EADA2}" EndProject @@ -460,4 +460,7 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {AD00A172-C020-4874-B00A-C396AAC893FC} + EndGlobalSection EndGlobal diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 29f08874..3f6397ca 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
@@ -30,7 +30,8 @@ <ClCompile Include="..\..\controller\DB.cpp" />
<ClCompile Include="..\..\controller\EmbeddedNetworkController.cpp" />
<ClCompile Include="..\..\controller\FileDB.cpp" />
- <ClCompile Include="..\..\controller\RethinkDB.cpp" />
+ <ClCompile Include="..\..\controller\PostgreSQL.cpp" />
+ <ClCompile Include="..\..\controller\RabbitMQ.cpp" />
<ClCompile Include="..\..\ext\http-parser\http_parser.c" />
<ClCompile Include="..\..\ext\libnatpmp\getgateway.c" />
<ClCompile Include="..\..\ext\libnatpmp\natpmp.c" />
@@ -115,7 +116,8 @@ <ClInclude Include="..\..\controller\DB.hpp" />
<ClInclude Include="..\..\controller\EmbeddedNetworkController.hpp" />
<ClInclude Include="..\..\controller\FileDB.hpp" />
- <ClInclude Include="..\..\controller\RethinkDB.hpp" />
+ <ClInclude Include="..\..\controller\PostgreSQL.hpp" />
+ <ClInclude Include="..\..\controller\RabbitMQ.hpp" />
<ClInclude Include="..\..\ext\http-parser\http_parser.h" />
<ClInclude Include="..\..\ext\json\json.hpp" />
<ClInclude Include="..\..\ext\libnatpmp\getgateway.h" />
@@ -153,6 +155,7 @@ <ClInclude Include="..\..\node\IncomingPacket.hpp" />
<ClInclude Include="..\..\node\InetAddress.hpp" />
<ClInclude Include="..\..\node\MAC.hpp" />
+ <ClInclude Include="..\..\node\Membership.hpp" />
<ClInclude Include="..\..\node\Multicaster.hpp" />
<ClInclude Include="..\..\node\MulticastGroup.hpp" />
<ClInclude Include="..\..\node\Mutex.hpp" />
@@ -197,45 +200,48 @@ <PropertyGroup Label="Globals">
<ProjectGuid>{B00A4957-5977-4AC1-9EF4-571DC27EADA2}</ProjectGuid>
<RootNamespace>ZeroTierOne</RootNamespace>
+ <WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v140</PlatformToolset>
+ <PlatformToolset>v141</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Profile|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v140</PlatformToolset>
+ <PlatformToolset>v141</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v140</PlatformToolset>
+ <PlatformToolset>v141</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Profile|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v140</PlatformToolset>
+ <PlatformToolset>v141</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v140</PlatformToolset>
+ <PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
+ <UseOfMfc>Static</UseOfMfc>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v140</PlatformToolset>
+ <PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
+ <UseOfMfc>Static</UseOfMfc>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@@ -298,6 +304,9 @@ </AdditionalIncludeDirectories>
<PreprocessorDefinitions>ZT_EXPORT;FD_SETSIZE=1024;NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions)</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
+ <RuntimeTypeInfo>false</RuntimeTypeInfo>
+ <LanguageStandard>stdcpp14</LanguageStandard>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -314,6 +323,8 @@ </AdditionalIncludeDirectories>
<PreprocessorDefinitions>ZT_EXPORT;FD_SETSIZE=1024;NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions)</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
+ <RuntimeTypeInfo>false</RuntimeTypeInfo>
+ <LanguageStandard>stdcpp14</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -329,8 +340,10 @@ <AdditionalIncludeDirectories>
</AdditionalIncludeDirectories>
<PreprocessorDefinitions>ZT_EXPORT;FD_SETSIZE=1024;NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_RULES_ENGINE_DEBUGGING;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <MultiProcessorCompilation>false</MultiProcessorCompilation>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
+ <RuntimeTypeInfo>false</RuntimeTypeInfo>
+ <LanguageStandard>stdcpp14</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -349,6 +362,8 @@ <PreprocessorDefinitions>ZT_EXPORT;FD_SETSIZE=1024;NOMINMAX;STATICLIB;WIN32;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MultiProcessorCompilation>false</MultiProcessorCompilation>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
+ <RuntimeTypeInfo>false</RuntimeTypeInfo>
+ <LanguageStandard>stdcpp14</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -359,7 +374,7 @@ </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
- <WarningLevel>Level3</WarningLevel>
+ <WarningLevel>Level2</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
@@ -375,9 +390,15 @@ <OmitFramePointers>true</OmitFramePointers>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
<ControlFlowGuard>Guard</ControlFlowGuard>
+ <RuntimeTypeInfo>false</RuntimeTypeInfo>
+ <LanguageStandard>stdcpp14</LanguageStandard>
+ <DebugInformationFormat>None</DebugInformationFormat>
+ <CompileAsManaged>false</CompileAsManaged>
+ <CompileAsWinRT>false</CompileAsWinRT>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
+ <GenerateDebugInformation>false</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies>wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;%(AdditionalDependencies)</AdditionalDependencies>
@@ -386,7 +407,7 @@ </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
- <WarningLevel>Level3</WarningLevel>
+ <WarningLevel>Level2</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
@@ -404,9 +425,15 @@ <ControlFlowGuard>Guard</ControlFlowGuard>
<EnableParallelCodeGeneration>false</EnableParallelCodeGeneration>
<CallingConvention>VectorCall</CallingConvention>
+ <RuntimeTypeInfo>false</RuntimeTypeInfo>
+ <LanguageStandard>stdcpp14</LanguageStandard>
+ <DebugInformationFormat>None</DebugInformationFormat>
+ <CompileAsManaged>false</CompileAsManaged>
+ <CompileAsWinRT>false</CompileAsWinRT>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
</ClCompile>
<Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
+ <GenerateDebugInformation>false</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies>wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;%(AdditionalDependencies)</AdditionalDependencies>
diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index d07c0638..1fcf7b69 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -258,7 +258,10 @@ <ClCompile Include="..\..\controller\FileDB.cpp">
<Filter>Source Files\controller</Filter>
</ClCompile>
- <ClCompile Include="..\..\controller\RethinkDB.cpp">
+ <ClCompile Include="..\..\controller\PostgreSQL.cpp">
+ <Filter>Source Files\controller</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\controller\RabbitMQ.cpp">
<Filter>Source Files\controller</Filter>
</ClCompile>
</ItemGroup>
@@ -494,7 +497,13 @@ <ClInclude Include="..\..\controller\FileDB.hpp">
<Filter>Header Files\controller</Filter>
</ClInclude>
- <ClInclude Include="..\..\controller\RethinkDB.hpp">
+ <ClInclude Include="..\..\node\Membership.hpp">
+ <Filter>Header Files\node</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\controller\PostgreSQL.hpp">
+ <Filter>Header Files\controller</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\controller\RabbitMQ.hpp">
<Filter>Header Files\controller</Filter>
</ClInclude>
</ItemGroup>
diff --git a/zerotier-one.spec b/zerotier-one.spec index 55cc1fb0..72a5a96a 100644 --- a/zerotier-one.spec +++ b/zerotier-one.spec @@ -1,5 +1,5 @@ Name: zerotier-one -Version: 1.2.12 +Version: 1.2.99 Release: 1%{?dist} Summary: ZeroTier One network virtualization service |
