From 22e44c762bf77aefe988ed7b6874054f84f95b75 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 28 Jul 2016 10:58:10 -0700 Subject: More rules engine work: key/value pair matching for microsegmentation. --- controller/SqliteNetworkController.cpp | 40 +++++++++++++++++++++++++++++----- controller/SqliteNetworkController.hpp | 1 - controller/schema.sql | 27 ++++++++--------------- controller/schema.sql.c | 27 ++++++++--------------- 4 files changed, 53 insertions(+), 42 deletions(-) (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 65051744..81017897 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -66,8 +66,8 @@ // Stored in database as schemaVersion key in Config. // If not present, database is assumed to be empty and at the current schema version // and this key/value is added automatically. -#define ZT_NETCONF_SQLITE_SCHEMA_VERSION 4 -#define ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR "4" +#define ZT_NETCONF_SQLITE_SCHEMA_VERSION 5 +#define ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR "5" // API version reported via JSON control plane #define ZT_NETCONF_CONTROLLER_API_VERSION 2 @@ -334,6 +334,38 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c } } + if (schemaVersion < 5) { + // Upgrade old rough draft Rule table to new release format + if (sqlite3_exec(_db, + "DROP INDEX Rule_networkId_ruleNo;\n" + "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" + "CREATE TABLE Rule (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " policyId varchar(32),\n" + " ruleNo integer NOT NULL,\n" + " ruleType integer NOT NULL DEFAULT(0),\n" + " \"addr\" blob(16),\n" + " \"int1\" integer,\n" + " \"int2\" integer,\n" + " \"int3\" integer,\n" + " \"int4\" integer\n" + ");\n" + "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" + "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" + "DROP TABLE RuleOld;\n" + "CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n" + "CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId);\n" + "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 5; + } + } + if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { sqlite3_close(_db); throw std::runtime_error("SqliteNetworkController database schema version mismatch"); @@ -365,8 +397,7 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) /* Rule */ - ||(sqlite3_prepare_v2(_db,"SELECT etherType FROM Rule WHERE networkId = ? AND \"action\" = 'accept'",-1,&_sGetEtherTypesFromRuleTable,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcP,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,flags,invFlags,\"action\") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) @@ -457,7 +488,6 @@ SqliteNetworkController::~SqliteNetworkController() sqlite3_finalize(_sCreateMember); sqlite3_finalize(_sGetNodeIdentity); sqlite3_finalize(_sCreateOrReplaceNode); - sqlite3_finalize(_sGetEtherTypesFromRuleTable); sqlite3_finalize(_sGetActiveBridges); sqlite3_finalize(_sGetIpAssignmentsForNode); sqlite3_finalize(_sGetIpAssignmentPools); diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp index 145788c7..b917f6fb 100644 --- a/controller/SqliteNetworkController.hpp +++ b/controller/SqliteNetworkController.hpp @@ -137,7 +137,6 @@ private: sqlite3_stmt *_sCreateMember; sqlite3_stmt *_sGetNodeIdentity; sqlite3_stmt *_sCreateOrReplaceNode; - sqlite3_stmt *_sGetEtherTypesFromRuleTable; sqlite3_stmt *_sGetActiveBridges; sqlite3_stmt *_sGetIpAssignmentsForNode; sqlite3_stmt *_sGetIpAssignmentPools; diff --git a/controller/schema.sql b/controller/schema.sql index 105db924..479daa68 100644 --- a/controller/schema.sql +++ b/controller/schema.sql @@ -96,24 +96,15 @@ CREATE UNIQUE INDEX Relay_networkId_address ON Relay (networkId,address); CREATE TABLE Rule ( networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + policyId varchar(32), ruleNo integer NOT NULL, - nodeId char(10) REFERENCES Node(id), - sourcePort char(10), - destPort char(10), - vlanId integer, - vlanPcp integer, - etherType integer, - macSource char(12), - macDest char(12), - ipSource varchar(64), - ipDest varchar(64), - ipTos integer, - ipProtocol integer, - ipSourcePort integer, - ipDestPort integer, - flags integer, - invFlags integer, - "action" varchar(4096) NOT NULL DEFAULT('accept') + ruleType integer NOT NULL DEFAULT(0), + "addr" blob(16), + "int1" integer, + "int2" integer, + "int3" integer, + "int4" integer ); -CREATE UNIQUE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo); +CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo); +CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId); diff --git a/controller/schema.sql.c b/controller/schema.sql.c index dab34138..e84ee766 100644 --- a/controller/schema.sql.c +++ b/controller/schema.sql.c @@ -97,25 +97,16 @@ "\n"\ "CREATE TABLE Rule (\n"\ " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ +" policyId varchar(32),\n"\ " ruleNo integer NOT NULL,\n"\ -" nodeId char(10) REFERENCES Node(id),\n"\ -" sourcePort char(10),\n"\ -" destPort char(10),\n"\ -" vlanId integer,\n"\ -" vlanPcp integer,\n"\ -" etherType integer,\n"\ -" macSource char(12),\n"\ -" macDest char(12),\n"\ -" ipSource varchar(64),\n"\ -" ipDest varchar(64),\n"\ -" ipTos integer,\n"\ -" ipProtocol integer,\n"\ -" ipSourcePort integer,\n"\ -" ipDestPort integer,\n"\ -" flags integer,\n"\ -" invFlags integer,\n"\ -" \"action\" varchar(4096) NOT NULL DEFAULT('accept')\n"\ +" ruleType integer NOT NULL DEFAULT(0),\n"\ +" \"addr\" blob(16),\n"\ +" \"int1\" integer,\n"\ +" \"int2\" integer,\n"\ +" \"int3\" integer,\n"\ +" \"int4\" integer\n"\ ");\n"\ "\n"\ -"CREATE UNIQUE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n"\ +"CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n"\ +"CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId);\n"\ "" -- cgit v1.2.3 From c30f74987ff9027e3f7be3a9f400134c17d555e6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 12 Aug 2016 11:30:27 -0700 Subject: Starting refactor of controller... --- controller/SqliteNetworkController.cpp | 672 +- controller/SqliteNetworkController.hpp | 74 +- controller/schema.sql | 24 +- ext/json/LICENSE.MIT | 22 - ext/json/README.md | 511 -- ext/json/json.hpp | 10435 ------------------------------- ext/offbase/README.md | 59 + ext/offbase/json/LICENSE.MIT | 22 + ext/offbase/json/README.md | 511 ++ ext/offbase/json/json.hpp | 10435 +++++++++++++++++++++++++++++++ ext/offbase/offbase.hpp | 393 ++ 11 files changed, 11552 insertions(+), 11606 deletions(-) delete mode 100644 ext/json/LICENSE.MIT delete mode 100644 ext/json/README.md delete mode 100644 ext/json/json.hpp create mode 100644 ext/offbase/README.md create mode 100644 ext/offbase/json/LICENSE.MIT create mode 100644 ext/offbase/json/README.md create mode 100644 ext/offbase/json/json.hpp create mode 100644 ext/offbase/offbase.hpp (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 81017897..56bddbc3 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -37,16 +37,11 @@ #include #include #include +#include #include "../include/ZeroTierOne.h" #include "../node/Constants.hpp" -#ifdef ZT_USE_SYSTEM_JSON_PARSER -#include -#else -#include "../ext/json-parser/json.h" -#endif - #include "SqliteNetworkController.hpp" #include "../node/Node.hpp" @@ -60,17 +55,17 @@ #include "../osdep/OSUtils.hpp" -// Include ZT_NETCONF_SCHEMA_SQL constant to init database -#include "schema.sql.c" +// offbase includes and builds upon nlohmann::json +using json = nlohmann::json; // Stored in database as schemaVersion key in Config. // If not present, database is assumed to be empty and at the current schema version // and this key/value is added automatically. -#define ZT_NETCONF_SQLITE_SCHEMA_VERSION 5 -#define ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR "5" +//#define ZT_NETCONF_SQLITE_SCHEMA_VERSION 5 +//#define ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR "5" // API version reported via JSON control plane -#define ZT_NETCONF_CONTROLLER_API_VERSION 2 +#define ZT_NETCONF_CONTROLLER_API_VERSION 3 // Number of requests to remember in member history #define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 8 @@ -79,155 +74,19 @@ #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 // Delay between backups in milliseconds -#define ZT_NETCONF_BACKUP_PERIOD 300000 +//#define ZT_NETCONF_BACKUP_PERIOD 300000 // Nodes are considered active if they've queried in less than this long #define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) -// Flags for Network 'flags' field in table -#define ZT_DB_NETWORK_FLAG_ZT_MANAGED_V4_AUTO_ASSIGN 1 -#define ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_RFC4193 2 -#define ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_6PLANE 4 -#define ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_AUTO_ASSIGN 8 - -// Flags with all V6 managed mode flags flipped off -- for masking in update operation and in string form for SQL building -#define ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_MASK_S "268435441" - -// Uncomment to trace Sqlite for debugging -//#define ZT_NETCONF_SQLITE_TRACE 1 - namespace ZeroTier { -namespace { - -static std::string _jsonEscape(const char *s) -{ - if (!s) - return std::string(); - std::string buf; - for(const char *p=s;(*p);++p) { - switch(*p) { - case '\t': buf.append("\\t"); break; - case '\b': buf.append("\\b"); break; - case '\r': buf.append("\\r"); break; - case '\n': buf.append("\\n"); break; - case '\f': buf.append("\\f"); break; - case '"': buf.append("\\\""); break; - case '\\': buf.append("\\\\"); break; - case '/': buf.append("\\/"); break; - default: buf.push_back(*p); break; - } - } - return buf; -} -static std::string _jsonEscape(const std::string &s) { return _jsonEscape(s.c_str()); } - -// Converts an InetAddress to a blob and an int for storage in database -static void _ipToBlob(const InetAddress &a,char *ipBlob,int &ipVersion) /* blob[16] */ -{ - switch(a.ss_family) { - case AF_INET: - memset(ipBlob,0,12); - memcpy(ipBlob + 12,a.rawIpData(),4); - ipVersion = 4; - break; - case AF_INET6: - memcpy(ipBlob,a.rawIpData(),16); - ipVersion = 6; - break; - } -} - -// Member.recentHistory is stored in a BLOB as an array of strings containing JSON objects. -// This is kind of hacky but efficient and quick to parse and send to the client. -class MemberRecentHistory : public std::list -{ -public: - inline std::string toBlob() const - { - std::string b; - for(const_iterator i(begin());i!=end();++i) { - b.append(*i); - b.push_back((char)0); - } - return b; - } - - inline void fromBlob(const char *blob,unsigned int len) - { - for(unsigned int i=0,k=0;i 0;\n" "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" "DROP TABLE RuleOld;\n" - "CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n" - "CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId);\n" + "CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n" + "CREATE TABLE MemberTC (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" + " tagId integer,\n" + " tagValue integer,\n" + " capId integer,\n" + " capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n" + ");\n" + "CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n" "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" ,0,0,0) != SQLITE_OK) { char err[1024]; @@ -383,7 +251,6 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c if ( - /* Network */ (sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(const char **)0) != SQLITE_OK) @@ -392,33 +259,23 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c ||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(const char **)0) != SQLITE_OK) - /* Node */ ||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) - /* Rule */ ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) - /* IpAssignmentPool */ ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(const char **)0) != SQLITE_OK) - /* IpAssignment */ ||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(const char **)0) != SQLITE_OK) - /* Relay */ - ||(sqlite3_prepare_v2(_db,"SELECT \"address\",\"phyAddress\" FROM Relay WHERE \"networkId\" = ? ORDER BY \"address\" ASC",-1,&_sGetRelays,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Relay WHERE networkId = ?",-1,&_sDeleteRelaysForNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Relay (\"networkId\",\"address\",\"phyAddress\") VALUES (?,?,?)",-1,&_sCreateRelay,(const char **)0) != SQLITE_OK) - - /* Member */ ||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(const char **)0) != SQLITE_OK) @@ -431,12 +288,10 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(const char **)0) != SQLITE_OK) - /* Route */ ||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(const char **)0) != SQLITE_OK) - /* Config */ ||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(const char **)0) != SQLITE_OK) @@ -446,9 +301,6 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c throw std::runtime_error(err); } - /* Generate a 128-bit / 32-character "instance ID" if one isn't already - * defined. Clients can use this to determine if this is the same controller - * database they know and love. */ sqlite3_reset(_sGetConfig); sqlite3_bind_text(_sGetConfig,1,"instanceId",10,SQLITE_STATIC); if (sqlite3_step(_sGetConfig) != SQLITE_ROW) { @@ -474,72 +326,32 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c #endif _backupThread = Thread::start(this); + */ + + _dbCommitThread = Thread::start(this); } SqliteNetworkController::~SqliteNetworkController() { - _backupThreadRun = false; - Thread::join(_backupThread); - - Mutex::Lock _l(_lock); - if (_db) { - sqlite3_finalize(_sGetNetworkById); - sqlite3_finalize(_sGetMember); - sqlite3_finalize(_sCreateMember); - sqlite3_finalize(_sGetNodeIdentity); - sqlite3_finalize(_sCreateOrReplaceNode); - sqlite3_finalize(_sGetActiveBridges); - sqlite3_finalize(_sGetIpAssignmentsForNode); - sqlite3_finalize(_sGetIpAssignmentPools); - sqlite3_finalize(_sCheckIfIpIsAllocated); - sqlite3_finalize(_sAllocateIp); - sqlite3_finalize(_sDeleteIpAllocations); - sqlite3_finalize(_sGetRelays); - sqlite3_finalize(_sListNetworks); - sqlite3_finalize(_sListNetworkMembers); - sqlite3_finalize(_sGetMember2); - sqlite3_finalize(_sGetIpAssignmentPools2); - sqlite3_finalize(_sListRules); - sqlite3_finalize(_sCreateRule); - sqlite3_finalize(_sCreateNetwork); - sqlite3_finalize(_sGetNetworkRevision); - sqlite3_finalize(_sSetNetworkRevision); - sqlite3_finalize(_sDeleteRelaysForNetwork); - sqlite3_finalize(_sCreateRelay); - sqlite3_finalize(_sDeleteIpAssignmentPoolsForNetwork); - sqlite3_finalize(_sDeleteRulesForNetwork); - sqlite3_finalize(_sCreateIpAssignmentPool); - sqlite3_finalize(_sUpdateMemberAuthorized); - sqlite3_finalize(_sUpdateMemberActiveBridge); - sqlite3_finalize(_sUpdateMemberHistory); - sqlite3_finalize(_sDeleteMember); - sqlite3_finalize(_sDeleteAllNetworkMembers); - sqlite3_finalize(_sGetActiveNodesOnNetwork); - sqlite3_finalize(_sDeleteNetwork); - sqlite3_finalize(_sCreateRoute); - sqlite3_finalize(_sGetRoutes); - sqlite3_finalize(_sDeleteRoutes); - sqlite3_finalize(_sIncrementMemberRevisionCounter); - sqlite3_finalize(_sGetConfig); - sqlite3_finalize(_sSetConfig); - sqlite3_close(_db); - } + _lock.lock(); + _dbCommitThreadRun = false; + _lock.unlock(); + Thread::join(_dbCommitThread); } -NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) +NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) { if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; } - const uint64_t now = OSUtils::now(); - - NetworkRecord network; - Utils::snprintf(network.id,sizeof(network.id),"%.16llx",(unsigned long long)nwid); + char nwids[24],nodeIds[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + Utils::snprintf(nodeIds,sizeof(nodeIds),"%.10llx",(unsigned long long)identity.address().toInt()); - MemberRecord member; - Utils::snprintf(member.nodeId,sizeof(member.nodeId),"%.10llx",(unsigned long long)identity.address().toInt()); + const uint64_t now = OSUtils::now(); + /* { // begin lock Mutex::Lock _l(_lock); @@ -874,13 +686,13 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co sqlite3_bind_text(_sCheckIfIpIsAllocated,1,network.id,16,SQLITE_STATIC); sqlite3_bind_blob(_sCheckIfIpIsAllocated,2,(const void *)ip6.rawIpData(),16,SQLITE_STATIC); sqlite3_bind_int(_sCheckIfIpIsAllocated,3,6); // 6 == IPv6 - sqlite3_bind_int(_sCheckIfIpIsAllocated,4,(int)0 /*ZT_IP_ASSIGNMENT_TYPE_ADDRESS*/); + sqlite3_bind_int(_sCheckIfIpIsAllocated,4,(int)0); if (sqlite3_step(_sCheckIfIpIsAllocated) != SQLITE_ROW) { // No rows returned, so the IP is available sqlite3_reset(_sAllocateIp); sqlite3_bind_text(_sAllocateIp,1,network.id,16,SQLITE_STATIC); sqlite3_bind_text(_sAllocateIp,2,member.nodeId,10,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,3,(int)0 /*ZT_IP_ASSIGNMENT_TYPE_ADDRESS*/); + sqlite3_bind_int(_sAllocateIp,3,(int)0); sqlite3_bind_blob(_sAllocateIp,4,(const void *)ip6.rawIpData(),16,SQLITE_STATIC); sqlite3_bind_int(_sAllocateIp,5,routedNetmaskBits); // IP netmask bits from matching route sqlite3_bind_int(_sAllocateIp,6,6); // 6 == IPv6 @@ -943,13 +755,13 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co sqlite3_bind_text(_sCheckIfIpIsAllocated,1,network.id,16,SQLITE_STATIC); sqlite3_bind_blob(_sCheckIfIpIsAllocated,2,(const void *)ipBlob,16,SQLITE_STATIC); sqlite3_bind_int(_sCheckIfIpIsAllocated,3,4); // 4 == IPv4 - sqlite3_bind_int(_sCheckIfIpIsAllocated,4,(int)0 /*ZT_IP_ASSIGNMENT_TYPE_ADDRESS*/); + sqlite3_bind_int(_sCheckIfIpIsAllocated,4,(int)0); if (sqlite3_step(_sCheckIfIpIsAllocated) != SQLITE_ROW) { // No rows returned, so the IP is available sqlite3_reset(_sAllocateIp); sqlite3_bind_text(_sAllocateIp,1,network.id,16,SQLITE_STATIC); sqlite3_bind_text(_sAllocateIp,2,member.nodeId,10,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,3,(int)0 /*ZT_IP_ASSIGNMENT_TYPE_ADDRESS*/); + sqlite3_bind_int(_sAllocateIp,3,(int)0); sqlite3_bind_blob(_sAllocateIp,4,(const void *)ipBlob,16,SQLITE_STATIC); sqlite3_bind_int(_sAllocateIp,5,routedNetmaskBits); // IP netmask bits from matching route sqlite3_bind_int(_sAllocateIp,6,4); // 4 == IPv4 @@ -980,6 +792,7 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co } return NetworkController::NETCONF_QUERY_OK; + */ } unsigned int SqliteNetworkController::handleControlPlaneHttpGET( @@ -1671,76 +1484,19 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpDELETE( void SqliteNetworkController::threadMain() throw() { - uint64_t lastBackupTime = OSUtils::now(); - uint64_t lastCleanupTime = OSUtils::now(); - - while (_backupThreadRun) { - if ((OSUtils::now() - lastCleanupTime) >= 5000) { - const uint64_t now = OSUtils::now(); - lastCleanupTime = now; - + bool run = true; + while(run) { + Thread::sleep(250); + try { + std::vector errors; Mutex::Lock _l(_lock); - - // Clean out really old circuit tests to prevent memory build-up - for(std::map< uint64_t,_CircuitTestEntry >::iterator ct(_circuitTests.begin());ct!=_circuitTests.end();) { - if (!ct->second.test) { - _circuitTests.erase(ct++); - } else if ((now - ct->second.test->timestamp) >= ZT_SQLITENETWORKCONTROLLER_CIRCUIT_TEST_TIMEOUT) { - _node->circuitTestEnd(ct->second.test); - ::free((void *)ct->second.test); - _circuitTests.erase(ct++); - } else ++ct; + run = _dbCommitThreadRun; + if (!_db.commit(&errors)) { + // TODO: handle anything really bad } + } catch ( ... ) { + // TODO: handle anything really bad } - - if (((OSUtils::now() - lastBackupTime) >= ZT_NETCONF_BACKUP_PERIOD)&&(_backupNeeded)) { - lastBackupTime = OSUtils::now(); - - char backupPath[4096],backupPath2[4096]; - Utils::snprintf(backupPath,sizeof(backupPath),"%s.backupInProgress",_dbPath.c_str()); - Utils::snprintf(backupPath2,sizeof(backupPath),"%s.backup",_dbPath.c_str()); - OSUtils::rm(backupPath); // delete any unfinished backups - - sqlite3 *bakdb = (sqlite3 *)0; - sqlite3_backup *bak = (sqlite3_backup *)0; - if (sqlite3_open_v2(backupPath,&bakdb,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) { - fprintf(stderr,"SqliteNetworkController: CRITICAL: backup failed on sqlite3_open_v2()"ZT_EOL_S); - continue; - } - bak = sqlite3_backup_init(bakdb,"main",_db,"main"); - if (!bak) { - sqlite3_close(bakdb); - OSUtils::rm(backupPath); // delete any unfinished backups - fprintf(stderr,"SqliteNetworkController: CRITICAL: backup failed on sqlite3_backup_init()"ZT_EOL_S); - continue; - } - - int rc = SQLITE_OK; - for(;;) { - if (!_backupThreadRun) { - sqlite3_backup_finish(bak); - sqlite3_close(bakdb); - OSUtils::rm(backupPath); - return; - } - _lock.lock(); - rc = sqlite3_backup_step(bak,64); - _lock.unlock(); - if ((rc == SQLITE_OK)||(rc == SQLITE_LOCKED)||(rc == SQLITE_BUSY)) - Thread::sleep(50); - else break; - } - - sqlite3_backup_finish(bak); - sqlite3_close(bakdb); - - OSUtils::rm(backupPath2); - ::rename(backupPath,backupPath2); - - _backupNeeded = false; - } - - Thread::sleep(250); } } @@ -1753,27 +1509,40 @@ unsigned int SqliteNetworkController::_doCPGet( std::string &responseContentType) { // Assumes _lock is locked - char json[65536]; - if ((path.size() > 0)&&(path[0] == "network")) { + auto networks = _db.get("network"); + if (!networks) return 404; if ((path.size() >= 2)&&(path[1].length() == 16)) { - uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + auto network = _db.get(nwids); + if (!network) return 404; if (path.size() >= 3) { - // /network//... if (path[2] == "member") { + auto members = network->get("member"); + if (!members) return 404; if (path.size() >= 4) { - // Get specific member info - - uint64_t address = Utils::hexStrToU64(path[3].c_str()); + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + auto member = members->get(addrs); + if (!member) return 404; + + nlohmann::json o(member); + o["nwid"] = nwids; + o["address"] = addrs; + o["controllerInstanceId"] = _instanceId; + o["clock"] = OSUtils::now(); + responseBody = o.dump(2); + responseContentType = "application/json"; + return 200; + /* sqlite3_reset(_sGetMember2); sqlite3_bind_text(_sGetMember2,1,nwids,16,SQLITE_STATIC); sqlite3_bind_text(_sGetMember2,2,addrs,10,SQLITE_STATIC); @@ -1838,20 +1607,18 @@ unsigned int SqliteNetworkController::_doCPGet( responseContentType = "application/json"; return 200; } // else 404 + */ } else { - // List members - sqlite3_reset(_sListNetworkMembers); - sqlite3_bind_text(_sListNetworkMembers,1,nwids,16,SQLITE_STATIC); responseBody.push_back('{'); - bool firstMember = true; - while (sqlite3_step(_sListNetworkMembers) == SQLITE_ROW) { - responseBody.append(firstMember ? "\"" : ",\""); - firstMember = false; - responseBody.append((const char *)sqlite3_column_text(_sListNetworkMembers,0)); - responseBody.append("\":"); - responseBody.append((const char *)sqlite3_column_text(_sListNetworkMembers,1)); + for(auto i(members->begin());i!=members->end();++i) { + responseBody.append((i == members->begin()) ? "\"" : ",\""); + responseBody.append(i->key()); + responseBody.append("\":\""); + const std::string rc = i->value().value("memberRevision","0"); + responseBody.append(rc); + responseBody.append('"'); } responseBody.push_back('}'); responseContentType = "application/json"; @@ -1861,33 +1628,23 @@ unsigned int SqliteNetworkController::_doCPGet( } else if ((path[2] == "active")&&(path.size() == 3)) { - sqlite3_reset(_sGetActiveNodesOnNetwork); - sqlite3_bind_text(_sGetActiveNodesOnNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_int64(_sGetActiveNodesOnNetwork,2,(int64_t)(OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD)); - responseBody.push_back('{'); - bool firstActiveMember = true; - while (sqlite3_step(_sGetActiveNodesOnNetwork) == SQLITE_ROW) { - const char *nodeId = (const char *)sqlite3_column_text(_sGetActiveNodesOnNetwork,0); - const char *rhblob = (const char *)sqlite3_column_blob(_sGetActiveNodesOnNetwork,1); - if ((nodeId)&&(rhblob)) { - MemberRecentHistory rh; - rh.fromBlob(rhblob,sqlite3_column_bytes(_sGetActiveNodesOnNetwork,1)); - if (rh.size() > 0) { - if (firstActiveMember) { - firstActiveMember = false; - } else { - responseBody.push_back(','); - } - responseBody.push_back('"'); - responseBody.append(nodeId); + bool firstMember = true; + const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; + for(auto i(members->begin());i!=members->end();++i) { + auto recentLog = i->value()->get("recentLog"); + if ((recentLog)&&(recentLog.size() > 0)) { + auto mostRecentLog = recentLog[0]; + if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + responseBody.append((firstMember) ? "\"" : ",\""); + firstMember = false; + responseBody.append(i->key()); responseBody.append("\":"); - responseBody.append(rh.front()); + responseBody.append(mostRecentLog.dump()); } } } responseBody.push_back('}'); - responseContentType = "application/json"; return 200; @@ -1909,249 +1666,42 @@ unsigned int SqliteNetworkController::_doCPGet( } else { - sqlite3_reset(_sGetNetworkById); - sqlite3_bind_text(_sGetNetworkById,1,nwids,16,SQLITE_STATIC); - if (sqlite3_step(_sGetNetworkById) == SQLITE_ROW) { - unsigned int fl = (unsigned int)sqlite3_column_int(_sGetNetworkById,4); - std::string v6modes; - if ((fl & ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_RFC4193) != 0) - v6modes.append("rfc4193"); - if ((fl & ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_6PLANE) != 0) { - if (v6modes.length() > 0) - v6modes.push_back(','); - v6modes.append("6plane"); - } - if ((fl & ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_AUTO_ASSIGN) != 0) { - if (v6modes.length() > 0) - v6modes.push_back(','); - v6modes.append("zt"); - } - - Utils::snprintf(json,sizeof(json), - "{\n" - "\t\"nwid\": \"%s\",\n" - "\t\"controllerInstanceId\": \"%s\",\n" - "\t\"clock\": %llu,\n" - "\t\"name\": \"%s\",\n" - "\t\"private\": %s,\n" - "\t\"enableBroadcast\": %s,\n" - "\t\"allowPassiveBridging\": %s,\n" - "\t\"v4AssignMode\": \"%s\",\n" - "\t\"v6AssignMode\": \"%s\",\n" - "\t\"multicastLimit\": %d,\n" - "\t\"creationTime\": %llu,\n" - "\t\"revision\": %llu,\n" - "\t\"memberRevisionCounter\": %llu,\n" - "\t\"authorizedMemberCount\": %llu,\n" - "\t\"relays\": [", - nwids, - _instanceId.c_str(), - (unsigned long long)OSUtils::now(), - _jsonEscape((const char *)sqlite3_column_text(_sGetNetworkById,0)).c_str(), - (sqlite3_column_int(_sGetNetworkById,1) > 0) ? "true" : "false", - (sqlite3_column_int(_sGetNetworkById,2) > 0) ? "true" : "false", - (sqlite3_column_int(_sGetNetworkById,3) > 0) ? "true" : "false", - (((fl & ZT_DB_NETWORK_FLAG_ZT_MANAGED_V4_AUTO_ASSIGN) != 0) ? "zt" : ""), - v6modes.c_str(), - sqlite3_column_int(_sGetNetworkById,5), - (unsigned long long)sqlite3_column_int64(_sGetNetworkById,6), - (unsigned long long)sqlite3_column_int64(_sGetNetworkById,7), - (unsigned long long)sqlite3_column_int64(_sGetNetworkById,8), - (unsigned long long)sqlite3_column_int64(_sGetNetworkById,9)); - responseBody = json; - - sqlite3_reset(_sGetRelays); - sqlite3_bind_text(_sGetRelays,1,nwids,16,SQLITE_STATIC); - bool firstRelay = true; - while (sqlite3_step(_sGetRelays) == SQLITE_ROW) { - responseBody.append(firstRelay ? "\n\t\t" : ",\n\t\t"); - firstRelay = false; - responseBody.append("{\"address\":\""); - responseBody.append((const char *)sqlite3_column_text(_sGetRelays,0)); - responseBody.append("\",\"phyAddress\":\""); - responseBody.append(_jsonEscape((const char *)sqlite3_column_text(_sGetRelays,1))); - responseBody.append("\"}"); - } - - responseBody.append("],\n\t\"routes\": ["); - - sqlite3_reset(_sGetRoutes); - sqlite3_bind_text(_sGetRoutes,1,nwids,16,SQLITE_STATIC); - bool firstRoute = true; - while (sqlite3_step(_sGetRoutes) == SQLITE_ROW) { - responseBody.append(firstRoute ? "\n\t\t" : ",\n\t\t"); - firstRoute = false; - responseBody.append("{\"target\":"); - char tmp[128]; - const unsigned char *ip = (const unsigned char *)sqlite3_column_blob(_sGetRoutes,0); - switch(sqlite3_column_int(_sGetRoutes,3)) { // ipVersion - case 4: - Utils::snprintf(tmp,sizeof(tmp),"\"%d.%d.%d.%d/%d\"",(int)ip[12],(int)ip[13],(int)ip[14],(int)ip[15],sqlite3_column_int(_sGetRoutes,2)); - break; - case 6: - Utils::snprintf(tmp,sizeof(tmp),"\"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x/%d\"",(int)ip[0],(int)ip[1],(int)ip[2],(int)ip[3],(int)ip[4],(int)ip[5],(int)ip[6],(int)ip[7],(int)ip[8],(int)ip[9],(int)ip[10],(int)ip[11],(int)ip[12],(int)ip[13],(int)ip[14],(int)ip[15],sqlite3_column_int(_sGetRoutes,2)); - break; - } - responseBody.append(tmp); - if (sqlite3_column_type(_sGetRoutes,1) == SQLITE_NULL) { - responseBody.append(",\"via\":null"); - } else { - responseBody.append(",\"via\":"); - ip = (const unsigned char *)sqlite3_column_blob(_sGetRoutes,1); - switch(sqlite3_column_int(_sGetRoutes,3)) { // ipVersion - case 4: - Utils::snprintf(tmp,sizeof(tmp),"\"%d.%d.%d.%d\"",(int)ip[12],(int)ip[13],(int)ip[14],(int)ip[15]); - break; - case 6: - Utils::snprintf(tmp,sizeof(tmp),"\"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x\"",(int)ip[0],(int)ip[1],(int)ip[2],(int)ip[3],(int)ip[4],(int)ip[5],(int)ip[6],(int)ip[7],(int)ip[8],(int)ip[9],(int)ip[10],(int)ip[11],(int)ip[12],(int)ip[13],(int)ip[14],(int)ip[15]); - break; - } - responseBody.append(tmp); - } - responseBody.append(",\"flags\":"); - responseBody.append((const char *)sqlite3_column_text(_sGetRoutes,4)); - responseBody.append(",\"metric\":"); - responseBody.append((const char *)sqlite3_column_text(_sGetRoutes,5)); - responseBody.push_back('}'); - } - - responseBody.append("],\n\t\"ipAssignmentPools\": ["); - - sqlite3_reset(_sGetIpAssignmentPools2); - sqlite3_bind_text(_sGetIpAssignmentPools2,1,nwids,16,SQLITE_STATIC); - bool firstIpAssignmentPool = true; - while (sqlite3_step(_sGetIpAssignmentPools2) == SQLITE_ROW) { - const char *ipRangeStartB = reinterpret_cast(sqlite3_column_blob(_sGetIpAssignmentPools2,0)); - const char *ipRangeEndB = reinterpret_cast(sqlite3_column_blob(_sGetIpAssignmentPools2,1)); - if ((ipRangeStartB)&&(ipRangeEndB)) { - InetAddress ipps,ippe; - int ipVersion = sqlite3_column_int(_sGetIpAssignmentPools2,2); - if (ipVersion == 4) { - ipps.set((const void *)(ipRangeStartB + 12),4,0); - ippe.set((const void *)(ipRangeEndB + 12),4,0); - } else if (ipVersion == 6) { - ipps.set((const void *)ipRangeStartB,16,0); - ippe.set((const void *)ipRangeEndB,16,0); - } - if (ipps) { - responseBody.append(firstIpAssignmentPool ? "\n\t\t" : ",\n\t\t"); - firstIpAssignmentPool = false; - Utils::snprintf(json,sizeof(json),"{\"ipRangeStart\":\"%s\",\"ipRangeEnd\":\"%s\"}", - _jsonEscape(ipps.toIpString()).c_str(), - _jsonEscape(ippe.toIpString()).c_str()); - responseBody.append(json); - } - } - } + nlohmann::json o(network); + o["nwid"] = nwids; + o["controllerInstanceId"] = _instanceId; + o["clock"] = OSUtils::now(); + responseBody = o.dump(2); + responseContentType = "application/json"; + return 200; - responseBody.append("],\n\t\"rules\": ["); - - sqlite3_reset(_sListRules); - sqlite3_bind_text(_sListRules,1,nwids,16,SQLITE_STATIC); - bool firstRule = true; - while (sqlite3_step(_sListRules) == SQLITE_ROW) { - responseBody.append(firstRule ? "\n\t{\n" : ",{\n"); - firstRule = false; - Utils::snprintf(json,sizeof(json),"\t\t\"ruleNo\": %lld,\n",sqlite3_column_int64(_sListRules,0)); - responseBody.append(json); - if (sqlite3_column_type(_sListRules,1) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"nodeId\": \"%s\",\n",(const char *)sqlite3_column_text(_sListRules,1)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,2) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"sourcePort\": \"%s\",\n",(const char *)sqlite3_column_text(_sListRules,2)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,3) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"destPort\": \"%s\",\n",(const char *)sqlite3_column_text(_sListRules,3)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,4) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"vlanId\": %d,\n",sqlite3_column_int(_sListRules,4)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,5) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"vlanPcp\": %d,\n",sqlite3_column_int(_sListRules,5)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,6) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"etherType\": %d,\n",sqlite3_column_int(_sListRules,6)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,7) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"macSource\": \"%s\",\n",MAC((const char *)sqlite3_column_text(_sListRules,7)).toString().c_str()); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,8) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"macDest\": \"%s\",\n",MAC((const char *)sqlite3_column_text(_sListRules,8)).toString().c_str()); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,9) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"ipSource\": \"%s\",\n",_jsonEscape((const char *)sqlite3_column_text(_sListRules,9)).c_str()); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,10) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"ipDest\": \"%s\",\n",_jsonEscape((const char *)sqlite3_column_text(_sListRules,10)).c_str()); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,11) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"ipTos\": %d,\n",sqlite3_column_int(_sListRules,11)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,12) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"ipProtocol\": %d,\n",sqlite3_column_int(_sListRules,12)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,13) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"ipSourcePort\": %d,\n",sqlite3_column_int(_sListRules,13)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,14) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"ipDestPort\": %d,\n",sqlite3_column_int(_sListRules,14)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,15) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"flags\": %lu,\n",(unsigned long)sqlite3_column_int64(_sListRules,15)); - responseBody.append(json); - } - if (sqlite3_column_type(_sListRules,16) != SQLITE_NULL) { - Utils::snprintf(json,sizeof(json),"\t\t\"invFlags\": %lu,\n",(unsigned long)sqlite3_column_int64(_sListRules,16)); - responseBody.append(json); - } - responseBody.append("\t\t\"action\": \""); - responseBody.append(_jsonEscape( (sqlite3_column_type(_sListRules,17) == SQLITE_NULL) ? "drop" : (const char *)sqlite3_column_text(_sListRules,17) )); - responseBody.append("\"\n\t}"); - } - - responseBody.append("]\n}\n"); - responseContentType = "application/json"; - return 200; - } // else 404 } } else if (path.size() == 1) { - // list networks - sqlite3_reset(_sListNetworks); - responseContentType = "application/json"; + responseBody = "["; - bool first = true; - while (sqlite3_step(_sListNetworks) == SQLITE_ROW) { - if (first) { - first = false; - responseBody.push_back('"'); - } else responseBody.append(",\""); - responseBody.append((const char *)sqlite3_column_text(_sListNetworks,0)); - responseBody.push_back('"'); + for(auto i(networks->begin());i!=networks.end();++i) { + responseBody.append((i == networks->begin()) ? "\"" : ",\""); + responseBody.append(i->key()); + responseBody.append("\""); } responseBody.push_back(']'); + responseContentType = "application/json"; return 200; + } // else 404 + } else if ((path.size() > 0)&&(path[0] == "_dump")) { + + responseBody = _db.dump(2); + responseContentType = "application/json"; + return 200; + } else { - // GET /controller returns status and API version if controller is supported + Utils::snprintf(json,sizeof(json),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu,\n\t\"instanceId\": \"%s\"\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now(),_instanceId.c_str()); responseBody = json; responseContentType = "application/json"; return 200; + } return 404; diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp index b917f6fb..0d549abc 100644 --- a/controller/SqliteNetworkController.hpp +++ b/controller/SqliteNetworkController.hpp @@ -30,22 +30,17 @@ #include -#include - #include #include #include #include "../node/Constants.hpp" + #include "../node/NetworkController.hpp" #include "../node/Mutex.hpp" #include "../osdep/Thread.hpp" -// Number of in-memory last log entries to maintain per user -#define ZT_SQLITENETWORKCONTROLLER_IN_MEMORY_LOG_SIZE 32 - -// How long do circuit tests last before they're forgotten? -#define ZT_SQLITENETWORKCONTROLLER_CIRCUIT_TEST_TIMEOUT 60000 +#include "../ext/offbase/offbase.hpp" namespace ZeroTier { @@ -54,7 +49,7 @@ class Node; class SqliteNetworkController : public NetworkController { public: - SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath); + SqliteNetworkController(Node *node,const char *dbPath); virtual ~SqliteNetworkController(); virtual NetworkController::ResultCode doNetworkConfigRequest( @@ -62,7 +57,7 @@ public: const Identity &signingId, const Identity &identity, uint64_t nwid, - const Dictionary &metaData, + const Dictionary &metaData, NetworkConfig &nc); unsigned int handleControlPlaneHttpGET( @@ -92,15 +87,6 @@ public: throw(); private: - /* deprecated - enum IpAssignmentType { - // IP assignment is a static IP address - ZT_IP_ASSIGNMENT_TYPE_ADDRESS = 0, - // IP assignment is a network -- a route via this interface, not an address - ZT_IP_ASSIGNMENT_TYPE_NETWORK = 1 - }; - */ - unsigned int _doCPGet( const std::vector &path, const std::map &urlArgs, @@ -111,13 +97,11 @@ private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); - Node *_node; - Thread _backupThread; - volatile bool _backupThreadRun; - volatile bool _backupNeeded; - std::string _dbPath; - std::string _circuitTestPath; + Node *const _node; std::string _instanceId; + offbase _db; + Thread _dbCommitThread; + volatile bool _dbCommitThreadRun; // Circuit tests outstanding struct _CircuitTestEntry @@ -130,48 +114,6 @@ private: // Last request time by address, for rate limitation std::map< std::pair,uint64_t > _lastRequestTime; - sqlite3 *_db; - - sqlite3_stmt *_sGetNetworkById; - sqlite3_stmt *_sGetMember; - sqlite3_stmt *_sCreateMember; - sqlite3_stmt *_sGetNodeIdentity; - sqlite3_stmt *_sCreateOrReplaceNode; - sqlite3_stmt *_sGetActiveBridges; - sqlite3_stmt *_sGetIpAssignmentsForNode; - sqlite3_stmt *_sGetIpAssignmentPools; - sqlite3_stmt *_sCheckIfIpIsAllocated; - sqlite3_stmt *_sAllocateIp; - sqlite3_stmt *_sDeleteIpAllocations; - sqlite3_stmt *_sGetRelays; - sqlite3_stmt *_sListNetworks; - sqlite3_stmt *_sListNetworkMembers; - sqlite3_stmt *_sGetMember2; - sqlite3_stmt *_sGetIpAssignmentPools2; - sqlite3_stmt *_sListRules; - sqlite3_stmt *_sCreateRule; - sqlite3_stmt *_sCreateNetwork; - sqlite3_stmt *_sGetNetworkRevision; - sqlite3_stmt *_sSetNetworkRevision; - sqlite3_stmt *_sDeleteRelaysForNetwork; - sqlite3_stmt *_sCreateRelay; - sqlite3_stmt *_sDeleteIpAssignmentPoolsForNetwork; - sqlite3_stmt *_sDeleteRulesForNetwork; - sqlite3_stmt *_sCreateIpAssignmentPool; - sqlite3_stmt *_sUpdateMemberAuthorized; - sqlite3_stmt *_sUpdateMemberActiveBridge; - sqlite3_stmt *_sUpdateMemberHistory; - sqlite3_stmt *_sDeleteMember; - sqlite3_stmt *_sDeleteAllNetworkMembers; - sqlite3_stmt *_sGetActiveNodesOnNetwork; - sqlite3_stmt *_sDeleteNetwork; - sqlite3_stmt *_sCreateRoute; - sqlite3_stmt *_sGetRoutes; - sqlite3_stmt *_sDeleteRoutes; - sqlite3_stmt *_sIncrementMemberRevisionCounter; - sqlite3_stmt *_sGetConfig; - sqlite3_stmt *_sSetConfig; - Mutex _lock; }; diff --git a/controller/schema.sql b/controller/schema.sql index 479daa68..d1daf8d0 100644 --- a/controller/schema.sql +++ b/controller/schema.sql @@ -86,17 +86,9 @@ CREATE TABLE Route ( CREATE INDEX Route_networkId ON Route (networkId); -CREATE TABLE Relay ( - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - address char(10) NOT NULL, - phyAddress varchar(64) NOT NULL -); - -CREATE UNIQUE INDEX Relay_networkId_address ON Relay (networkId,address); - CREATE TABLE Rule ( networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - policyId varchar(32), + capId integer, ruleNo integer NOT NULL, ruleType integer NOT NULL DEFAULT(0), "addr" blob(16), @@ -106,5 +98,15 @@ CREATE TABLE Rule ( "int4" integer ); -CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo); -CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId); +CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId); + +CREATE TABLE MemberTC ( + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE, + tagId integer, + tagValue integer, + capId integer, + capMaxCustodyChainLength integer NOT NULL DEFAULT(1) +); + +CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId); diff --git a/ext/json/LICENSE.MIT b/ext/json/LICENSE.MIT deleted file mode 100644 index e2ac4891..00000000 --- a/ext/json/LICENSE.MIT +++ /dev/null @@ -1,22 +0,0 @@ -The library is licensed under the MIT License -: - -Copyright (c) 2013-2016 Niels Lohmann - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/ext/json/README.md b/ext/json/README.md deleted file mode 100644 index c0bb61b1..00000000 --- a/ext/json/README.md +++ /dev/null @@ -1,511 +0,0 @@ -[![JSON for Modern C++](https://raw.githubusercontent.com/nlohmann/json/master/doc/json.gif)](https://github.com/nlohmann/json/releases) - -[![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) -[![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk?svg=true)](https://ci.appveyor.com/project/nlohmann/json) -[![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) -[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) -[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/p5o4znPnGHJpDVqN) -[![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) -[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) -[![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) -[![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) - -## Design goals - -There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: - -- **Intuitive syntax**. In languages such as Python, JSON feels like a first class data type. We used all the operator magic of modern C++ to achieve the same feeling in your code. Check out the [examples below](#examples) and you know, what I mean. - -- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/src/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. - -- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/blob/master/test/src/unit.cpp) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) that there are no memory leaks. - -Other aspects were not so important to us: - -- **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. - -- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). - -See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. - - -## Integration - -The single required source, file `json.hpp` is in the `src` directory or [released here](https://github.com/nlohmann/json/releases). All you need to do is add - -```cpp -#include "json.hpp" - -// for convenience -using json = nlohmann::json; -``` - -to the files you want to use JSON objects. That's it. Do not forget to set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). - -:beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. - - -## Examples - -Here are some examples to give you an idea how to use the class. - -Assume you want to create the JSON object - -```json -{ - "pi": 3.141, - "happy": true, - "name": "Niels", - "nothing": null, - "answer": { - "everything": 42 - }, - "list": [1, 0, 2], - "object": { - "currency": "USD", - "value": 42.99 - } -} -``` - -With the JSON class, you could write: - -```cpp -// create an empty structure (null) -json j; - -// add a number that is stored as double (note the implicit conversion of j to an object) -j["pi"] = 3.141; - -// add a Boolean that is stored as bool -j["happy"] = true; - -// add a string that is stored as std::string -j["name"] = "Niels"; - -// add another null object by passing nullptr -j["nothing"] = nullptr; - -// add an object inside the object -j["answer"]["everything"] = 42; - -// add an array that is stored as std::vector (using an initializer list) -j["list"] = { 1, 0, 2 }; - -// add another object (using an initializer list of pairs) -j["object"] = { {"currency", "USD"}, {"value", 42.99} }; - -// instead, you could also write (which looks very similar to the JSON above) -json j2 = { - {"pi", 3.141}, - {"happy", true}, - {"name", "Niels"}, - {"nothing", nullptr}, - {"answer", { - {"everything", 42} - }}, - {"list", {1, 0, 2}}, - {"object", { - {"currency", "USD"}, - {"value", 42.99} - }} -}; -``` - -Note that in all these cases, you never need to "tell" the compiler which JSON value you want to use. If you want to be explicit or express some edge cases, the functions `json::array` and `json::object` will help: - -```cpp -// a way to express the empty array [] -json empty_array_explicit = json::array(); - -// ways to express the empty object {} -json empty_object_implicit = json({}); -json empty_object_explicit = json::object(); - -// a way to express an _array_ of key/value pairs [["currency", "USD"], ["value", 42.99]] -json array_not_object = { json::array({"currency", "USD"}), json::array({"value", 42.99}) }; -``` - - -### Serialization / Deserialization - -You can create an object (deserialization) by appending `_json` to a string literal: - -```cpp -// create object from string literal -json j = "{ \"happy\": true, \"pi\": 3.141 }"_json; - -// or even nicer with a raw string literal -auto j2 = R"( - { - "happy": true, - "pi": 3.141 - } -)"_json; - -// or explicitly -auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); -``` - -You can also get a string representation (serialize): - -```cpp -// explicit conversion to string -std::string s = j.dump(); // {\"happy\":true,\"pi\":3.141} - -// serialization with pretty printing -// pass in the amount of spaces to indent -std::cout << j.dump(4) << std::endl; -// { -// "happy": true, -// "pi": 3.141 -// } -``` - -You can also use streams to serialize and deserialize: - -```cpp -// deserialize from standard input -json j; -std::cin >> j; - -// serialize to standard output -std::cout << j; - -// the setw manipulator was overloaded to set the indentation for pretty printing -std::cout << std::setw(4) << j << std::endl; -``` - -These operators work for any subclasses of `std::istream` or `std::ostream`. - -Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. - - -### STL-like access - -We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) requirement. - -```cpp -// create an array using push_back -json j; -j.push_back("foo"); -j.push_back(1); -j.push_back(true); - -// iterate the array -for (json::iterator it = j.begin(); it != j.end(); ++it) { - std::cout << *it << '\n'; -} - -// range-based for -for (auto& element : j) { - std::cout << element << '\n'; -} - -// getter/setter -const std::string tmp = j[0]; -j[1] = 42; -bool foo = j.at(2); - -// other stuff -j.size(); // 3 entries -j.empty(); // false -j.type(); // json::value_t::array -j.clear(); // the array is empty again - -// convenience type checkers -j.is_null(); -j.is_boolean(); -j.is_number(); -j.is_object(); -j.is_array(); -j.is_string(); - -// comparison -j == "[\"foo\", 1, true]"_json; // true - -// create an object -json o; -o["foo"] = 23; -o["bar"] = false; -o["baz"] = 3.141; - -// special iterator member functions for objects -for (json::iterator it = o.begin(); it != o.end(); ++it) { - std::cout << it.key() << " : " << it.value() << "\n"; -} - -// find an entry -if (o.find("foo") != o.end()) { - // there is an entry with key "foo" -} - -// or simpler using count() -int foo_present = o.count("foo"); // 1 -int fob_present = o.count("fob"); // 0 - -// delete an entry -o.erase("foo"); -``` - - -### Conversion from STL containers - -Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON types (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends how the elements are ordered in the respective STL container. - -```cpp -std::vector c_vector {1, 2, 3, 4}; -json j_vec(c_vector); -// [1, 2, 3, 4] - -std::deque c_deque {1.2, 2.3, 3.4, 5.6}; -json j_deque(c_deque); -// [1.2, 2.3, 3.4, 5.6] - -std::list c_list {true, true, false, true}; -json j_list(c_list); -// [true, true, false, true] - -std::forward_list c_flist {12345678909876, 23456789098765, 34567890987654, 45678909876543}; -json j_flist(c_flist); -// [12345678909876, 23456789098765, 34567890987654, 45678909876543] - -std::array c_array {{1, 2, 3, 4}}; -json j_array(c_array); -// [1, 2, 3, 4] - -std::set c_set {"one", "two", "three", "four", "one"}; -json j_set(c_set); // only one entry for "one" is used -// ["four", "one", "three", "two"] - -std::unordered_set c_uset {"one", "two", "three", "four", "one"}; -json j_uset(c_uset); // only one entry for "one" is used -// maybe ["two", "three", "four", "one"] - -std::multiset c_mset {"one", "two", "one", "four"}; -json j_mset(c_mset); // only one entry for "one" is used -// maybe ["one", "two", "four"] - -std::unordered_multiset c_umset {"one", "two", "one", "four"}; -json j_umset(c_umset); // both entries for "one" are used -// maybe ["one", "two", "one", "four"] -``` - -Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON types (see examples above) can be used to to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. - -```cpp -std::map c_map { {"one", 1}, {"two", 2}, {"three", 3} }; -json j_map(c_map); -// {"one": 1, "three": 3, "two": 2 } - -std::unordered_map c_umap { {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; -json j_umap(c_umap); -// {"one": 1.2, "two": 2.3, "three": 3.4} - -std::multimap c_mmap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; -json j_mmap(c_mmap); // only one entry for key "three" is used -// maybe {"one": true, "two": true, "three": true} - -std::unordered_multimap c_ummap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; -json j_ummap(c_ummap); // only one entry for key "three" is used -// maybe {"one": true, "two": true, "three": true} -``` - -### JSON Pointer and JSON Patch - -The library supports **JSON Pointer** ([RFC 6901](https://tools.ietf.org/html/rfc6901)) as alternative means to address structured values. On top of this, **JSON Patch** ([RFC 6902](https://tools.ietf.org/html/rfc6902)) allows to describe differences between two JSON values - effectively allowing patch and diff operations known from Unix. - -```cpp -// a JSON value -json j_original = R"({ - "baz": ["one", "two", "three"], - "foo": "bar" -})"_json; - -// access members with a JSON pointer (RFC 6901) -j_original["/baz/1"_json_pointer]; -// "two" - -// a JSON patch (RFC 6902) -json j_patch = R"([ - { "op": "replace", "path": "/baz", "value": "boo" }, - { "op": "add", "path": "/hello", "value": ["world"] }, - { "op": "remove", "path": "/foo"} -])"_json; - -// apply the patch -json j_result = j_original.patch(j_patch); -// { -// "baz": "boo", -// "hello": ["world"] -// } - -// calculate a JSON patch from two JSON values -json::diff(j_result, j_original); -// [ -// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, -// { "op": "remove","path": "/hello" }, -// { "op": "add", "path": "/foo", "value": "bar" } -// ] -``` - - -### Implicit conversions - -The type of the JSON object is determined automatically by the expression to store. Likewise, the stored value is implicitly converted. - -```cpp -// strings -std::string s1 = "Hello, world!"; -json js = s1; -std::string s2 = js; - -// Booleans -bool b1 = true; -json jb = b1; -bool b2 = jb; - -// numbers -int i = 42; -json jn = i; -double f = jn; - -// etc. -``` - -You can also explicitly ask for the value: - -```cpp -std::string vs = js.get(); -bool vb = jb.get(); -int vi = jn.get(); - -// etc. -``` - - -## Supported compilers - -Though it's 2016 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: - -- GCC 4.9 - 6.0 (and possibly later) -- Clang 3.4 - 3.9 (and possibly later) -- Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) - -I would be happy to learn about other compilers/versions. - -Please note: - -- GCC 4.8 does not work because of two bugs ([55817](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55817) and [57824](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57824)) in the C++11 support. Note there is a [pull request](https://github.com/nlohmann/json/pull/212) to fix some of the issues. -- Android defaults to using very old compilers and C++ libraries. To fix this, add the following to your `Application.mk`. This will switch to the LLVM C++ library, the Clang compiler, and enable C++11 and other features disabled by default. - - ``` - APP_STL := c++_shared - NDK_TOOLCHAIN_VERSION := clang3.6 - APP_CPPFLAGS += -frtti -fexceptions - ``` - - The code compiles successfully with [Android NDK](https://developer.android.com/ndk/index.html?hl=ml), Revision 9 - 11 (and possibly later) and [CrystaX's Android NDK](https://www.crystax.net/en/android/ndk) version 10. - -- For GCC running on MinGW or Android SDK, the error `'to_string' is not a member of 'std'` (or similarly, for `strtod`) may occur. Note this is not an issue with the code, but rather with the compiler itself. On Android, see above to build with a newer environment. For MinGW, please refer to [this site](http://tehsausage.com/mingw-to-string) and [this discussion](https://github.com/nlohmann/json/issues/136) for information on how to fix this bug. For Android NDK using `APP_STL := gnustl_static`, please refer to [this discussion](https://github.com/nlohmann/json/issues/219). - -The following compilers are currently used in continuous integration at [Travis](https://travis-ci.org/nlohmann/json) and [AppVeyor](https://ci.appveyor.com/project/nlohmann/json): - -| Compiler | Operating System | Version String | -|-----------------|------------------------------|----------------| -| GCC 4.9.3 | Ubuntu 14.04.4 LTS | g++-4.9 (Ubuntu 4.9.3-8ubuntu2~14.04) 4.9.3 | -| GCC 5.3.0 | Ubuntu 14.04.4 LTS | g++-5 (Ubuntu 5.3.0-3ubuntu1~14.04) 5.3.0 20151204 | -| GCC 6.1.1 | Ubuntu 14.04.4 LTS | g++-6 (Ubuntu 6.1.1-3ubuntu11~14.04.1) 6.1.1 20160511 | -| Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | -| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | -| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | -| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | -| Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | -| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | -| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | -| Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | -| Clang Xcode 8.0 | Darwin Kernel Version 15.5.0 (OSX 10.11.5) | Apple LLVM version 8.0.0 (clang-800.0.24.1) | -| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | - - -## License - - - -The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): - -Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -## Thanks - -I deeply appreciate the help of the following people. - -- [Teemperor](https://github.com/Teemperor) implemented CMake support and lcov integration, realized escape and Unicode handling in the string parser, and fixed the JSON serialization. -- [elliotgoodrich](https://github.com/elliotgoodrich) fixed an issue with double deletion in the iterator classes. -- [kirkshoop](https://github.com/kirkshoop) made the iterators of the class composable to other libraries. -- [wancw](https://github.com/wanwc) fixed a bug that hindered the class to compile with Clang. -- Tomas Åblad found a bug in the iterator implementation. -- [Joshua C. Randall](https://github.com/jrandall) fixed a bug in the floating-point serialization. -- [Aaron Burghardt](https://github.com/aburgh) implemented code to parse streams incrementally. Furthermore, he greatly improved the parser class by allowing the definition of a filter function to discard undesired elements while parsing. -- [Daniel Kopeček](https://github.com/dkopecek) fixed a bug in the compilation with GCC 5.0. -- [Florian Weber](https://github.com/Florianjw) fixed a bug in and improved the performance of the comparison operators. -- [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. -- [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. -- [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. -- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. -- [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. -- [dariomt](https://github.com/dariomt) fixed some typos in the examples. -- [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. -- [Colin Hirsch](https://github.com/ColinH) took care of a small namespace issue. -- [Huu Nguyen](https://github.com/whoshuu) correct a variable name in the documentation. -- [Silverweed](https://github.com/silverweed) overloaded `parse()` to accept an rvalue reference. -- [dariomt](https://github.com/dariomt) fixed a subtlety in MSVC type support and implemented the `get_ref()` function to get a reference to stored values. -- [ZahlGraf](https://github.com/ZahlGraf) added a workaround that allows compilation using Android NDK. -- [whackashoe](https://github.com/whackashoe) replaced a function that was marked as unsafe by Visual Studio. -- [406345](https://github.com/406345) fixed two small warnings. -- [Glen Fernandes](https://github.com/glenfe) noted a potential portability problem in the `has_mapped_type` function. -- [Corbin Hughes](https://github.com/nibroc) fixed some typos in the contribution guidelines. -- [twelsby](https://github.com/twelsby) fixed the array subscript operator, an issue that failed the MSVC build, and floating-point parsing/dumping. He further added support for unsigned integer numbers and implemented better roundtrip support for parsed numbers. -- [Volker Diels-Grabsch](https://github.com/vog) fixed a link in the README file. -- [msm-](https://github.com/msm-) added support for american fuzzy lop. -- [Annihil](https://github.com/Annihil) fixed an example in the README file. -- [Themercee](https://github.com/Themercee) noted a wrong URL in the README file. -- [Lv Zheng](https://github.com/lv-zheng) fixed a namespace issue with `int64_t` and `uint64_t`. -- [abc100m](https://github.com/abc100m) analyzed the issues with GCC 4.8 and proposed a [partial solution](https://github.com/nlohmann/json/pull/212). -- [zewt](https://github.com/zewt) added useful notes to the README file about Android. -- [Róbert Márki](https://github.com/robertmrk) added a fix to use move iterators and improved the integration via CMake. -- [Chris Kitching](https://github.com/ChrisKitching) cleaned up the CMake files. -- [Tom Needham](https://github.com/06needhamt) fixed a subtle bug with MSVC 2015 which was also proposed by [Michael K.](https://github.com/Epidal). -- [Mário Feroldi](https://github.com/thelostt) fixed a small typo. -- [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. -- [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. - -Thanks a lot for helping out! - - -## Notes - -- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). -- As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions. - - -## Execute unit tests - -To compile and run the tests, you need to execute - -```sh -$ make -$ ./json_unit "*" - -=============================================================================== -All tests passed (8905012 assertions in 32 test cases) -``` - -For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). diff --git a/ext/json/json.hpp b/ext/json/json.hpp deleted file mode 100644 index 878fb899..00000000 --- a/ext/json/json.hpp +++ /dev/null @@ -1,10435 +0,0 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 2.0.2 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -Copyright (c) 2013-2016 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// exclude unsupported compilers -#if defined(__clang__) - #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) - #if CLANG_VERSION < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#elif defined(__GNUC__) - #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - #if GCC_VERSION < 40900 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ - - -/*! -@brief unnamed namespace with internal helper functions -@since version 1.0.0 -*/ -namespace -{ -/*! -@brief Helper to determine whether there's a key_type for T. - -Thus helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. - -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0 -*/ -template -struct has_mapped_type -{ - private: - template static char test(typename C::mapped_type*); - template static char (&test(...))[2]; - public: - static constexpr bool value = sizeof(test(0)) == 1; -}; - -/*! -@brief helper class to create locales with decimal point - -This struct is used a default locale during the JSON serialization. JSON -requires the decimal point to be `.`, so this function overloads the -`do_decimal_point()` function to return `.`. This function is called by -float-to-string conversions to retrieve the decimal separator between integer -and fractional parts. - -@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 -@since version 2.0.0 -*/ -struct DecimalSeparator : std::numpunct -{ - char do_decimal_point() const - { - return '.'; - } -}; - -} - -/*! -@brief a class to store JSON values - -@tparam ObjectType type for JSON objects (`std::map` by default; will be used -in @ref object_t) -@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used -in @ref array_t) -@tparam StringType type for JSON strings and object keys (`std::string` by -default; will be used in @ref string_t) -@tparam BooleanType type for JSON booleans (`bool` by default; will be used -in @ref boolean_t) -@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by -default; will be used in @ref number_integer_t) -@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c -`uint64_t` by default; will be used in @ref number_unsigned_t) -@tparam NumberFloatType type for JSON floating-point numbers (`double` by -default; will be used in @ref number_float_t) -@tparam AllocatorType type of the allocator to use (`std::allocator` by -default) - -@requirement The class satisfies the following concept requirements: -- Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): - JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): - A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): - A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): - A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): - A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): - JSON values can be destructed. -- Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): - JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): - All non-static data members are private and standard layout types, the class - has no virtual functions or (virtual) base classes. -- Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): - JSON values can be compared with `==`, see @ref - operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): - JSON values can be compared with `<`, see @ref - operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): - Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of - other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): - JSON values can be compared against `std::nullptr_t` objects which are used - to model the `null` value. -- Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): - JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); - JSON values can be used like STL containers and provide reverse iterator - access. - -@invariant The member variables @a m_value and @a m_type have the following -relationship: -- If `m_type == value_t::object`, then `m_value.object != nullptr`. -- If `m_type == value_t::array`, then `m_value.array != nullptr`. -- If `m_type == value_t::string`, then `m_value.string != nullptr`. -The invariants are checked by member function assert_invariant(). - -@internal -@note ObjectType trick from http://stackoverflow.com/a/9860911 -@endinternal - -@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange -Format](http://rfc7159.net/rfc7159) - -@since version 1.0.0 - -@nosubgrouping -*/ -template < - template class ObjectType = std::map, - template class ArrayType = std::vector, - class StringType = std::string, - class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator - > -class basic_json -{ - private: - /// workaround type for MSVC - using basic_json_t = basic_json; - - public: - // forward declarations - template class json_reverse_iterator; - class json_pointer; - - ///////////////////// - // container types // - ///////////////////// - - /// @name container types - /// The canonic container types to use @ref basic_json like any other STL - /// container. - /// @{ - - /// the type of elements in a basic_json container - using value_type = basic_json; - - /// the type of an element reference - using reference = value_type&; - /// the type of an element const reference - using const_reference = const value_type&; - - /// a type to represent differences between iterators - using difference_type = std::ptrdiff_t; - /// a type to represent container sizes - using size_type = std::size_t; - - /// the allocator type - using allocator_type = AllocatorType; - - /// the type of an element pointer - using pointer = typename std::allocator_traits::pointer; - /// the type of an element const pointer - using const_pointer = typename std::allocator_traits::const_pointer; - - /// an iterator for a basic_json container - class iterator; - /// a const iterator for a basic_json container - class const_iterator; - /// a reverse iterator for a basic_json container - using reverse_iterator = json_reverse_iterator; - /// a const reverse iterator for a basic_json container - using const_reverse_iterator = json_reverse_iterator; - - /// @} - - - /*! - @brief returns the allocator associated with the container - */ - static allocator_type get_allocator() - { - return allocator_type(); - } - - - /////////////////////////// - // JSON value data types // - /////////////////////////// - - /// @name JSON value data types - /// The data types to store a JSON value. These types are derived from - /// the template arguments passed to class @ref basic_json. - /// @{ - - /*! - @brief a type for an object - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: - > An object is an unordered collection of zero or more name/value pairs, - > where a name is a string and a value is a string, number, boolean, null, - > object, or array. - - To store objects in C++, a type is defined by the template parameters - described below. - - @tparam ObjectType the container to store objects (e.g., `std::map` or - `std::unordered_map`) - @tparam StringType the type of the keys or names (e.g., `std::string`). - The comparison function `std::less` is used to order elements - inside the container. - @tparam AllocatorType the allocator to use for objects (e.g., - `std::allocator`) - - #### Default type - - With the default values for @a ObjectType (`std::map`), @a StringType - (`std::string`), and @a AllocatorType (`std::allocator`), the default - value for @a object_t is: - - @code {.cpp} - std::map< - std::string, // key_type - basic_json, // value_type - std::less, // key_compare - std::allocator> // allocator_type - > - @endcode - - #### Behavior - - The choice of @a object_t influences the behavior of the JSON class. With - the default type, objects have the following behavior: - - - When all names are unique, objects will be interoperable in the sense - that all software implementations receiving that object will agree on - the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. - - Internally, name/value pairs are stored in lexicographical order of the - names. Objects will also be serialized (see @ref dump) in this order. - For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored - and serialized as `{"a": 2, "b": 1}`. - - When comparing objects, the order of the name/value pairs is irrelevant. - This makes objects interoperable in the sense that they will not be - affected by these differences. For instance, `{"b": 1, "a": 2}` and - `{"a": 2, "b": 1}` will be treated as equal. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the object's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON object. - - #### Storage - - Objects are stored as pointers in a @ref basic_json type. That is, for any - access to object values, a pointer of type `object_t*` must be - dereferenced. - - @sa @ref array_t -- type for an array value - - @since version 1.0.0 - - @note The order name/value pairs are added to the object is *not* - preserved by the library. Therefore, iterating an object may return - name/value pairs in a different order than they were originally stored. In - fact, keys will be traversed in alphabetical order as `std::map` with - `std::less` is used by default. Please note this behavior conforms to [RFC - 7159](http://rfc7159.net/rfc7159), because any order implements the - specified "unordered" nature of JSON objects. - */ - using object_t = ObjectType, - AllocatorType>>; - - /*! - @brief a type for an array - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: - > An array is an ordered sequence of zero or more values. - - To store objects in C++, a type is defined by the template parameters - explained below. - - @tparam ArrayType container type to store arrays (e.g., `std::vector` or - `std::list`) - @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) - - #### Default type - - With the default values for @a ArrayType (`std::vector`) and @a - AllocatorType (`std::allocator`), the default value for @a array_t is: - - @code {.cpp} - std::vector< - basic_json, // value_type - std::allocator // allocator_type - > - @endcode - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the array's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON array. - - #### Storage - - Arrays are stored as pointers in a @ref basic_json type. That is, for any - access to array values, a pointer of type `array_t*` must be dereferenced. - - @sa @ref object_t -- type for an object value - - @since version 1.0.0 - */ - using array_t = ArrayType>; - - /*! - @brief a type for a string - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: - > A string is a sequence of zero or more Unicode characters. - - To store objects in C++, a type is defined by the template parameter - described below. Unicode values are split by the JSON class into - byte-sized characters during deserialization. - - @tparam StringType the container to store strings (e.g., `std::string`). - Note this container is used for keys/names in objects, see @ref object_t. - - #### Default type - - With the default values for @a StringType (`std::string`), the default - value for @a string_t is: - - @code {.cpp} - std::string - @endcode - - #### String comparison - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > Software implementations are typically required to test names of object - > members for equality. Implementations that transform the textual - > representation into sequences of Unicode code units and then perform the - > comparison numerically, code unit by code unit, are interoperable in the - > sense that implementations will agree in all cases on equality or - > inequality of two strings. For example, implementations that compare - > strings with escaped characters unconverted may incorrectly find that - > `"a\\b"` and `"a\u005Cb"` are not equal. - - This implementation is interoperable as it does compare strings code unit - by code unit. - - #### Storage - - String values are stored as pointers in a @ref basic_json type. That is, - for any access to string values, a pointer of type `string_t*` must be - dereferenced. - - @since version 1.0.0 - */ - using string_t = StringType; - - /*! - @brief a type for a boolean - - [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a - type which differentiates the two literals `true` and `false`. - - To store objects in C++, a type is defined by the template parameter @a - BooleanType which chooses the type to use. - - #### Default type - - With the default values for @a BooleanType (`bool`), the default value for - @a boolean_t is: - - @code {.cpp} - bool - @endcode - - #### Storage - - Boolean values are stored directly inside a @ref basic_json type. - - @since version 1.0.0 - */ - using boolean_t = BooleanType; - - /*! - @brief a type for a number (integer) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store integer numbers in C++, a type is defined by the template - parameter @a NumberIntegerType which chooses the type to use. - - #### Default type - - With the default values for @a NumberIntegerType (`int64_t`), the default - value for @a number_integer_t is: - - @code {.cpp} - int64_t - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. - - When the default type is used, the maximal integer number that can be - stored is `9223372036854775807` (INT64_MAX) and the minimal integer number - that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers - that are out of range will yield over/underflow when used in a - constructor. During deserialization, too large or small integer numbers - will be automatically be stored as @ref number_unsigned_t or @ref - number_float_t. - - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. - - As this range is a subrange of the exactly supported range [INT64_MIN, - INT64_MAX], this class's integer type is interoperable. - - #### Storage - - Integer number values are stored directly inside a @ref basic_json type. - - @sa @ref number_float_t -- type for number values (floating-point) - - @sa @ref number_unsigned_t -- type for number values (unsigned integer) - - @since version 1.0.0 - */ - using number_integer_t = NumberIntegerType; - - /*! - @brief a type for a number (unsigned) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store unsigned integer numbers in C++, a type is defined by the - template parameter @a NumberUnsignedType which chooses the type to use. - - #### Default type - - With the default values for @a NumberUnsignedType (`uint64_t`), the - default value for @a number_unsigned_t is: - - @code {.cpp} - uint64_t - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. - - When the default type is used, the maximal integer number that can be - stored is `18446744073709551615` (UINT64_MAX) and the minimal integer - number that can be stored is `0`. Integer numbers that are out of range - will yield over/underflow when used in a constructor. During - deserialization, too large or small integer numbers will be automatically - be stored as @ref number_integer_t or @ref number_float_t. - - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. - - As this range is a subrange (when considered in conjunction with the - number_integer_t type) of the exactly supported range [0, UINT64_MAX], - this class's integer type is interoperable. - - #### Storage - - Integer number values are stored directly inside a @ref basic_json type. - - @sa @ref number_float_t -- type for number values (floating-point) - @sa @ref number_integer_t -- type for number values (integer) - - @since version 2.0.0 - */ - using number_unsigned_t = NumberUnsignedType; - - /*! - @brief a type for a number (floating-point) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store floating-point numbers in C++, a type is defined by the template - parameter @a NumberFloatType which chooses the type to use. - - #### Default type - - With the default values for @a NumberFloatType (`double`), the default - value for @a number_float_t is: - - @code {.cpp} - double - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in floating-point literals will be ignored. Internally, - the value will be stored as decimal number. For instance, the C++ - floating-point literal `01.2` will be serialized to `1.2`. During - deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > This specification allows implementations to set limits on the range and - > precision of numbers accepted. Since software that implements IEEE - > 754-2008 binary64 (double precision) numbers is generally available and - > widely used, good interoperability can be achieved by implementations - > that expect no more precision or range than these provide, in the sense - > that implementations will approximate JSON numbers within the expected - > precision. - - This implementation does exactly follow this approach, as it uses double - precision floating-point numbers. Note values smaller than - `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` - will be stored as NaN internally and be serialized to `null`. - - #### Storage - - Floating-point number values are stored directly inside a @ref basic_json - type. - - @sa @ref number_integer_t -- type for number values (integer) - - @sa @ref number_unsigned_t -- type for number values (unsigned integer) - - @since version 1.0.0 - */ - using number_float_t = NumberFloatType; - - /// @} - - - /////////////////////////// - // JSON type enumeration // - /////////////////////////// - - /*! - @brief the JSON type enumeration - - This enumeration collects the different JSON types. It is internally used - to distinguish the stored values, and the functions @ref is_null(), @ref - is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref - is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and - @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and - @ref is_structured() rely on it. - - @note There are three enumeration entries (number_integer, - number_unsigned, and number_float), because the library distinguishes - these three types for numbers: @ref number_unsigned_t is used for unsigned - integers, @ref number_integer_t is used for signed integers, and @ref - number_float_t is used for floating-point numbers or to approximate - integers which do not fit in the limits of their respective type. - - @sa @ref basic_json(const value_t value_type) -- create a JSON value with - the default value for a given type - - @since version 1.0.0 - */ - enum class value_t : uint8_t - { - null, ///< null value - object, ///< object (unordered set of name/value pairs) - array, ///< array (ordered collection of values) - string, ///< string value - boolean, ///< boolean value - number_integer, ///< number value (signed integer) - number_unsigned, ///< number value (unsigned integer) - number_float, ///< number value (floating-point) - discarded ///< discarded by the the parser callback function - }; - - - private: - - /// helper for exception-safe object creation - template - static T* create(Args&& ... args) - { - AllocatorType alloc; - auto deleter = [&](T * object) - { - alloc.deallocate(object, 1); - }; - std::unique_ptr object(alloc.allocate(1), deleter); - alloc.construct(object.get(), std::forward(args)...); - assert(object.get() != nullptr); - return object.release(); - } - - //////////////////////// - // JSON value storage // - //////////////////////// - - /*! - @brief a JSON value - - The actual storage for a JSON value of the @ref basic_json class. This - union combines the different storage types for the JSON value types - defined in @ref value_t. - - JSON type | value_t type | used type - --------- | --------------- | ------------------------ - object | object | pointer to @ref object_t - array | array | pointer to @ref array_t - string | string | pointer to @ref string_t - boolean | boolean | @ref boolean_t - number | number_integer | @ref number_integer_t - number | number_unsigned | @ref number_unsigned_t - number | number_float | @ref number_float_t - null | null | *no value is stored* - - @note Variable-length types (objects, arrays, and strings) are stored as - pointers. The size of the union should not exceed 64 bits if the default - value types are used. - - @since version 1.0.0 - */ - union json_value - { - /// object (stored with pointer to save storage) - object_t* object; - /// array (stored with pointer to save storage) - array_t* array; - /// string (stored with pointer to save storage) - string_t* string; - /// boolean - boolean_t boolean; - /// number (integer) - number_integer_t number_integer; - /// number (unsigned integer) - number_unsigned_t number_unsigned; - /// number (floating-point) - number_float_t number_float; - - /// default constructor (for null values) - json_value() = default; - /// constructor for booleans - json_value(boolean_t v) noexcept : boolean(v) {} - /// constructor for numbers (integer) - json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} - /// constructor for numbers (floating-point) - json_value(number_float_t v) noexcept : number_float(v) {} - /// constructor for empty values of a given type - json_value(value_t t) - { - switch (t) - { - case value_t::object: - { - object = create(); - break; - } - - case value_t::array: - { - array = create(); - break; - } - - case value_t::string: - { - string = create(""); - break; - } - - case value_t::boolean: - { - boolean = boolean_t(false); - break; - } - - case value_t::number_integer: - { - number_integer = number_integer_t(0); - break; - } - - case value_t::number_unsigned: - { - number_unsigned = number_unsigned_t(0); - break; - } - - case value_t::number_float: - { - number_float = number_float_t(0.0); - break; - } - - default: - { - break; - } - } - } - - /// constructor for strings - json_value(const string_t& value) - { - string = create(value); - } - - /// constructor for objects - json_value(const object_t& value) - { - object = create(value); - } - - /// constructor for arrays - json_value(const array_t& value) - { - array = create(value); - } - }; - - /*! - @brief checks the class invariants - - This function asserts the class invariants. It needs to be called at the - end of every constructor to make sure that created objects respect the - invariant. Furthermore, it has to be called each time the type of a JSON - value is changed, because the invariant expresses a relationship between - @a m_type and @a m_value. - */ - void assert_invariant() const - { - assert(m_type != value_t::object or m_value.object != nullptr); - assert(m_type != value_t::array or m_value.array != nullptr); - assert(m_type != value_t::string or m_value.string != nullptr); - } - - public: - ////////////////////////// - // JSON parser callback // - ////////////////////////// - - /*! - @brief JSON callback events - - This enumeration lists the parser events that can trigger calling a - callback function of type @ref parser_callback_t during parsing. - - @image html callback_events.png "Example when certain parse events are triggered" - - @since version 1.0.0 - */ - enum class parse_event_t : uint8_t - { - /// the parser read `{` and started to process a JSON object - object_start, - /// the parser read `}` and finished processing a JSON object - object_end, - /// the parser read `[` and started to process a JSON array - array_start, - /// the parser read `]` and finished processing a JSON array - array_end, - /// the parser read a key of a value in an object - key, - /// the parser finished reading a JSON value - value - }; - - /*! - @brief per-element parser callback type - - With a parser callback function, the result of parsing a JSON text can be - influenced. When passed to @ref parse(std::istream&, const - parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), - it is called on certain events (passed as @ref parse_event_t via parameter - @a event) with a set recursion depth @a depth and context JSON value - @a parsed. The return value of the callback function is a boolean - indicating whether the element that emitted the callback shall be kept or - not. - - We distinguish six scenarios (determined by the event type) in which the - callback function can be called. The following table describes the values - of the parameters @a depth, @a event, and @a parsed. - - parameter @a event | description | parameter @a depth | parameter @a parsed - ------------------ | ----------- | ------------------ | ------------------- - parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded - parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key - parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object - parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded - parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array - parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value - - @image html callback_events.png "Example when certain parse events are triggered" - - Discarding a value (i.e., returning `false`) has different effects - depending on the context in which function was called: - - - Discarded values in structured types are skipped. That is, the parser - will behave as if the discarded value was never read. - - In case a value outside a structured type is skipped, it is replaced - with `null`. This case happens if the top-level element is skipped. - - @param[in] depth the depth of the recursion during parsing - - @param[in] event an event of type parse_event_t indicating the context in - the callback function has been called - - @param[in,out] parsed the current intermediate parse result; note that - writing to this value has no effect for parse_event_t::key events - - @return Whether the JSON value which called the function during parsing - should be kept (`true`) or not (`false`). In the latter case, it is either - skipped completely or replaced by an empty discarded object. - - @sa @ref parse(std::istream&, parser_callback_t) or - @ref parse(const string_t&, parser_callback_t) for examples - - @since version 1.0.0 - */ - using parser_callback_t = std::function; - - - ////////////////// - // constructors // - ////////////////// - - /// @name constructors and destructors - /// Constructors of class @ref basic_json, copy/move constructor, copy - /// assignment, static functions creating objects, and the destructor. - /// @{ - - /*! - @brief create an empty value with a given type - - Create an empty JSON value with a given type. The value will be default - initialized with an empty value which depends on the type: - - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` - - @param[in] value_type the type of the value to create - - @complexity Constant. - - @throw std::bad_alloc if allocation for object, array, or string value - fails - - @liveexample{The following code shows the constructor for different @ref - value_t values,basic_json__value_t} - - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - @sa @ref basic_json(boolean_t value) -- create a boolean value - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const object_t&) -- create a object value - @sa @ref basic_json(const array_t&) -- create a array value - @sa @ref basic_json(const number_float_t) -- create a number - (floating-point) value - @sa @ref basic_json(const number_integer_t) -- create a number (integer) - value - @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) - value - - @since version 1.0.0 - */ - basic_json(const value_t value_type) - : m_type(value_type), m_value(value_type) - { - assert_invariant(); - } - - /*! - @brief create a null object (implicitly) - - Create a `null` JSON value. This is the implicit version of the `null` - value constructor as it takes no parameters. - - @note The class invariant is satisfied, because it poses no requirements - for null values. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - As postcondition, it holds: `basic_json().empty() == true`. - - @liveexample{The following code shows the constructor for a `null` JSON - value.,basic_json} - - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - - @since version 1.0.0 - */ - basic_json() = default; - - /*! - @brief create a null object (explicitly) - - Create a `null` JSON value. This is the explicitly version of the `null` - value constructor as it takes a null pointer as parameter. It allows to - create `null` values by explicitly assigning a `nullptr` to a JSON value. - The passed null pointer itself is not read -- it is only used to choose - the right constructor. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. - - @liveexample{The following code shows the constructor with null pointer - parameter.,basic_json__nullptr_t} - - @sa @ref basic_json() -- default constructor (implicitly creating a `null` - value) - - @since version 1.0.0 - */ - basic_json(std::nullptr_t) noexcept - : basic_json(value_t::null) - { - assert_invariant(); - } - - /*! - @brief create an object (explicit) - - Create an object JSON value with a given content. - - @param[in] val a value for the object - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for object value fails - - @liveexample{The following code shows the constructor with an @ref - object_t parameter.,basic_json__object_t} - - @sa @ref basic_json(const CompatibleObjectType&) -- create an object value - from a compatible STL container - - @since version 1.0.0 - */ - basic_json(const object_t& val) - : m_type(value_t::object), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an object (implicit) - - Create an object JSON value with a given content. This constructor allows - any type @a CompatibleObjectType that can be used to construct values of - type @ref object_t. - - @tparam CompatibleObjectType An object type whose `key_type` and - `value_type` is compatible to @ref object_t. Examples include `std::map`, - `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with - a `key_type` of `std::string`, and a `value_type` from which a @ref - basic_json value can be constructed. - - @param[in] val a value for the object - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for object value fails - - @liveexample{The following code shows the constructor with several - compatible object type parameters.,basic_json__CompatibleObjectType} - - @sa @ref basic_json(const object_t&) -- create an object value - - @since version 1.0.0 - */ - template ::value and - std::is_constructible::value, int>::type - = 0> - basic_json(const CompatibleObjectType& val) - : m_type(value_t::object) - { - using std::begin; - using std::end; - m_value.object = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create an array (explicit) - - Create an array JSON value with a given content. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with an @ref array_t - parameter.,basic_json__array_t} - - @sa @ref basic_json(const CompatibleArrayType&) -- create an array value - from a compatible STL containers - - @since version 1.0.0 - */ - basic_json(const array_t& val) - : m_type(value_t::array), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an array (implicit) - - Create an array JSON value with a given content. This constructor allows - any type @a CompatibleArrayType that can be used to construct values of - type @ref array_t. - - @tparam CompatibleArrayType An object type whose `value_type` is - compatible to @ref array_t. Examples include `std::vector`, `std::deque`, - `std::list`, `std::forward_list`, `std::array`, `std::set`, - `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a - `value_type` from which a @ref basic_json value can be constructed. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with several - compatible array type parameters.,basic_json__CompatibleArrayType} - - @sa @ref basic_json(const array_t&) -- create an array value - - @since version 1.0.0 - */ - template ::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type - = 0> - basic_json(const CompatibleArrayType& val) - : m_type(value_t::array) - { - using std::begin; - using std::end; - m_value.array = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create an string JSON value with a given content. - - @param[in] val a value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with an @ref - string_t parameter.,basic_json__string_t} - - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const string_t& val) - : m_type(value_t::string), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create a string JSON value with a given content. - - @param[in] val a literal value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with string literal - parameter.,basic_json__string_t_value_type} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const typename string_t::value_type* val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a string (implicit) - - Create a string JSON value with a given content. - - @param[in] val a value for the string - - @tparam CompatibleStringType an string type which is compatible to @ref - string_t, for instance `std::string`. - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the construction of a string value - from a compatible type.,basic_json__CompatibleStringType} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - - @since version 1.0.0 - */ - template ::value, int>::type - = 0> - basic_json(const CompatibleStringType& val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a boolean (explicit) - - Creates a JSON boolean type from a given value. - - @param[in] val a boolean value to store - - @complexity Constant. - - @liveexample{The example below demonstrates boolean - values.,basic_json__boolean_t} - - @since version 1.0.0 - */ - basic_json(boolean_t val) noexcept - : m_type(value_t::boolean), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number (explicit) - - Create an integer number JSON value with a given content. - - @tparam T A helper type to remove this function via SFINAE in case @ref - number_integer_t is the same as `int`. In this case, this constructor - would have the same signature as @ref basic_json(const int value). Note - the helper type @a T is not visible in this constructor's interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value.,basic_json__number_integer_t} - - @sa @ref basic_json(const int) -- create a number value (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - template::value) - and std::is_same::value - , int>::type - = 0> - basic_json(const number_integer_t val) noexcept - : m_type(value_t::number_integer), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number from an enum type (explicit) - - Create an integer number JSON value with a given content. - - @param[in] val an integer to create a JSON number from - - @note This constructor allows to pass enums directly to a constructor. As - C++ has no way of specifying the type of an anonymous enum explicitly, we - can only rely on the fact that such values implicitly convert to int. As - int may already be the same type of number_integer_t, we may need to - switch off the constructor @ref basic_json(const number_integer_t). - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value from an anonymous enum.,basic_json__const_int} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const int val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an integer number (implicit) - - Create an integer number JSON value with a given content. This constructor - allows any type @a CompatibleNumberIntegerType that can be used to - construct values of type @ref number_integer_t. - - @tparam CompatibleNumberIntegerType An integer type which is compatible to - @ref number_integer_t. Examples include the types `int`, `int32_t`, - `long`, and `short`. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of several integer - number values from compatible - types.,basic_json__CompatibleIntegerNumberType} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const int) -- create a number value (integer) - - @since version 1.0.0 - */ - template::value and - std::numeric_limits::is_integer and - std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type - = 0> - basic_json(const CompatibleNumberIntegerType val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an unsigned integer number (explicit) - - Create an unsigned integer number JSON value with a given content. - - @tparam T helper type to compare number_unsigned_t and unsigned int (not - visible in) the interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number - value (unsigned integer) from a compatible number type - - @since version 2.0.0 - */ - template::value) - and std::is_same::value - , int>::type - = 0> - basic_json(const number_unsigned_t val) noexcept - : m_type(value_t::number_unsigned), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an unsigned number (implicit) - - Create an unsigned number JSON value with a given content. This - constructor allows any type @a CompatibleNumberUnsignedType that can be - used to construct values of type @ref number_unsigned_t. - - @tparam CompatibleNumberUnsignedType An integer type which is compatible - to @ref number_unsigned_t. Examples may include the types `unsigned int`, - `uint32_t`, or `unsigned short`. - - @param[in] val an unsigned integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const number_unsigned_t) -- create a number value - (unsigned) - - @since version 2.0.0 - */ - template ::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type - = 0> - basic_json(const CompatibleNumberUnsignedType val) noexcept - : m_type(value_t::number_unsigned), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create a floating-point number (explicit) - - Create a floating-point number JSON value with a given content. - - @param[in] val a floating-point value to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is created - instead. - - @complexity Constant. - - @liveexample{The following example creates several floating-point - values.,basic_json__number_float_t} - - @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number - value (floating-point) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const number_float_t val) noexcept - : m_type(value_t::number_float), m_value(val) - { - // replace infinity and NAN by null - if (not std::isfinite(val)) - { - m_type = value_t::null; - m_value = json_value(); - } - - assert_invariant(); - } - - /*! - @brief create an floating-point number (implicit) - - Create an floating-point number JSON value with a given content. This - constructor allows any type @a CompatibleNumberFloatType that can be used - to construct values of type @ref number_float_t. - - @tparam CompatibleNumberFloatType A floating-point type which is - compatible to @ref number_float_t. Examples may include the types `float` - or `double`. - - @param[in] val a floating-point to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is - created instead. - - @complexity Constant. - - @liveexample{The example below shows the construction of several - floating-point number values from compatible - types.,basic_json__CompatibleNumberFloatType} - - @sa @ref basic_json(const number_float_t) -- create a number value - (floating-point) - - @since version 1.0.0 - */ - template::value and - std::is_floating_point::value>::type - > - basic_json(const CompatibleNumberFloatType val) noexcept - : basic_json(number_float_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a container (array or object) from an initializer list - - Creates a JSON value of type array or object from the passed initializer - list @a init. In case @a type_deduction is `true` (default), the type of - the JSON value to be created is deducted from the initializer list @a init - according to the following rules: - - 1. If the list is empty, an empty JSON object value `{}` is created. - 2. If the list consists of pairs whose first element is a string, a JSON - object value is created where the first elements of the pairs are - treated as keys and the second elements are as values. - 3. In all other cases, an array is created. - - The rules aim to create the best fit between a C++ initializer list and - JSON values. The rationale is as follows: - - 1. The empty initializer list is written as `{}` which is exactly an empty - JSON object. - 2. C++ has now way of describing mapped types other than to list a list of - pairs. As JSON requires that keys must be of type string, rule 2 is the - weakest constraint one can pose on initializer lists to interpret them - as an object. - 3. In all other cases, the initializer list could not be interpreted as - JSON object type, so interpreting it as JSON array type is safe. - - With the rules described above, the following JSON values cannot be - expressed by an initializer list: - - - the empty array (`[]`): use @ref array(std::initializer_list) - with an empty initializer list in this case - - arrays whose elements satisfy rule 2: use @ref - array(std::initializer_list) with the same initializer list - in this case - - @note When used without parentheses around an empty initializer list, @ref - basic_json() is called instead of this function, yielding the JSON null - value. - - @param[in] init initializer list with JSON values - - @param[in] type_deduction internal parameter; when set to `true`, the type - of the JSON value is deducted from the initializer list @a init; when set - to `false`, the type provided via @a manual_type is forced. This mode is - used by the functions @ref array(std::initializer_list) and - @ref object(std::initializer_list). - - @param[in] manual_type internal parameter; when @a type_deduction is set - to `false`, the created JSON value will use the provided type (only @ref - value_t::array and @ref value_t::object are valid); when @a type_deduction - is set to `true`, this parameter has no effect - - @throw std::domain_error if @a type_deduction is `false`, @a manual_type - is `value_t::object`, but @a init contains an element which is not a pair - whose first element is a string; example: `"cannot create object from - initializer list"` - - @complexity Linear in the size of the initializer list @a init. - - @liveexample{The example below shows how JSON values are created from - initializer lists.,basic_json__list_init_t} - - @sa @ref array(std::initializer_list) -- create a JSON array - value from an initializer list - @sa @ref object(std::initializer_list) -- create a JSON object - value from an initializer list - - @since version 1.0.0 - */ - basic_json(std::initializer_list init, - bool type_deduction = true, - value_t manual_type = value_t::array) - { - // check if each element is an array with two elements whose first - // element is a string - bool is_an_object = std::all_of(init.begin(), init.end(), - [](const basic_json & element) - { - return element.is_array() and element.size() == 2 and element[0].is_string(); - }); - - // adjust type if type deduction is not wanted - if (not type_deduction) - { - // if array is wanted, do not create an object though possible - if (manual_type == value_t::array) - { - is_an_object = false; - } - - // if object is wanted but impossible, throw an exception - if (manual_type == value_t::object and not is_an_object) - { - throw std::domain_error("cannot create object from initializer list"); - } - } - - if (is_an_object) - { - // the initializer list is a list of pairs -> create object - m_type = value_t::object; - m_value = value_t::object; - - std::for_each(init.begin(), init.end(), [this](const basic_json & element) - { - m_value.object->emplace(*(element[0].m_value.string), element[1]); - }); - } - else - { - // the initializer list describes an array -> create array - m_type = value_t::array; - m_value.array = create(init); - } - - assert_invariant(); - } - - /*! - @brief explicitly create an array from an initializer list - - Creates a JSON array value from a given initializer list. That is, given a - list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the - initializer list is empty, the empty array `[]` is created. - - @note This function is only needed to express two edge cases that cannot - be realized with the initializer list constructor (@ref - basic_json(std::initializer_list, bool, value_t)). These cases - are: - 1. creating an array whose elements are all pairs whose first element is a - string -- in this case, the initializer list constructor would create an - object, taking the first elements as keys - 2. creating an empty array -- passing the empty initializer list to the - initializer list constructor yields an empty object - - @param[in] init initializer list with JSON values to create an array from - (optional) - - @return JSON array value - - @complexity Linear in the size of @a init. - - @liveexample{The following code shows an example for the `array` - function.,array} - - @sa @ref basic_json(std::initializer_list, bool, value_t) -- - create a JSON value from an initializer list - @sa @ref object(std::initializer_list) -- create a JSON object - value from an initializer list - - @since version 1.0.0 - */ - static basic_json array(std::initializer_list init = - std::initializer_list()) - { - return basic_json(init, false, value_t::array); - } - - /*! - @brief explicitly create an object from an initializer list - - Creates a JSON object value from a given initializer list. The initializer - lists elements must be pairs, and their first elements must be strings. If - the initializer list is empty, the empty object `{}` is created. - - @note This function is only added for symmetry reasons. In contrast to the - related function @ref array(std::initializer_list), there are - no cases which can only be expressed by this function. That is, any - initializer list @a init can also be passed to the initializer list - constructor @ref basic_json(std::initializer_list, bool, - value_t). - - @param[in] init initializer list to create an object from (optional) - - @return JSON object value - - @throw std::domain_error if @a init is not a pair whose first elements are - strings; thrown by - @ref basic_json(std::initializer_list, bool, value_t) - - @complexity Linear in the size of @a init. - - @liveexample{The following code shows an example for the `object` - function.,object} - - @sa @ref basic_json(std::initializer_list, bool, value_t) -- - create a JSON value from an initializer list - @sa @ref array(std::initializer_list) -- create a JSON array - value from an initializer list - - @since version 1.0.0 - */ - static basic_json object(std::initializer_list init = - std::initializer_list()) - { - return basic_json(init, false, value_t::object); - } - - /*! - @brief construct an array with count copies of given value - - Constructs a JSON array value by creating @a cnt copies of a passed value. - In case @a cnt is `0`, an empty array is created. As postcondition, - `std::distance(begin(),end()) == cnt` holds. - - @param[in] cnt the number of JSON copies of @a val to create - @param[in] val the JSON value to copy - - @complexity Linear in @a cnt. - - @liveexample{The following code shows examples for the @ref - basic_json(size_type\, const basic_json&) - constructor.,basic_json__size_type_basic_json} - - @since version 1.0.0 - */ - basic_json(size_type cnt, const basic_json& val) - : m_type(value_t::array) - { - m_value.array = create(cnt, val); - assert_invariant(); - } - - /*! - @brief construct a JSON container given an iterator range - - Constructs the JSON value with the contents of the range `[first, last)`. - The semantics depends on the different types a JSON value can have: - - In case of primitive types (number, boolean, or string), @a first must - be `begin()` and @a last must be `end()`. In this case, the value is - copied. Otherwise, std::out_of_range is thrown. - - In case of structured types (array, object), the constructor behaves as - similar versions for `std::vector`. - - In case of a null type, std::domain_error is thrown. - - @tparam InputIT an input iterator type (@ref iterator or @ref - const_iterator) - - @param[in] first begin of the range to copy from (included) - @param[in] last end of the range to copy from (excluded) - - @pre Iterators @a first and @a last must be initialized. - - @throw std::domain_error if iterators are not compatible; that is, do not - belong to the same JSON value; example: `"iterators are not compatible"` - @throw std::out_of_range if iterators are for a primitive type (number, - boolean, or string) where an out of range error can be detected easily; - example: `"iterators out of range"` - @throw std::bad_alloc if allocation for object, array, or string fails - @throw std::domain_error if called with a null value; example: `"cannot - use construct with iterators from null"` - - @complexity Linear in distance between @a first and @a last. - - @liveexample{The example below shows several ways to create JSON values by - specifying a subrange with iterators.,basic_json__InputIt_InputIt} - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - basic_json(InputIT first, InputIT last) - { - assert(first.m_object != nullptr); - assert(last.m_object != nullptr); - - // make sure iterator fits the current value - if (first.m_object != last.m_object) - { - throw std::domain_error("iterators are not compatible"); - } - - // copy type from first iterator - m_type = first.m_object->m_type; - - // check if iterator range is complete for primitive values - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) - { - throw std::out_of_range("iterators out of range"); - } - break; - } - - default: - { - break; - } - } - - switch (m_type) - { - case value_t::number_integer: - { - m_value.number_integer = first.m_object->m_value.number_integer; - break; - } - - case value_t::number_unsigned: - { - m_value.number_unsigned = first.m_object->m_value.number_unsigned; - break; - } - - case value_t::number_float: - { - m_value.number_float = first.m_object->m_value.number_float; - break; - } - - case value_t::boolean: - { - m_value.boolean = first.m_object->m_value.boolean; - break; - } - - case value_t::string: - { - m_value = *first.m_object->m_value.string; - break; - } - - case value_t::object: - { - m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); - break; - } - - case value_t::array: - { - m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); - } - } - - assert_invariant(); - } - - /*! - @brief construct a JSON value given an input stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates constructing a JSON value from - a `std::stringstream` with and without callback - function.,basic_json__istream} - - @since version 2.0.0 - */ - explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) - { - *this = parser(i, cb).parse(); - assert_invariant(); - } - - /////////////////////////////////////// - // other constructors and destructor // - /////////////////////////////////////// - - /*! - @brief copy constructor - - Creates a copy of a given JSON value. - - @param[in] other the JSON value to copy - - @complexity Linear in the size of @a other. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - As postcondition, it holds: `other == basic_json(other)`. - - @throw std::bad_alloc if allocation for object, array, or string fails. - - @liveexample{The following code shows an example for the copy - constructor.,basic_json__basic_json} - - @since version 1.0.0 - */ - basic_json(const basic_json& other) - : m_type(other.m_type) - { - // check of passed value is valid - other.assert_invariant(); - - switch (m_type) - { - case value_t::object: - { - m_value = *other.m_value.object; - break; - } - - case value_t::array: - { - m_value = *other.m_value.array; - break; - } - - case value_t::string: - { - m_value = *other.m_value.string; - break; - } - - case value_t::boolean: - { - m_value = other.m_value.boolean; - break; - } - - case value_t::number_integer: - { - m_value = other.m_value.number_integer; - break; - } - - case value_t::number_unsigned: - { - m_value = other.m_value.number_unsigned; - break; - } - - case value_t::number_float: - { - m_value = other.m_value.number_float; - break; - } - - default: - { - break; - } - } - - assert_invariant(); - } - - /*! - @brief move constructor - - Move constructor. Constructs a JSON value with the contents of the given - value @a other using move semantics. It "steals" the resources from @a - other and leaves it as JSON null value. - - @param[in,out] other value to move to this object - - @post @a other is a JSON null value - - @complexity Constant. - - @liveexample{The code below shows the move constructor explicitly called - via std::move.,basic_json__moveconstructor} - - @since version 1.0.0 - */ - basic_json(basic_json&& other) noexcept - : m_type(std::move(other.m_type)), - m_value(std::move(other.m_value)) - { - // check that passed value is valid - other.assert_invariant(); - - // invalidate payload - other.m_type = value_t::null; - other.m_value = {}; - - assert_invariant(); - } - - /*! - @brief copy assignment - - Copy assignment operator. Copies a JSON value via the "copy and swap" - strategy: It is expressed in terms of the copy constructor, destructor, - and the swap() member function. - - @param[in] other value to copy from - - @complexity Linear. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - @liveexample{The code below shows and example for the copy assignment. It - creates a copy of value `a` which is then swapped with `b`. Finally\, the - copy of `a` (which is the null value after the swap) is - destroyed.,basic_json__copyassignment} - - @since version 1.0.0 - */ - reference& operator=(basic_json other) noexcept ( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - // check that passed value is valid - other.assert_invariant(); - - using std::swap; - swap(m_type, other.m_type); - swap(m_value, other.m_value); - - assert_invariant(); - return *this; - } - - /*! - @brief destructor - - Destroys the JSON value and frees all allocated memory. - - @complexity Linear. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - All stored elements are destroyed and all memory is freed. - - @since version 1.0.0 - */ - ~basic_json() - { - assert_invariant(); - - switch (m_type) - { - case value_t::object: - { - AllocatorType alloc; - alloc.destroy(m_value.object); - alloc.deallocate(m_value.object, 1); - break; - } - - case value_t::array: - { - AllocatorType alloc; - alloc.destroy(m_value.array); - alloc.deallocate(m_value.array, 1); - break; - } - - case value_t::string: - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - break; - } - - default: - { - // all other types need no specific destructor - break; - } - } - } - - /// @} - - public: - /////////////////////// - // object inspection // - /////////////////////// - - /// @name object inspection - /// Functions to inspect the type of a JSON value. - /// @{ - - /*! - @brief serialization - - Serialization function for JSON values. The function tries to mimic - Python's `json.dumps()` function, and currently supports its @a indent - parameter. - - @param[in] indent If indent is nonnegative, then array elements and object - members will be pretty-printed with that indent level. An indent level of - `0` will only insert newlines. `-1` (the default) selects the most compact - representation. - - @return string containing the serialization of the JSON value - - @complexity Linear. - - @liveexample{The following example shows the effect of different @a indent - parameters to the result of the serialization.,dump} - - @see https://docs.python.org/2/library/json.html#json.dump - - @since version 1.0.0 - */ - string_t dump(const int indent = -1) const - { - std::stringstream ss; - // fix locale problems - ss.imbue(std::locale(std::locale(), new DecimalSeparator)); - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - ss.precision(std::numeric_limits::digits10); - - if (indent >= 0) - { - dump(ss, true, static_cast(indent)); - } - else - { - dump(ss, false, 0); - } - - return ss.str(); - } - - /*! - @brief return the type of the JSON value (explicit) - - Return the type of the JSON value as a value from the @ref value_t - enumeration. - - @return the type of the JSON value - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `type()` for all JSON - types.,type} - - @since version 1.0.0 - */ - constexpr value_t type() const noexcept - { - return m_type; - } - - /*! - @brief return whether type is primitive - - This function returns true iff the JSON type is primitive (string, number, - boolean, or null). - - @return `true` if type is primitive (string, number, boolean, or null), - `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_primitive()` for all JSON - types.,is_primitive} - - @sa @ref is_structured() -- returns whether JSON value is structured - @sa @ref is_null() -- returns whether JSON value is `null` - @sa @ref is_string() -- returns whether JSON value is a string - @sa @ref is_boolean() -- returns whether JSON value is a boolean - @sa @ref is_number() -- returns whether JSON value is a number - - @since version 1.0.0 - */ - constexpr bool is_primitive() const noexcept - { - return is_null() or is_string() or is_boolean() or is_number(); - } - - /*! - @brief return whether type is structured - - This function returns true iff the JSON type is structured (array or - object). - - @return `true` if type is structured (array or object), `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_structured()` for all JSON - types.,is_structured} - - @sa @ref is_primitive() -- returns whether value is primitive - @sa @ref is_array() -- returns whether value is an array - @sa @ref is_object() -- returns whether value is an object - - @since version 1.0.0 - */ - constexpr bool is_structured() const noexcept - { - return is_array() or is_object(); - } - - /*! - @brief return whether value is null - - This function returns true iff the JSON value is null. - - @return `true` if type is null, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_null()` for all JSON - types.,is_null} - - @since version 1.0.0 - */ - constexpr bool is_null() const noexcept - { - return m_type == value_t::null; - } - - /*! - @brief return whether value is a boolean - - This function returns true iff the JSON value is a boolean. - - @return `true` if type is boolean, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_boolean()` for all JSON - types.,is_boolean} - - @since version 1.0.0 - */ - constexpr bool is_boolean() const noexcept - { - return m_type == value_t::boolean; - } - - /*! - @brief return whether value is a number - - This function returns true iff the JSON value is a number. This includes - both integer and floating-point values. - - @return `true` if type is number (regardless whether integer, unsigned - integer or floating-type), `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number()` for all JSON - types.,is_number} - - @sa @ref is_number_integer() -- check if value is an integer or unsigned - integer number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 1.0.0 - */ - constexpr bool is_number() const noexcept - { - return is_number_integer() or is_number_float(); - } - - /*! - @brief return whether value is an integer number - - This function returns true iff the JSON value is an integer or unsigned - integer number. This excludes floating-point values. - - @return `true` if type is an integer or unsigned integer number, `false` - otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_integer()` for all - JSON types.,is_number_integer} - - @sa @ref is_number() -- check if value is a number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 1.0.0 - */ - constexpr bool is_number_integer() const noexcept - { - return m_type == value_t::number_integer or m_type == value_t::number_unsigned; - } - - /*! - @brief return whether value is an unsigned integer number - - This function returns true iff the JSON value is an unsigned integer - number. This excludes floating-point and (signed) integer values. - - @return `true` if type is an unsigned integer number, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_unsigned()` for all - JSON types.,is_number_unsigned} - - @sa @ref is_number() -- check if value is a number - @sa @ref is_number_integer() -- check if value is an integer or unsigned - integer number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 2.0.0 - */ - constexpr bool is_number_unsigned() const noexcept - { - return m_type == value_t::number_unsigned; - } - - /*! - @brief return whether value is a floating-point number - - This function returns true iff the JSON value is a floating-point number. - This excludes integer and unsigned integer values. - - @return `true` if type is a floating-point number, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_float()` for all - JSON types.,is_number_float} - - @sa @ref is_number() -- check if value is number - @sa @ref is_number_integer() -- check if value is an integer number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - - @since version 1.0.0 - */ - constexpr bool is_number_float() const noexcept - { - return m_type == value_t::number_float; - } - - /*! - @brief return whether value is an object - - This function returns true iff the JSON value is an object. - - @return `true` if type is object, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_object()` for all JSON - types.,is_object} - - @since version 1.0.0 - */ - constexpr bool is_object() const noexcept - { - return m_type == value_t::object; - } - - /*! - @brief return whether value is an array - - This function returns true iff the JSON value is an array. - - @return `true` if type is array, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_array()` for all JSON - types.,is_array} - - @since version 1.0.0 - */ - constexpr bool is_array() const noexcept - { - return m_type == value_t::array; - } - - /*! - @brief return whether value is a string - - This function returns true iff the JSON value is a string. - - @return `true` if type is string, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_string()` for all JSON - types.,is_string} - - @since version 1.0.0 - */ - constexpr bool is_string() const noexcept - { - return m_type == value_t::string; - } - - /*! - @brief return whether value is discarded - - This function returns true iff the JSON value was discarded during parsing - with a callback function (see @ref parser_callback_t). - - @note This function will always be `false` for JSON values after parsing. - That is, discarded values can only occur during parsing, but will be - removed when inside a structured value or replaced by null in other cases. - - @return `true` if type is discarded, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_discarded()` for all JSON - types.,is_discarded} - - @since version 1.0.0 - */ - constexpr bool is_discarded() const noexcept - { - return m_type == value_t::discarded; - } - - /*! - @brief return the type of the JSON value (implicit) - - Implicitly return the type of the JSON value as a value from the @ref - value_t enumeration. - - @return the type of the JSON value - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies the @ref value_t operator for - all JSON types.,operator__value_t} - - @since version 1.0.0 - */ - constexpr operator value_t() const noexcept - { - return m_type; - } - - /// @} - - private: - ////////////////// - // value access // - ////////////////// - - /// get an object (explicit) - template ::value and - std::is_convertible::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_object()) - { - return T(m_value.object->begin(), m_value.object->end()); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an object (explicit) - object_t get_impl(object_t*) const - { - if (is_object()) - { - return *(m_value.object); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - T to_vector; - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not std::is_same::value - , int>::type = 0> - std::vector get_impl(std::vector*) const - { - if (is_array()) - { - std::vector to_vector; - to_vector.reserve(m_value.array->size()); - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not has_mapped_type::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - return T(m_value.array->begin(), m_value.array->end()); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - array_t get_impl(array_t*) const - { - if (is_array()) - { - return *(m_value.array); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get a string (explicit) - template ::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_string()) - { - return *m_value.string; - } - else - { - throw std::domain_error("type must be string, but is " + type_name()); - } - } - - /// get a number (explicit) - template::value - , int>::type = 0> - T get_impl(T*) const - { - switch (m_type) - { - case value_t::number_integer: - { - return static_cast(m_value.number_integer); - } - - case value_t::number_unsigned: - { - return static_cast(m_value.number_unsigned); - } - - case value_t::number_float: - { - return static_cast(m_value.number_float); - } - - default: - { - throw std::domain_error("type must be number, but is " + type_name()); - } - } - } - - /// get a boolean (explicit) - constexpr boolean_t get_impl(boolean_t*) const - { - return is_boolean() - ? m_value.boolean - : throw std::domain_error("type must be boolean, but is " + type_name()); - } - - /// get a pointer to the value (object) - object_t* get_impl_ptr(object_t*) noexcept - { - return is_object() ? m_value.object : nullptr; - } - - /// get a pointer to the value (object) - constexpr const object_t* get_impl_ptr(const object_t*) const noexcept - { - return is_object() ? m_value.object : nullptr; - } - - /// get a pointer to the value (array) - array_t* get_impl_ptr(array_t*) noexcept - { - return is_array() ? m_value.array : nullptr; - } - - /// get a pointer to the value (array) - constexpr const array_t* get_impl_ptr(const array_t*) const noexcept - { - return is_array() ? m_value.array : nullptr; - } - - /// get a pointer to the value (string) - string_t* get_impl_ptr(string_t*) noexcept - { - return is_string() ? m_value.string : nullptr; - } - - /// get a pointer to the value (string) - constexpr const string_t* get_impl_ptr(const string_t*) const noexcept - { - return is_string() ? m_value.string : nullptr; - } - - /// get a pointer to the value (boolean) - boolean_t* get_impl_ptr(boolean_t*) noexcept - { - return is_boolean() ? &m_value.boolean : nullptr; - } - - /// get a pointer to the value (boolean) - constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept - { - return is_boolean() ? &m_value.boolean : nullptr; - } - - /// get a pointer to the value (integer number) - number_integer_t* get_impl_ptr(number_integer_t*) noexcept - { - return is_number_integer() ? &m_value.number_integer : nullptr; - } - - /// get a pointer to the value (integer number) - constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept - { - return is_number_integer() ? &m_value.number_integer : nullptr; - } - - /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept - { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; - } - - /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept - { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; - } - - /// get a pointer to the value (floating-point number) - number_float_t* get_impl_ptr(number_float_t*) noexcept - { - return is_number_float() ? &m_value.number_float : nullptr; - } - - /// get a pointer to the value (floating-point number) - constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept - { - return is_number_float() ? &m_value.number_float : nullptr; - } - - /*! - @brief helper function to implement get_ref() - - This funcion helps to implement get_ref() without code duplication for - const and non-const overloads - - @tparam ThisType will be deduced as `basic_json` or `const basic_json` - - @throw std::domain_error if ReferenceType does not match underlying value - type of the current JSON - */ - template - static ReferenceType get_ref_impl(ThisType& obj) - { - // helper type - using PointerType = typename std::add_pointer::type; - - // delegate the call to get_ptr<>() - auto ptr = obj.template get_ptr(); - - if (ptr != nullptr) - { - return *ptr; - } - else - { - throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + - obj.type_name()); - } - } - - public: - - /// @name value access - /// Direct access to the stored value of a JSON value. - /// @{ - - /*! - @brief get a value (explicit) - - Explicit type conversion between the JSON value and a compatible value. - - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays - - @return copy of the JSON value, converted to type @a ValueType - - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON; example: `"type must be object, but is null"` - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows several conversions from JSON values - to other types. There a few things to note: (1) Floating-point numbers can - be converted to integers\, (2) A JSON array can be converted to a standard - `std::vector`\, (3) A JSON object can be converted to C++ - associative containers such as `std::unordered_map`.,get__ValueType_const} - - @internal - The idea of using a casted null pointer to choose the correct - implementation is from . - @endinternal - - @sa @ref operator ValueType() const for implicit conversion - @sa @ref get() for pointer-member access - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - ValueType get() const - { - return get_impl(static_cast(nullptr)); - } - - /*! - @brief get a pointer value (explicit) - - Explicit pointer access to the internally stored JSON value. No copies are - made. - - @warning The pointer becomes invalid if the underlying JSON object - changes. - - @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref - object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. - - @return pointer to the internally stored JSON value if the requested - pointer type @a PointerType fits to the JSON value; `nullptr` otherwise - - @complexity Constant. - - @liveexample{The example below shows how pointers to internal values of a - JSON value can be requested. Note that no type conversions are made and a - `nullptr` is returned if the value and the requested pointer type does not - match.,get__PointerType} - - @sa @ref get_ptr() for explicit pointer-member access - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - PointerType get() noexcept - { - // delegate the call to get_ptr - return get_ptr(); - } - - /*! - @brief get a pointer value (explicit) - @copydoc get() - */ - template::value - , int>::type = 0> - constexpr const PointerType get() const noexcept - { - // delegate the call to get_ptr - return get_ptr(); - } - - /*! - @brief get a pointer value (implicit) - - Implicit pointer access to the internally stored JSON value. No copies are - made. - - @warning Writing data to the pointee of the result yields an undefined - state. - - @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref - object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. Enforced by a static - assertion. - - @return pointer to the internally stored JSON value if the requested - pointer type @a PointerType fits to the JSON value; `nullptr` otherwise - - @complexity Constant. - - @liveexample{The example below shows how pointers to internal values of a - JSON value can be requested. Note that no type conversions are made and a - `nullptr` is returned if the value and the requested pointer type does not - match.,get_ptr} - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - PointerType get_ptr() noexcept - { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() - return get_impl_ptr(static_cast(nullptr)); - } - - /*! - @brief get a pointer value (implicit) - @copydoc get_ptr() - */ - template::value - and std::is_const::type>::value - , int>::type = 0> - constexpr const PointerType get_ptr() const noexcept - { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() const - return get_impl_ptr(static_cast(nullptr)); - } - - /*! - @brief get a reference value (implicit) - - Implict reference access to the internally stored JSON value. No copies - are made. - - @warning Writing data to the referee of the result yields an undefined - state. - - @tparam ReferenceType reference type; must be a reference to @ref array_t, - @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or - @ref number_float_t. Enforced by static assertion. - - @return reference to the internally stored JSON value if the requested - reference type @a ReferenceType fits to the JSON value; throws - std::domain_error otherwise - - @throw std::domain_error in case passed type @a ReferenceType is - incompatible with the stored JSON value - - @complexity Constant. - - @liveexample{The example shows several calls to `get_ref()`.,get_ref} - - @since version 1.1.0 - */ - template::value - , int>::type = 0> - ReferenceType get_ref() - { - // delegate call to get_ref_impl - return get_ref_impl(*this); - } - - /*! - @brief get a reference value (implicit) - @copydoc get_ref() - */ - template::value - and std::is_const::type>::value - , int>::type = 0> - ReferenceType get_ref() const - { - // delegate call to get_ref_impl - return get_ref_impl(*this); - } - - /*! - @brief get a value (implicit) - - Implicit type conversion between the JSON value and a compatible value. - The call is realized by calling @ref get() const. - - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays. The character type of @ref string_t - as well as an initializer list of this type is excluded to avoid - ambiguities as these types implicitly convert to `std::string`. - - @return copy of the JSON value, converted to type @a ValueType - - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON, thrown by @ref get() const - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows several conversions from JSON values - to other types. There a few things to note: (1) Floating-point numbers can - be converted to integers\, (2) A JSON array can be converted to a standard - `std::vector`\, (3) A JSON object can be converted to C++ - associative containers such as `std::unordered_map`.,operator__ValueType} - - @since version 1.0.0 - */ - template < typename ValueType, typename - std::enable_if < - not std::is_pointer::value - and not std::is_same::value -#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 - and not std::is_same>::value -#endif - , int >::type = 0 > - operator ValueType() const - { - // delegate the call to get<>() const - return get(); - } - - /// @} - - - //////////////////// - // element access // - //////////////////// - - /// @name element access - /// Access to the JSON value. - /// @{ - - /*! - @brief access specified array element with bounds checking - - Returns a reference to the element at specified location @a idx, with - bounds checking. - - @param[in] idx index of the element to access - - @return reference to the element at index @a idx - - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read and - written using `at()`.,at__size_type} - - @since version 1.0.0 - */ - reference at(size_type idx) - { - // at only works for arrays - if (is_array()) - { - try - { - return m_value.array->at(idx); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified array element with bounds checking - - Returns a const reference to the element at specified location @a idx, - with bounds checking. - - @param[in] idx index of the element to access - - @return const reference to the element at index @a idx - - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read using - `at()`.,at__size_type_const} - - @since version 1.0.0 - */ - const_reference at(size_type idx) const - { - // at only works for arrays - if (is_array()) - { - try - { - return m_value.array->at(idx); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified object element with bounds checking - - Returns a reference to the element at with specified key @a key, with - bounds checking. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using `at()`.,at__object_t_key_type} - - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - reference at(const typename object_t::key_type& key) - { - // at only works for objects - if (is_object()) - { - try - { - return m_value.object->at(key); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified object element with bounds checking - - Returns a const reference to the element at with specified key @a key, - with bounds checking. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - `at()`.,at__object_t_key_type_const} - - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - const_reference at(const typename object_t::key_type& key) const - { - // at only works for objects - if (is_object()) - { - try - { - return m_value.object->at(key); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified array element - - Returns a reference to the element at specified location @a idx. - - @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), - then the array is silently filled up with `null` values to make `idx` a - valid reference to the last stored element. - - @param[in] idx index of the element to access - - @return reference to the element at index @a idx - - @throw std::domain_error if JSON is not an array or null; example: - `"cannot use operator[] with string"` - - @complexity Constant if @a idx is in the range of the array. Otherwise - linear in `idx - size()`. - - @liveexample{The example below shows how array elements can be read and - written using `[]` operator. Note the addition of `null` - values.,operatorarray__size_type} - - @since version 1.0.0 - */ - reference operator[](size_type idx) - { - // implicitly convert null value to an empty array - if (is_null()) - { - m_type = value_t::array; - m_value.array = create(); - assert_invariant(); - } - - // operator[] only works for arrays - if (is_array()) - { - // fill up array with null values if given idx is outside range - if (idx >= m_value.array->size()) - { - m_value.array->insert(m_value.array->end(), - idx - m_value.array->size() + 1, - basic_json()); - } - - return m_value.array->operator[](idx); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified array element - - Returns a const reference to the element at specified location @a idx. - - @param[in] idx index of the element to access - - @return const reference to the element at index @a idx - - @throw std::domain_error if JSON is not an array; example: `"cannot use - operator[] with null"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read using - the `[]` operator.,operatorarray__size_type_const} - - @since version 1.0.0 - */ - const_reference operator[](size_type idx) const - { - // const operator[] only works for arrays - if (is_array()) - { - return m_value.array->operator[](idx); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - reference operator[](const typename object_t::key_type& key) - { - // implicitly convert null value to an empty object - if (is_null()) - { - m_type = value_t::object; - m_value.object = create(); - assert_invariant(); - } - - // operator[] only works for objects - if (is_object()) - { - return m_value.object->operator[](key); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - const_reference operator[](const typename object_t::key_type& key) const - { - // const operator[] only works for objects - if (is_object()) - { - assert(m_value.object->find(key) != m_value.object->end()); - return m_value.object->find(key)->second; - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - template - reference operator[](T * (&key)[n]) - { - return operator[](static_cast(key)); - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @note This function is required for compatibility reasons with Clang. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - template - const_reference operator[](T * (&key)[n]) const - { - return operator[](static_cast(key)); - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.1.0 - */ - template - reference operator[](T* key) - { - // implicitly convert null to object - if (is_null()) - { - m_type = value_t::object; - m_value = value_t::object; - assert_invariant(); - } - - // at only works for objects - if (is_object()) - { - return m_value.object->operator[](key); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.1.0 - */ - template - const_reference operator[](T* key) const - { - // at only works for objects - if (is_object()) - { - assert(m_value.object->find(key) != m_value.object->end()); - return m_value.object->find(key)->second; - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element with default value - - Returns either a copy of an object's element at the specified key @a key - or a given default value if no element with key @a key exists. - - The function is basically equivalent to executing - @code {.cpp} - try { - return at(key); - } catch(std::out_of_range) { - return default_value; - } - @endcode - - @note Unlike @ref at(const typename object_t::key_type&), this function - does not throw if the given key @a key was not found. - - @note Unlike @ref operator[](const typename object_t::key_type& key), this - function does not implicitly add an element to the position defined by @a - key. This function is furthermore also applicable to const objects. - - @param[in] key key of the element to access - @param[in] default_value the value to return if @a key is not found - - @tparam ValueType type compatible to JSON values, for instance `int` for - JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for - JSON arrays. Note the type of the expected value at @a key and the default - value @a default_value must be compatible. - - @return copy of the element at key @a key or @a default_value if @a key - is not found - - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be queried - with a default value.,basic_json__value} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - - @since version 1.0.0 - */ - template ::value - , int>::type = 0> - ValueType value(const typename object_t::key_type& key, ValueType default_value) const - { - // at only works for objects - if (is_object()) - { - // if key is found, return value and given default value otherwise - const auto it = find(key); - if (it != end()) - { - return *it; - } - else - { - return default_value; - } - } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } - } - - /*! - @brief overload for a default value of type const char* - @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const - */ - string_t value(const typename object_t::key_type& key, const char* default_value) const - { - return value(key, string_t(default_value)); - } - - /*! - @brief access specified object element via JSON Pointer with default value - - Returns either a copy of an object's element at the specified key @a key - or a given default value if no element with key @a key exists. - - The function is basically equivalent to executing - @code {.cpp} - try { - return at(ptr); - } catch(std::out_of_range) { - return default_value; - } - @endcode - - @note Unlike @ref at(const json_pointer&), this function does not throw - if the given key @a key was not found. - - @param[in] ptr a JSON pointer to the element to access - @param[in] default_value the value to return if @a ptr found no value - - @tparam ValueType type compatible to JSON values, for instance `int` for - JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for - JSON arrays. Note the type of the expected value at @a key and the default - value @a default_value must be compatible. - - @return copy of the element at key @a key or @a default_value if @a key - is not found - - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be queried - with a default value.,basic_json__value_ptr} - - @sa @ref operator[](const json_pointer&) for unchecked access by reference - - @since version 2.0.2 - */ - template ::value - , int>::type = 0> - ValueType value(const json_pointer& ptr, ValueType default_value) const - { - // at only works for objects - if (is_object()) - { - // if pointer resolves a value, return it or use default value - try - { - return ptr.get_checked(this); - } - catch (std::out_of_range&) - { - return default_value; - } - } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } - } - - /*! - @brief overload for a default value of type const char* - @copydoc basic_json::value(const json_pointer&, ValueType) const - */ - string_t value(const json_pointer& ptr, const char* default_value) const - { - return value(ptr, string_t(default_value)); - } - - /*! - @brief access the first element - - Returns a reference to the first element in the container. For a JSON - container `c`, the expression `c.front()` is equivalent to `*c.begin()`. - - @return In case of a structured type (array or object), a reference to the - first element is returned. In cast of number, string, or boolean values, a - reference to the value is returned. - - @complexity Constant. - - @pre The JSON value must not be `null` (would throw `std::out_of_range`) - or an empty array or object (undefined behavior, guarded by assertions). - @post The JSON value remains unchanged. - - @throw std::out_of_range when called on `null` value - - @liveexample{The following code shows an example for `front()`.,front} - - @sa @ref back() -- access the last element - - @since version 1.0.0 - */ - reference front() - { - return *begin(); - } - - /*! - @copydoc basic_json::front() - */ - const_reference front() const - { - return *cbegin(); - } - - /*! - @brief access the last element - - Returns a reference to the last element in the container. For a JSON - container `c`, the expression `c.back()` is equivalent to - @code {.cpp} - auto tmp = c.end(); - --tmp; - return *tmp; - @endcode - - @return In case of a structured type (array or object), a reference to the - last element is returned. In cast of number, string, or boolean values, a - reference to the value is returned. - - @complexity Constant. - - @pre The JSON value must not be `null` (would throw `std::out_of_range`) - or an empty array or object (undefined behavior, guarded by assertions). - @post The JSON value remains unchanged. - - @throw std::out_of_range when called on `null` value. - - @liveexample{The following code shows an example for `back()`.,back} - - @sa @ref front() -- access the first element - - @since version 1.0.0 - */ - reference back() - { - auto tmp = end(); - --tmp; - return *tmp; - } - - /*! - @copydoc basic_json::back() - */ - const_reference back() const - { - auto tmp = cend(); - --tmp; - return *tmp; - } - - /*! - @brief remove element given an iterator - - Removes the element specified by iterator @a pos. The iterator @a pos must - be valid and dereferenceable. Thus the `end()` iterator (which is valid, - but is not dereferenceable) cannot be used as a value for @a pos. - - If called on a primitive type other than `null`, the resulting JSON value - will be `null`. - - @param[in] pos iterator to the element to remove - @return Iterator following the last removed element. If the iterator @a - pos refers to the last element, the `end()` iterator is returned. - - @tparam InteratorType an @ref iterator or @ref const_iterator - - @post Invalidates iterators and references at or after the point of the - erase, including the `end()` iterator. - - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on an iterator which does not belong to - the current JSON value; example: `"iterator does not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid - iterator (i.e., any iterator which is not `begin()`); example: `"iterator - out of range"` - - @complexity The complexity depends on the type: - - objects: amortized constant - - arrays: linear in distance between pos and the end of the container - - strings: linear in the length of the string - - other types: constant - - @liveexample{The example shows the result of `erase()` for different JSON - types.,erase__IteratorType} - - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - InteratorType erase(InteratorType pos) - { - // make sure iterator fits the current value - if (this != pos.m_object) - { - throw std::domain_error("iterator does not fit current value"); - } - - InteratorType result = end(); - - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not pos.m_it.primitive_iterator.is_begin()) - { - throw std::out_of_range("iterator out of range"); - } - - if (is_string()) - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - m_value.string = nullptr; - } - - m_type = value_t::null; - assert_invariant(); - break; - } - - case value_t::object: - { - result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); - break; - } - - case value_t::array: - { - result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - return result; - } - - /*! - @brief remove elements given an iterator range - - Removes the element specified by the range `[first; last)`. The iterator - @a first does not need to be dereferenceable if `first == last`: erasing - an empty range is a no-op. - - If called on a primitive type other than `null`, the resulting JSON value - will be `null`. - - @param[in] first iterator to the beginning of the range to remove - @param[in] last iterator past the end of the range to remove - @return Iterator following the last removed element. If the iterator @a - second refers to the last element, the `end()` iterator is returned. - - @tparam InteratorType an @ref iterator or @ref const_iterator - - @post Invalidates iterators and references at or after the point of the - erase, including the `end()` iterator. - - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on iterators which does not belong to - the current JSON value; example: `"iterators do not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid - iterators (i.e., if `first != begin()` and `last != end()`); example: - `"iterators out of range"` - - @complexity The complexity depends on the type: - - objects: `log(size()) + std::distance(first, last)` - - arrays: linear in the distance between @a first and @a last, plus linear - in the distance between @a last and end of the container - - strings: linear in the length of the string - - other types: constant - - @liveexample{The example shows the result of `erase()` for different JSON - types.,erase__IteratorType_IteratorType} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - InteratorType erase(InteratorType first, InteratorType last) - { - // make sure iterator fits the current value - if (this != first.m_object or this != last.m_object) - { - throw std::domain_error("iterators do not fit current value"); - } - - InteratorType result = end(); - - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) - { - throw std::out_of_range("iterators out of range"); - } - - if (is_string()) - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - m_value.string = nullptr; - } - - m_type = value_t::null; - assert_invariant(); - break; - } - - case value_t::object: - { - result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, - last.m_it.object_iterator); - break; - } - - case value_t::array: - { - result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, - last.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - return result; - } - - /*! - @brief remove element from a JSON object given a key - - Removes elements from a JSON object with the key value @a key. - - @param[in] key value of the elements to remove - - @return Number of elements removed. If @a ObjectType is the default - `std::map` type, the return value will always be `0` (@a key was not - found) or `1` (@a key was found). - - @post References and iterators to the erased elements are invalidated. - Other references and iterators are not affected. - - @throw std::domain_error when called on a type other than JSON object; - example: `"cannot use erase() with null"` - - @complexity `log(size()) + count(key)` - - @liveexample{The example shows the effect of `erase()`.,erase__key_type} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - size_type erase(const typename object_t::key_type& key) - { - // this erase only works for objects - if (is_object()) - { - return m_value.object->erase(key); - } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - /*! - @brief remove element from a JSON array given an index - - Removes element from a JSON array at the index @a idx. - - @param[in] idx index of the element to remove - - @throw std::domain_error when called on a type other than JSON array; - example: `"cannot use erase() with null"` - @throw std::out_of_range when `idx >= size()`; example: `"array index 17 - is out of range"` - - @complexity Linear in distance between @a idx and the end of the container. - - @liveexample{The example shows the effect of `erase()`.,erase__size_type} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - - @since version 1.0.0 - */ - void erase(const size_type idx) - { - // this erase only works for arrays - if (is_array()) - { - if (idx >= size()) - { - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - - m_value.array->erase(m_value.array->begin() + static_cast(idx)); - } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - /// @} - - - //////////// - // lookup // - //////////// - - /// @name lookup - /// @{ - - /*! - @brief find an element in a JSON object - - Finds an element in a JSON object with key equivalent to @a key. If the - element is not found or the JSON value is not an object, end() is - returned. - - @param[in] key key value of the element to search for - - @return Iterator to an element with key equivalent to @a key. If no such - element is found, past-the-end (see end()) iterator is returned. - - @complexity Logarithmic in the size of the JSON object. - - @liveexample{The example shows how `find()` is used.,find__key_type} - - @since version 1.0.0 - */ - iterator find(typename object_t::key_type key) - { - auto result = end(); - - if (is_object()) - { - result.m_it.object_iterator = m_value.object->find(key); - } - - return result; - } - - /*! - @brief find an element in a JSON object - @copydoc find(typename object_t::key_type) - */ - const_iterator find(typename object_t::key_type key) const - { - auto result = cend(); - - if (is_object()) - { - result.m_it.object_iterator = m_value.object->find(key); - } - - return result; - } - - /*! - @brief returns the number of occurrences of a key in a JSON object - - Returns the number of elements with key @a key. If ObjectType is the - default `std::map` type, the return value will always be `0` (@a key was - not found) or `1` (@a key was found). - - @param[in] key key value of the element to count - - @return Number of elements with key @a key. If the JSON value is not an - object, the return value will be `0`. - - @complexity Logarithmic in the size of the JSON object. - - @liveexample{The example shows how `count()` is used.,count} - - @since version 1.0.0 - */ - size_type count(typename object_t::key_type key) const - { - // return 0 for all nonobject types - return is_object() ? m_value.object->count(key) : 0; - } - - /// @} - - - /////////////// - // iterators // - /////////////// - - /// @name iterators - /// @{ - - /*! - @brief returns an iterator to the first element - - Returns an iterator to the first element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return iterator to the first element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - @liveexample{The following code shows an example for `begin()`.,begin} - - @sa @ref cbegin() -- returns a const iterator to the beginning - @sa @ref end() -- returns an iterator to the end - @sa @ref cend() -- returns a const iterator to the end - - @since version 1.0.0 - */ - iterator begin() noexcept - { - iterator result(this); - result.set_begin(); - return result; - } - - /*! - @copydoc basic_json::cbegin() - */ - const_iterator begin() const noexcept - { - return cbegin(); - } - - /*! - @brief returns a const iterator to the first element - - Returns a const iterator to the first element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return const iterator to the first element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).begin()`. - - @liveexample{The following code shows an example for `cbegin()`.,cbegin} - - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref end() -- returns an iterator to the end - @sa @ref cend() -- returns a const iterator to the end - - @since version 1.0.0 - */ - const_iterator cbegin() const noexcept - { - const_iterator result(this); - result.set_begin(); - return result; - } - - /*! - @brief returns an iterator to one past the last element - - Returns an iterator to one past the last element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return iterator one past the last element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - @liveexample{The following code shows an example for `end()`.,end} - - @sa @ref cend() -- returns a const iterator to the end - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref cbegin() -- returns a const iterator to the beginning - - @since version 1.0.0 - */ - iterator end() noexcept - { - iterator result(this); - result.set_end(); - return result; - } - - /*! - @copydoc basic_json::cend() - */ - const_iterator end() const noexcept - { - return cend(); - } - - /*! - @brief returns a const iterator to one past the last element - - Returns a const iterator to one past the last element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return const iterator one past the last element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).end()`. - - @liveexample{The following code shows an example for `cend()`.,cend} - - @sa @ref end() -- returns an iterator to the end - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref cbegin() -- returns a const iterator to the beginning - - @since version 1.0.0 - */ - const_iterator cend() const noexcept - { - const_iterator result(this); - result.set_end(); - return result; - } - - /*! - @brief returns an iterator to the reverse-beginning - - Returns an iterator to the reverse-beginning; that is, the last element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `reverse_iterator(end())`. - - @liveexample{The following code shows an example for `rbegin()`.,rbegin} - - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref crend() -- returns a const reverse iterator to the end - - @since version 1.0.0 - */ - reverse_iterator rbegin() noexcept - { - return reverse_iterator(end()); - } - - /*! - @copydoc basic_json::crbegin() - */ - const_reverse_iterator rbegin() const noexcept - { - return crbegin(); - } - - /*! - @brief returns an iterator to the reverse-end - - Returns an iterator to the reverse-end; that is, one before the first - element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `reverse_iterator(begin())`. - - @liveexample{The following code shows an example for `rend()`.,rend} - - @sa @ref crend() -- returns a const reverse iterator to the end - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - - @since version 1.0.0 - */ - reverse_iterator rend() noexcept - { - return reverse_iterator(begin()); - } - - /*! - @copydoc basic_json::crend() - */ - const_reverse_iterator rend() const noexcept - { - return crend(); - } - - /*! - @brief returns a const reverse iterator to the last element - - Returns a const iterator to the reverse-beginning; that is, the last - element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).rbegin()`. - - @liveexample{The following code shows an example for `crbegin()`.,crbegin} - - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref crend() -- returns a const reverse iterator to the end - - @since version 1.0.0 - */ - const_reverse_iterator crbegin() const noexcept - { - return const_reverse_iterator(cend()); - } - - /*! - @brief returns a const reverse iterator to one before the first - - Returns a const reverse iterator to the reverse-end; that is, one before - the first element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).rend()`. - - @liveexample{The following code shows an example for `crend()`.,crend} - - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - - @since version 1.0.0 - */ - const_reverse_iterator crend() const noexcept - { - return const_reverse_iterator(cbegin()); - } - - private: - // forward declaration - template class iteration_proxy; - - public: - /*! - @brief wrapper to access iterator member functions in range-based for - - This function allows to access @ref iterator::key() and @ref - iterator::value() during range-based for loops. In these loops, a - reference to the JSON values is returned, so there is no access to the - underlying iterator. - - @note The name of this function is not yet final and may change in the - future. - */ - static iteration_proxy iterator_wrapper(reference cont) - { - return iteration_proxy(cont); - } - - /*! - @copydoc iterator_wrapper(reference) - */ - static iteration_proxy iterator_wrapper(const_reference cont) - { - return iteration_proxy(cont); - } - - /// @} - - - ////////////// - // capacity // - ////////////// - - /// @name capacity - /// @{ - - /*! - @brief checks whether the container is empty - - Checks if a JSON value has no elements. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `true` - boolean | `false` - string | `false` - number | `false` - object | result of function `object_t::empty()` - array | result of function `array_t::empty()` - - @note This function does not return whether a string stored as JSON value - is empty - it returns whether the JSON container itself is empty which is - false in the case of a string. - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their `empty()` functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `begin() == end()`. - - @liveexample{The following code uses `empty()` to check if a JSON - object contains any elements.,empty} - - @sa @ref size() -- returns the number of elements - - @since version 1.0.0 - */ - bool empty() const noexcept - { - switch (m_type) - { - case value_t::null: - { - // null values are empty - return true; - } - - case value_t::array: - { - // delegate call to array_t::empty() - return m_value.array->empty(); - } - - case value_t::object: - { - // delegate call to object_t::empty() - return m_value.object->empty(); - } - - default: - { - // all other types are nonempty - return false; - } - } - } - - /*! - @brief returns the number of elements - - Returns the number of elements in a JSON value. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `0` - boolean | `1` - string | `1` - number | `1` - object | result of function object_t::size() - array | result of function array_t::size() - - @note This function does not return the length of a string stored as JSON - value - it returns the number of elements in the JSON value which is 1 in - the case of a string. - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their size() functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `std::distance(begin(), end())`. - - @liveexample{The following code calls `size()` on the different value - types.,size} - - @sa @ref empty() -- checks whether the container is empty - @sa @ref max_size() -- returns the maximal number of elements - - @since version 1.0.0 - */ - size_type size() const noexcept - { - switch (m_type) - { - case value_t::null: - { - // null values are empty - return 0; - } - - case value_t::array: - { - // delegate call to array_t::size() - return m_value.array->size(); - } - - case value_t::object: - { - // delegate call to object_t::size() - return m_value.object->size(); - } - - default: - { - // all other types have size 1 - return 1; - } - } - } - - /*! - @brief returns the maximum possible number of elements - - Returns the maximum number of elements a JSON value is able to hold due to - system or library implementation limitations, i.e. `std::distance(begin(), - end())` for the JSON value. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `0` (same as `size()`) - boolean | `1` (same as `size()`) - string | `1` (same as `size()`) - number | `1` (same as `size()`) - object | result of function `object_t::max_size()` - array | result of function `array_t::max_size()` - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their `max_size()` functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of returning `b.size()` where `b` is the largest - possible JSON value. - - @liveexample{The following code calls `max_size()` on the different value - types. Note the output is implementation specific.,max_size} - - @sa @ref size() -- returns the number of elements - - @since version 1.0.0 - */ - size_type max_size() const noexcept - { - switch (m_type) - { - case value_t::array: - { - // delegate call to array_t::max_size() - return m_value.array->max_size(); - } - - case value_t::object: - { - // delegate call to object_t::max_size() - return m_value.object->max_size(); - } - - default: - { - // all other types have max_size() == size() - return size(); - } - } - } - - /// @} - - - /////////////// - // modifiers // - /////////////// - - /// @name modifiers - /// @{ - - /*! - @brief clears the contents - - Clears the content of a JSON value and resets it to the default value as - if @ref basic_json(value_t) would have been called: - - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` - - @note Floating-point numbers are set to `0.0` which will be serialized to - `0`. The vale type remains @ref number_float_t. - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows the effect of `clear()` to different - JSON types.,clear} - - @since version 1.0.0 - */ - void clear() noexcept - { - switch (m_type) - { - case value_t::number_integer: - { - m_value.number_integer = 0; - break; - } - - case value_t::number_unsigned: - { - m_value.number_unsigned = 0; - break; - } - - case value_t::number_float: - { - m_value.number_float = 0.0; - break; - } - - case value_t::boolean: - { - m_value.boolean = false; - break; - } - - case value_t::string: - { - m_value.string->clear(); - break; - } - - case value_t::array: - { - m_value.array->clear(); - break; - } - - case value_t::object: - { - m_value.object->clear(); - break; - } - - default: - { - break; - } - } - } - - /*! - @brief add an object to an array - - Appends the given element @a val to the end of the JSON value. If the - function is called on a JSON null value, an empty array is created before - appending @a val. - - @param[in] val the value to add to the JSON array - - @throw std::domain_error when called on a type other than JSON array or - null; example: `"cannot use push_back() with number"` - - @complexity Amortized constant. - - @liveexample{The example shows how `push_back()` and `+=` can be used to - add elements to a JSON array. Note how the `null` value was silently - converted to a JSON array.,push_back} - - @since version 1.0.0 - */ - void push_back(basic_json&& val) - { - // push_back only works for null objects or arrays - if (not(is_null() or is_array())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an array - if (is_null()) - { - m_type = value_t::array; - m_value = value_t::array; - assert_invariant(); - } - - // add element to array (move semantics) - m_value.array->push_back(std::move(val)); - // invalidate object - val.m_type = value_t::null; - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - reference operator+=(basic_json&& val) - { - push_back(std::move(val)); - return *this; - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - void push_back(const basic_json& val) - { - // push_back only works for null objects or arrays - if (not(is_null() or is_array())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an array - if (is_null()) - { - m_type = value_t::array; - m_value = value_t::array; - assert_invariant(); - } - - // add element to array - m_value.array->push_back(val); - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - reference operator+=(const basic_json& val) - { - push_back(val); - return *this; - } - - /*! - @brief add an object to an object - - Inserts the given element @a val to the JSON object. If the function is - called on a JSON null value, an empty object is created before inserting - @a val. - - @param[in] val the value to add to the JSON object - - @throw std::domain_error when called on a type other than JSON object or - null; example: `"cannot use push_back() with number"` - - @complexity Logarithmic in the size of the container, O(log(`size()`)). - - @liveexample{The example shows how `push_back()` and `+=` can be used to - add elements to a JSON object. Note how the `null` value was silently - converted to a JSON object.,push_back__object_t__value} - - @since version 1.0.0 - */ - void push_back(const typename object_t::value_type& val) - { - // push_back only works for null objects or objects - if (not(is_null() or is_object())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an object - if (is_null()) - { - m_type = value_t::object; - m_value = value_t::object; - assert_invariant(); - } - - // add element to array - m_value.object->insert(val); - } - - /*! - @brief add an object to an object - @copydoc push_back(const typename object_t::value_type&) - */ - reference operator+=(const typename object_t::value_type& val) - { - push_back(val); - return *this; - } - - /*! - @brief add an object to an object - - This function allows to use `push_back` with an initializer list. In case - - 1. the current value is an object, - 2. the initializer list @a init contains only two elements, and - 3. the first element of @a init is a string, - - @a init is converted into an object element and added using - @ref push_back(const typename object_t::value_type&). Otherwise, @a init - is converted to a JSON value and added using @ref push_back(basic_json&&). - - @param init an initializer list - - @complexity Linear in the size of the initializer list @a init. - - @note This function is required to resolve an ambiguous overload error, - because pairs like `{"key", "value"}` can be both interpreted as - `object_t::value_type` or `std::initializer_list`, see - https://github.com/nlohmann/json/issues/235 for more information. - - @liveexample{The example shows how initializer lists are treated as - objects when possible.,push_back__initializer_list} - */ - void push_back(std::initializer_list init) - { - if (is_object() and init.size() == 2 and init.begin()->is_string()) - { - const string_t key = *init.begin(); - push_back(typename object_t::value_type(key, *(init.begin() + 1))); - } - else - { - push_back(basic_json(init)); - } - } - - /*! - @brief add an object to an object - @copydoc push_back(std::initializer_list) - */ - reference operator+=(std::initializer_list init) - { - push_back(init); - return *this; - } - - /*! - @brief inserts element - - Inserts element @a val before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] val element to insert - @return iterator pointing to the inserted @a val. - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @complexity Constant plus linear in the distance between pos and end of the - container. - - @liveexample{The example shows how `insert()` is used.,insert} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, const basic_json& val) - { - // insert only works for arrays - if (is_array()) - { - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); - return result; - } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - } - - /*! - @brief inserts element - @copydoc insert(const_iterator, const basic_json&) - */ - iterator insert(const_iterator pos, basic_json&& val) - { - return insert(pos, val); - } - - /*! - @brief inserts elements - - Inserts @a cnt copies of @a val before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] cnt number of copies of @a val to insert - @param[in] val element to insert - @return iterator pointing to the first element inserted, or @a pos if - `cnt==0` - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @complexity Linear in @a cnt plus linear in the distance between @a pos - and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__count} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, size_type cnt, const basic_json& val) - { - // insert only works for arrays - if (is_array()) - { - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); - return result; - } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - } - - /*! - @brief inserts elements - - Inserts elements from range `[first, last)` before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] first begin of the range of elements to insert - @param[in] last end of the range of elements to insert - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - @throw std::domain_error if @a first and @a last do not belong to the same - JSON value; example: `"iterators do not fit"` - @throw std::domain_error if @a first or @a last are iterators into - container for which insert is called; example: `"passed iterators may not - belong to container"` - - @return iterator pointing to the first element inserted, or @a pos if - `first==last` - - @complexity Linear in `std::distance(first, last)` plus linear in the - distance between @a pos and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__range} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, const_iterator first, const_iterator last) - { - // insert only works for arrays - if (not is_array()) - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // check if range iterators belong to the same JSON object - if (first.m_object != last.m_object) - { - throw std::domain_error("iterators do not fit"); - } - - if (first.m_object == this or last.m_object == this) - { - throw std::domain_error("passed iterators may not belong to container"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert( - pos.m_it.array_iterator, - first.m_it.array_iterator, - last.m_it.array_iterator); - return result; - } - - /*! - @brief inserts elements - - Inserts elements from initializer list @a ilist before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] ilist initializer list to insert the values from - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @return iterator pointing to the first element inserted, or @a pos if - `ilist` is empty - - @complexity Linear in `ilist.size()` plus linear in the distance between - @a pos and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__ilist} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, std::initializer_list ilist) - { - // insert only works for arrays - if (not is_array()) - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); - return result; - } - - /*! - @brief exchanges the values - - Exchanges the contents of the JSON value with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other JSON value to exchange the contents with - - @complexity Constant. - - @liveexample{The example below shows how JSON values can be swapped with - `swap()`.,swap__reference} - - @since version 1.0.0 - */ - void swap(reference other) noexcept ( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - std::swap(m_type, other.m_type); - std::swap(m_value, other.m_value); - assert_invariant(); - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON array with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other array to exchange the contents with - - @throw std::domain_error when JSON value is not an array; example: `"cannot - use swap() with string"` - - @complexity Constant. - - @liveexample{The example below shows how arrays can be swapped with - `swap()`.,swap__array_t} - - @since version 1.0.0 - */ - void swap(array_t& other) - { - // swap only works for arrays - if (is_array()) - { - std::swap(*(m_value.array), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON object with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other object to exchange the contents with - - @throw std::domain_error when JSON value is not an object; example: - `"cannot use swap() with string"` - - @complexity Constant. - - @liveexample{The example below shows how objects can be swapped with - `swap()`.,swap__object_t} - - @since version 1.0.0 - */ - void swap(object_t& other) - { - // swap only works for objects - if (is_object()) - { - std::swap(*(m_value.object), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON string with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other string to exchange the contents with - - @throw std::domain_error when JSON value is not a string; example: `"cannot - use swap() with boolean"` - - @complexity Constant. - - @liveexample{The example below shows how strings can be swapped with - `swap()`.,swap__string_t} - - @since version 1.0.0 - */ - void swap(string_t& other) - { - // swap only works for strings - if (is_string()) - { - std::swap(*(m_value.string), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /// @} - - - ////////////////////////////////////////// - // lexicographical comparison operators // - ////////////////////////////////////////// - - /// @name lexicographical comparison operators - /// @{ - - private: - /*! - @brief comparison operator for JSON types - - Returns an ordering that is similar to Python: - - order: null < boolean < number < object < array < string - - furthermore, each type is not smaller than itself - - @since version 1.0.0 - */ - friend bool operator<(const value_t lhs, const value_t rhs) noexcept - { - static constexpr std::array order = {{ - 0, // null - 3, // object - 4, // array - 5, // string - 1, // boolean - 2, // integer - 2, // unsigned - 2, // float - } - }; - - // discarded values are not comparable - if (lhs == value_t::discarded or rhs == value_t::discarded) - { - return false; - } - - return order[static_cast(lhs)] < order[static_cast(rhs)]; - } - - public: - /*! - @brief comparison: equal - - Compares two JSON values for equality according to the following rules: - - Two JSON values are equal if (1) they are from the same type and (2) - their stored values are the same. - - Integer and floating-point numbers are automatically converted before - comparison. Floating-point numbers are compared indirectly: two - floating-point numbers `f1` and `f2` are considered equal if neither - `f1 > f2` nor `f2 > f1` holds. - - Two JSON null values are equal. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are equal - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__equal} - - @since version 1.0.0 - */ - friend bool operator==(const_reference lhs, const_reference rhs) noexcept - { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); - - if (lhs_type == rhs_type) - { - switch (lhs_type) - { - case value_t::array: - { - return *lhs.m_value.array == *rhs.m_value.array; - } - case value_t::object: - { - return *lhs.m_value.object == *rhs.m_value.object; - } - case value_t::null: - { - return true; - } - case value_t::string: - { - return *lhs.m_value.string == *rhs.m_value.string; - } - case value_t::boolean: - { - return lhs.m_value.boolean == rhs.m_value.boolean; - } - case value_t::number_integer: - { - return lhs.m_value.number_integer == rhs.m_value.number_integer; - } - case value_t::number_unsigned: - { - return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; - } - case value_t::number_float: - { - return lhs.m_value.number_float == rhs.m_value.number_float; - } - default: - { - return false; - } - } - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) - { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); - } - - return false; - } - - /*! - @brief comparison: equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__equal__nullptr_t} - - @since version 1.0.0 - */ - friend bool operator==(const_reference v, std::nullptr_t) noexcept - { - return v.is_null(); - } - - /*! - @brief comparison: equal - @copydoc operator==(const_reference, std::nullptr_t) - */ - friend bool operator==(std::nullptr_t, const_reference v) noexcept - { - return v.is_null(); - } - - /*! - @brief comparison: not equal - - Compares two JSON values for inequality by calculating `not (lhs == rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are not equal - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__notequal} - - @since version 1.0.0 - */ - friend bool operator!=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs == rhs); - } - - /*! - @brief comparison: not equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `not v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is not null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__notequal__nullptr_t} - - @since version 1.0.0 - */ - friend bool operator!=(const_reference v, std::nullptr_t) noexcept - { - return not v.is_null(); - } - - /*! - @brief comparison: not equal - @copydoc operator!=(const_reference, std::nullptr_t) - */ - friend bool operator!=(std::nullptr_t, const_reference v) noexcept - { - return not v.is_null(); - } - - /*! - @brief comparison: less than - - Compares whether one JSON value @a lhs is less than another JSON value @a - rhs according to the following rules: - - If @a lhs and @a rhs have the same type, the values are compared using - the default `<` operator. - - Integer and floating-point numbers are automatically converted before - comparison - - In case @a lhs and @a rhs have different types, the values are ignored - and the order of the types is considered, see - @ref operator<(const value_t, const value_t). - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__less} - - @since version 1.0.0 - */ - friend bool operator<(const_reference lhs, const_reference rhs) noexcept - { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); - - if (lhs_type == rhs_type) - { - switch (lhs_type) - { - case value_t::array: - { - return *lhs.m_value.array < *rhs.m_value.array; - } - case value_t::object: - { - return *lhs.m_value.object < *rhs.m_value.object; - } - case value_t::null: - { - return false; - } - case value_t::string: - { - return *lhs.m_value.string < *rhs.m_value.string; - } - case value_t::boolean: - { - return lhs.m_value.boolean < rhs.m_value.boolean; - } - case value_t::number_integer: - { - return lhs.m_value.number_integer < rhs.m_value.number_integer; - } - case value_t::number_unsigned: - { - return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; - } - case value_t::number_float: - { - return lhs.m_value.number_float < rhs.m_value.number_float; - } - default: - { - return false; - } - } - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; - } - - // We only reach this line if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - return operator<(lhs_type, rhs_type); - } - - /*! - @brief comparison: less than or equal - - Compares whether one JSON value @a lhs is less than or equal to another - JSON value by calculating `not (rhs < lhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than or equal to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__greater} - - @since version 1.0.0 - */ - friend bool operator<=(const_reference lhs, const_reference rhs) noexcept - { - return not (rhs < lhs); - } - - /*! - @brief comparison: greater than - - Compares whether one JSON value @a lhs is greater than another - JSON value by calculating `not (lhs <= rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__lessequal} - - @since version 1.0.0 - */ - friend bool operator>(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs <= rhs); - } - - /*! - @brief comparison: greater than or equal - - Compares whether one JSON value @a lhs is greater than or equal to another - JSON value by calculating `not (lhs < rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than or equal to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__greaterequal} - - @since version 1.0.0 - */ - friend bool operator>=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs < rhs); - } - - /// @} - - - /////////////////// - // serialization // - /////////////////// - - /// @name serialization - /// @{ - - /*! - @brief serialize to stream - - Serialize the given JSON value @a j to the output stream @a o. The JSON - value will be serialized using the @ref dump member function. The - indentation of the output can be controlled with the member variable - `width` of the output stream @a o. For instance, using the manipulator - `std::setw(4)` on @a o sets the indentation level to `4` and the - serialization result is the same as calling `dump(4)`. - - @note During serializaion, the locale and the precision of the output - stream @a o are changed. The original values are restored when the - function returns. - - @param[in,out] o stream to serialize to - @param[in] j JSON value to serialize - - @return the stream @a o - - @complexity Linear. - - @liveexample{The example below shows the serialization with different - parameters to `width` to adjust the indentation level.,operator_serialize} - - @since version 1.0.0 - */ - friend std::ostream& operator<<(std::ostream& o, const basic_json& j) - { - // read width member and use it as indentation parameter if nonzero - const bool pretty_print = (o.width() > 0); - const auto indentation = (pretty_print ? o.width() : 0); - - // reset width to 0 for subsequent calls to this stream - o.width(0); - - // fix locale problems - const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator)); - // set precision - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - const auto old_precision = o.precision(std::numeric_limits::digits10); - - // do the actual serialization - j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale and precision - o.imbue(old_locale); - o.precision(old_precision); - return o; - } - - /*! - @brief serialize to stream - @copydoc operator<<(std::ostream&, const basic_json&) - */ - friend std::ostream& operator>>(const basic_json& j, std::ostream& o) - { - return o << j; - } - - /// @} - - - ///////////////////// - // deserialization // - ///////////////////// - - /// @name deserialization - /// @{ - - /*! - @brief deserialize from string - - @param[in] s string to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__string__parser_callback_t} - - @sa @ref parse(std::istream&, const parser_callback_t) for a version that - reads from an input stream - - @since version 1.0.0 - */ - static basic_json parse(const string_t& s, - const parser_callback_t cb = nullptr) - { - return parser(s, cb).parse(); - } - - /*! - @brief deserialize from stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__istream__parser_callback_t} - - @sa @ref parse(const string_t&, const parser_callback_t) for a version - that reads from a string - - @since version 1.0.0 - */ - static basic_json parse(std::istream& i, - const parser_callback_t cb = nullptr) - { - return parser(i, cb).parse(); - } - - /*! - @copydoc parse(std::istream&, const parser_callback_t) - */ - static basic_json parse(std::istream&& i, - const parser_callback_t cb = nullptr) - { - return parser(i, cb).parse(); - } - - /*! - @brief deserialize from stream - - Deserializes an input stream to a JSON value. - - @param[in,out] i input stream to read a serialized JSON value from - @param[in,out] j JSON value to write the deserialized input to - - @throw std::invalid_argument in case of parse errors - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below shows how a JSON value is constructed by - reading a serialization from a stream.,operator_deserialize} - - @sa parse(std::istream&, const parser_callback_t) for a variant with a - parser callback function to filter values while parsing - - @since version 1.0.0 - */ - friend std::istream& operator<<(basic_json& j, std::istream& i) - { - j = parser(i).parse(); - return i; - } - - /*! - @brief deserialize from stream - @copydoc operator<<(basic_json&, std::istream&) - */ - friend std::istream& operator>>(std::istream& i, basic_json& j) - { - j = parser(i).parse(); - return i; - } - - /// @} - - - private: - /////////////////////////// - // convenience functions // - /////////////////////////// - - /*! - @brief return the type as string - - Returns the type name as string to be used in error messages - usually to - indicate that a function was called on a wrong JSON type. - - @return basically a string representation of a the @ref m_type member - - @complexity Constant. - - @since version 1.0.0 - */ - std::string type_name() const - { - switch (m_type) - { - case value_t::null: - return "null"; - case value_t::object: - return "object"; - case value_t::array: - return "array"; - case value_t::string: - return "string"; - case value_t::boolean: - return "boolean"; - case value_t::discarded: - return "discarded"; - default: - return "number"; - } - } - - /*! - @brief calculates the extra space to escape a JSON string - - @param[in] s the string to escape - @return the number of characters required to escape string @a s - - @complexity Linear in the length of string @a s. - */ - static std::size_t extra_space(const string_t& s) noexcept - { - return std::accumulate(s.begin(), s.end(), size_t{}, - [](size_t res, typename string_t::value_type c) - { - switch (c) - { - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - return res + 1; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // from c (1 byte) to \uxxxx (6 bytes) - return res + 5; - } - else - { - return res; - } - } - } - }); - } - - /*! - @brief escape a string - - Escape a string by replacing certain special characters by a sequence of - an escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. - - @param[in] s the string to escape - @return the escaped string - - @complexity Linear in the length of string @a s. - */ - static string_t escape_string(const string_t& s) - { - const auto space = extra_space(s); - if (space == 0) - { - return s; - } - - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; - - for (const auto& c : s) - { - switch (c) - { - // quotation mark (0x22) - case '"': - { - result[pos + 1] = '"'; - pos += 2; - break; - } - - // reverse solidus (0x5c) - case '\\': - { - // nothing to change - pos += 2; - break; - } - - // backspace (0x08) - case '\b': - { - result[pos + 1] = 'b'; - pos += 2; - break; - } - - // formfeed (0x0c) - case '\f': - { - result[pos + 1] = 'f'; - pos += 2; - break; - } - - // newline (0x0a) - case '\n': - { - result[pos + 1] = 'n'; - pos += 2; - break; - } - - // carriage return (0x0d) - case '\r': - { - result[pos + 1] = 'r'; - pos += 2; - break; - } - - // horizontal tab (0x09) - case '\t': - { - result[pos + 1] = 't'; - pos += 2; - break; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // convert a number 0..15 to its hex representation - // (0..f) - static const char hexify[16] = - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - - // print character c as \uxxxx - for (const char m : - { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] - }) - { - result[++pos] = m; - } - - ++pos; - } - else - { - // all other characters are added as-is - result[pos++] = c; - } - break; - } - } - } - - return result; - } - - /*! - @brief internal implementation of the serialization function - - This function is called by the public member function dump and organizes - the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. Note that - - - strings and object keys are escaped using `escape_string()` - - integer numbers are converted implicitly via `operator<<` - - floating-point numbers are converted to a string using `"%g"` format - - @param[out] o stream to write to - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] indent_step the indent level - @param[in] current_indent the current indent level (only used internally) - */ - void dump(std::ostream& o, - const bool pretty_print, - const unsigned int indent_step, - const unsigned int current_indent = 0) const - { - // variable to hold indentation for recursive calls - unsigned int new_indent = current_indent; - - switch (m_type) - { - case value_t::object: - { - if (m_value.object->empty()) - { - o << "{}"; - return; - } - - o << "{"; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) - { - if (i != m_value.object->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' ') << "\"" - << escape_string(i->first) << "\":" - << (pretty_print ? " " : ""); - i->second.dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') + "}"; - return; - } - - case value_t::array: - { - if (m_value.array->empty()) - { - o << "[]"; - return; - } - - o << "["; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) - { - if (i != m_value.array->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' '); - i->dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') << "]"; - return; - } - - case value_t::string: - { - o << string_t("\"") << escape_string(*m_value.string) << "\""; - return; - } - - case value_t::boolean: - { - o << (m_value.boolean ? "true" : "false"); - return; - } - - case value_t::number_integer: - { - o << m_value.number_integer; - return; - } - - case value_t::number_unsigned: - { - o << m_value.number_unsigned; - return; - } - - case value_t::number_float: - { - if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); - } - else - { - o << m_value.number_float; - } - return; - } - - case value_t::discarded: - { - o << ""; - return; - } - - case value_t::null: - { - o << "null"; - return; - } - } - } - - private: - ////////////////////// - // member variables // - ////////////////////// - - /// the type of the current element - value_t m_type = value_t::null; - - /// the value of the current element - json_value m_value = {}; - - - private: - /////////////// - // iterators // - /////////////// - - /*! - @brief an iterator for primitive JSON types - - This class models an iterator for primitive JSON types (boolean, number, - string). It's only purpose is to allow the iterator/const_iterator classes - to "iterate" over primitive values. Internally, the iterator is modeled by - a `difference_type` variable. Value begin_value (`0`) models the begin, - end_value (`1`) models past the end. - */ - class primitive_iterator_t - { - public: - /// set iterator to a defined beginning - void set_begin() noexcept - { - m_it = begin_value; - } - - /// set iterator to a defined past the end - void set_end() noexcept - { - m_it = end_value; - } - - /// return whether the iterator can be dereferenced - constexpr bool is_begin() const noexcept - { - return (m_it == begin_value); - } - - /// return whether the iterator is at end - constexpr bool is_end() const noexcept - { - return (m_it == end_value); - } - - /// return reference to the value to change and compare - operator difference_type& () noexcept - { - return m_it; - } - - /// return value to compare - constexpr operator difference_type () const noexcept - { - return m_it; - } - - private: - static constexpr difference_type begin_value = 0; - static constexpr difference_type end_value = begin_value + 1; - - /// iterator as signed integer type - difference_type m_it = std::numeric_limits::denorm_min(); - }; - - /*! - @brief an iterator value - - @note This structure could easily be a union, but MSVC currently does not - allow unions members with complex constructors, see - https://github.com/nlohmann/json/pull/105. - */ - struct internal_iterator - { - /// iterator for JSON objects - typename object_t::iterator object_iterator; - /// iterator for JSON arrays - typename array_t::iterator array_iterator; - /// generic iterator for all other types - primitive_iterator_t primitive_iterator; - - /// create an uninitialized internal_iterator - internal_iterator() noexcept - : object_iterator(), array_iterator(), primitive_iterator() - {} - }; - - /// proxy class for the iterator_wrapper functions - template - class iteration_proxy - { - private: - /// helper class for iteration - class iteration_proxy_internal - { - private: - /// the iterator - IteratorType anchor; - /// an index for arrays (used to create key names) - size_t array_index = 0; - - public: - explicit iteration_proxy_internal(IteratorType it) noexcept - : anchor(it) - {} - - /// dereference operator (needed for range-based for) - iteration_proxy_internal& operator*() - { - return *this; - } - - /// increment operator (needed for range-based for) - iteration_proxy_internal& operator++() - { - ++anchor; - ++array_index; - - return *this; - } - - /// inequality operator (needed for range-based for) - bool operator!= (const iteration_proxy_internal& o) const - { - return anchor != o.anchor; - } - - /// return key of the iterator - typename basic_json::string_t key() const - { - assert(anchor.m_object != nullptr); - - switch (anchor.m_object->type()) - { - // use integer array index as key - case value_t::array: - { - return std::to_string(array_index); - } - - // use key from the object - case value_t::object: - { - return anchor.key(); - } - - // use an empty key for all primitive types - default: - { - return ""; - } - } - } - - /// return value of the iterator - typename IteratorType::reference value() const - { - return anchor.value(); - } - }; - - /// the container to iterate - typename IteratorType::reference container; - - public: - /// construct iteration proxy from a container - explicit iteration_proxy(typename IteratorType::reference cont) - : container(cont) - {} - - /// return iterator begin (needed for range-based for) - iteration_proxy_internal begin() noexcept - { - return iteration_proxy_internal(container.begin()); - } - - /// return iterator end (needed for range-based for) - iteration_proxy_internal end() noexcept - { - return iteration_proxy_internal(container.end()); - } - }; - - public: - /*! - @brief a const random access iterator for the @ref basic_json class - - This class implements a const iterator for the @ref basic_json class. From - this class, the @ref iterator class is derived. - - @note An iterator is called *initialized* when a pointer to a JSON value - has been set (e.g., by a constructor or a copy assignment). If the - iterator is default-constructed, it is *uninitialized* and most - methods are undefined. The library uses assertions to detect calls - on uninitialized iterators. - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - @since version 1.0.0 - */ - class const_iterator : public std::iterator - { - /// allow basic_json to access private members - friend class basic_json; - - public: - /// the type of the values when the iterator is dereferenced - using value_type = typename basic_json::value_type; - /// a type to represent differences between iterators - using difference_type = typename basic_json::difference_type; - /// defines a pointer to the type iterated over (value_type) - using pointer = typename basic_json::const_pointer; - /// defines a reference to the type iterated over (value_type) - using reference = typename basic_json::const_reference; - /// the category of the iterator - using iterator_category = std::bidirectional_iterator_tag; - - /// default constructor - const_iterator() = default; - - /*! - @brief constructor for a given JSON instance - @param[in] object pointer to a JSON object for this iterator - @pre object != nullptr - @post The iterator is initialized; i.e. `m_object != nullptr`. - */ - explicit const_iterator(pointer object) noexcept - : m_object(object) - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = typename object_t::iterator(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = typename array_t::iterator(); - break; - } - - default: - { - m_it.primitive_iterator = primitive_iterator_t(); - break; - } - } - } - - /*! - @brief copy constructor given a non-const iterator - @param[in] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - explicit const_iterator(const iterator& other) noexcept - : m_object(other.m_object) - { - if (m_object != nullptr) - { - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = other.m_it.object_iterator; - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = other.m_it.array_iterator; - break; - } - - default: - { - m_it.primitive_iterator = other.m_it.primitive_iterator; - break; - } - } - } - } - - /*! - @brief copy constructor - @param[in] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - const_iterator(const const_iterator& other) noexcept - : m_object(other.m_object), m_it(other.m_it) - {} - - /*! - @brief copy assignment - @param[in,out] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - const_iterator& operator=(const_iterator other) noexcept( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - std::swap(m_object, other.m_object); - std::swap(m_it, other.m_it); - return *this; - } - - private: - /*! - @brief set the iterator to the first value - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - void set_begin() noexcept - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = m_object->m_value.object->begin(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = m_object->m_value.array->begin(); - break; - } - - case basic_json::value_t::null: - { - // set to end so begin()==end() is true: null is empty - m_it.primitive_iterator.set_end(); - break; - } - - default: - { - m_it.primitive_iterator.set_begin(); - break; - } - } - } - - /*! - @brief set the iterator past the last value - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - void set_end() noexcept - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = m_object->m_value.object->end(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = m_object->m_value.array->end(); - break; - } - - default: - { - m_it.primitive_iterator.set_end(); - break; - } - } - } - - public: - /*! - @brief return a reference to the value pointed to by the iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference operator*() const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - assert(m_it.object_iterator != m_object->m_value.object->end()); - return m_it.object_iterator->second; - } - - case basic_json::value_t::array: - { - assert(m_it.array_iterator != m_object->m_value.array->end()); - return *m_it.array_iterator; - } - - case basic_json::value_t::null: - { - throw std::out_of_range("cannot get value"); - } - - default: - { - if (m_it.primitive_iterator.is_begin()) - { - return *m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief dereference the iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - pointer operator->() const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - assert(m_it.object_iterator != m_object->m_value.object->end()); - return &(m_it.object_iterator->second); - } - - case basic_json::value_t::array: - { - assert(m_it.array_iterator != m_object->m_value.array->end()); - return &*m_it.array_iterator; - } - - default: - { - if (m_it.primitive_iterator.is_begin()) - { - return m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief post-increment (it++) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator++(int) - { - auto result = *this; - ++(*this); - return result; - } - - /*! - @brief pre-increment (++it) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator++() - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - std::advance(m_it.object_iterator, 1); - break; - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, 1); - break; - } - - default: - { - ++m_it.primitive_iterator; - break; - } - } - - return *this; - } - - /*! - @brief post-decrement (it--) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator--(int) - { - auto result = *this; - --(*this); - return result; - } - - /*! - @brief pre-decrement (--it) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator--() - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - std::advance(m_it.object_iterator, -1); - break; - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, -1); - break; - } - - default: - { - --m_it.primitive_iterator; - break; - } - } - - return *this; - } - - /*! - @brief comparison: equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator==(const const_iterator& other) const - { - // if objects are not the same, the comparison is undefined - if (m_object != other.m_object) - { - throw std::domain_error("cannot compare iterators of different containers"); - } - - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - return (m_it.object_iterator == other.m_it.object_iterator); - } - - case basic_json::value_t::array: - { - return (m_it.array_iterator == other.m_it.array_iterator); - } - - default: - { - return (m_it.primitive_iterator == other.m_it.primitive_iterator); - } - } - } - - /*! - @brief comparison: not equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator!=(const const_iterator& other) const - { - return not operator==(other); - } - - /*! - @brief comparison: smaller - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator<(const const_iterator& other) const - { - // if objects are not the same, the comparison is undefined - if (m_object != other.m_object) - { - throw std::domain_error("cannot compare iterators of different containers"); - } - - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot compare order of object iterators"); - } - - case basic_json::value_t::array: - { - return (m_it.array_iterator < other.m_it.array_iterator); - } - - default: - { - return (m_it.primitive_iterator < other.m_it.primitive_iterator); - } - } - } - - /*! - @brief comparison: less than or equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator<=(const const_iterator& other) const - { - return not other.operator < (*this); - } - - /*! - @brief comparison: greater than - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator>(const const_iterator& other) const - { - return not operator<=(other); - } - - /*! - @brief comparison: greater than or equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator>=(const const_iterator& other) const - { - return not operator<(other); - } - - /*! - @brief add to iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator+=(difference_type i) - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use offsets with object iterators"); - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, i); - break; - } - - default: - { - m_it.primitive_iterator += i; - break; - } - } - - return *this; - } - - /*! - @brief subtract from iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator-=(difference_type i) - { - return operator+=(-i); - } - - /*! - @brief add to iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator+(difference_type i) - { - auto result = *this; - result += i; - return result; - } - - /*! - @brief subtract from iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator-(difference_type i) - { - auto result = *this; - result -= i; - return result; - } - - /*! - @brief return difference - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - difference_type operator-(const const_iterator& other) const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use offsets with object iterators"); - } - - case basic_json::value_t::array: - { - return m_it.array_iterator - other.m_it.array_iterator; - } - - default: - { - return m_it.primitive_iterator - other.m_it.primitive_iterator; - } - } - } - - /*! - @brief access to successor - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference operator[](difference_type n) const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use operator[] for object iterators"); - } - - case basic_json::value_t::array: - { - return *std::next(m_it.array_iterator, n); - } - - case basic_json::value_t::null: - { - throw std::out_of_range("cannot get value"); - } - - default: - { - if (m_it.primitive_iterator == -n) - { - return *m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief return the key of an object iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - typename object_t::key_type key() const - { - assert(m_object != nullptr); - - if (m_object->is_object()) - { - return m_it.object_iterator->first; - } - else - { - throw std::domain_error("cannot use key() for non-object iterators"); - } - } - - /*! - @brief return the value of an iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference value() const - { - return operator*(); - } - - private: - /// associated JSON instance - pointer m_object = nullptr; - /// the actual iterator of the associated instance - internal_iterator m_it = internal_iterator(); - }; - - /*! - @brief a mutable random access iterator for the @ref basic_json class - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): - It is possible to write to the pointed-to element. - - @since version 1.0.0 - */ - class iterator : public const_iterator - { - public: - using base_iterator = const_iterator; - using pointer = typename basic_json::pointer; - using reference = typename basic_json::reference; - - /// default constructor - iterator() = default; - - /// constructor for a given JSON instance - explicit iterator(pointer object) noexcept - : base_iterator(object) - {} - - /// copy constructor - iterator(const iterator& other) noexcept - : base_iterator(other) - {} - - /// copy assignment - iterator& operator=(iterator other) noexcept( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - base_iterator::operator=(other); - return *this; - } - - /// return a reference to the value pointed to by the iterator - reference operator*() const - { - return const_cast(base_iterator::operator*()); - } - - /// dereference the iterator - pointer operator->() const - { - return const_cast(base_iterator::operator->()); - } - - /// post-increment (it++) - iterator operator++(int) - { - iterator result = *this; - base_iterator::operator++(); - return result; - } - - /// pre-increment (++it) - iterator& operator++() - { - base_iterator::operator++(); - return *this; - } - - /// post-decrement (it--) - iterator operator--(int) - { - iterator result = *this; - base_iterator::operator--(); - return result; - } - - /// pre-decrement (--it) - iterator& operator--() - { - base_iterator::operator--(); - return *this; - } - - /// add to iterator - iterator& operator+=(difference_type i) - { - base_iterator::operator+=(i); - return *this; - } - - /// subtract from iterator - iterator& operator-=(difference_type i) - { - base_iterator::operator-=(i); - return *this; - } - - /// add to iterator - iterator operator+(difference_type i) - { - auto result = *this; - result += i; - return result; - } - - /// subtract from iterator - iterator operator-(difference_type i) - { - auto result = *this; - result -= i; - return result; - } - - /// return difference - difference_type operator-(const iterator& other) const - { - return base_iterator::operator-(other); - } - - /// access to successor - reference operator[](difference_type n) const - { - return const_cast(base_iterator::operator[](n)); - } - - /// return the value of an iterator - reference value() const - { - return const_cast(base_iterator::value()); - } - }; - - /*! - @brief a template for a reverse iterator class - - @tparam Base the base iterator type to reverse. Valid types are @ref - iterator (to create @ref reverse_iterator) and @ref const_iterator (to - create @ref const_reverse_iterator). - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): - It is possible to write to the pointed-to element (only if @a Base is - @ref iterator). - - @since version 1.0.0 - */ - template - class json_reverse_iterator : public std::reverse_iterator - { - public: - /// shortcut to the reverse iterator adaptor - using base_iterator = std::reverse_iterator; - /// the reference type for the pointed-to element - using reference = typename Base::reference; - - /// create reverse iterator from iterator - json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept - : base_iterator(it) - {} - - /// create reverse iterator from base class - json_reverse_iterator(const base_iterator& it) noexcept - : base_iterator(it) - {} - - /// post-increment (it++) - json_reverse_iterator operator++(int) - { - return base_iterator::operator++(1); - } - - /// pre-increment (++it) - json_reverse_iterator& operator++() - { - base_iterator::operator++(); - return *this; - } - - /// post-decrement (it--) - json_reverse_iterator operator--(int) - { - return base_iterator::operator--(1); - } - - /// pre-decrement (--it) - json_reverse_iterator& operator--() - { - base_iterator::operator--(); - return *this; - } - - /// add to iterator - json_reverse_iterator& operator+=(difference_type i) - { - base_iterator::operator+=(i); - return *this; - } - - /// add to iterator - json_reverse_iterator operator+(difference_type i) const - { - auto result = *this; - result += i; - return result; - } - - /// subtract from iterator - json_reverse_iterator operator-(difference_type i) const - { - auto result = *this; - result -= i; - return result; - } - - /// return difference - difference_type operator-(const json_reverse_iterator& other) const - { - return this->base() - other.base(); - } - - /// access to successor - reference operator[](difference_type n) const - { - return *(this->operator+(n)); - } - - /// return the key of an object iterator - typename object_t::key_type key() const - { - auto it = --this->base(); - return it.key(); - } - - /// return the value of an iterator - reference value() const - { - auto it = --this->base(); - return it.operator * (); - } - }; - - - private: - ////////////////////// - // lexer and parser // - ////////////////////// - - /*! - @brief lexical analysis - - This class organizes the lexical analysis during JSON deserialization. The - core of it is a scanner generated by [re2c](http://re2c.org) that - processes a buffer and recognizes tokens according to RFC 7159. - */ - class lexer - { - public: - /// token types for the parser - enum class token_type - { - uninitialized, ///< indicating the scanner is uninitialized - literal_true, ///< the `true` literal - literal_false, ///< the `false` literal - literal_null, ///< the `null` literal - value_string, ///< a string -- use get_string() for actual value - value_number, ///< a number -- use get_number() for actual value - begin_array, ///< the character for array begin `[` - begin_object, ///< the character for object begin `{` - end_array, ///< the character for array end `]` - end_object, ///< the character for object end `}` - name_separator, ///< the name separator `:` - value_separator, ///< the value separator `,` - parse_error, ///< indicating a parse error - end_of_input ///< indicating the end of the input buffer - }; - - /// the char type to use in the lexer - using lexer_char_t = unsigned char; - - /// constructor with a given buffer - explicit lexer(const string_t& s) noexcept - : m_stream(nullptr), m_buffer(s) - { - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + s.size(); - } - - /// constructor with a given stream - explicit lexer(std::istream* s) noexcept - : m_stream(s), m_buffer() - { - assert(m_stream != nullptr); - std::getline(*m_stream, m_buffer); - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + m_buffer.size(); - } - - /// default constructor - lexer() = default; - - // switch off unwanted functions - lexer(const lexer&) = delete; - lexer operator=(const lexer&) = delete; - - /*! - @brief create a string from one or two Unicode code points - - There are two cases: (1) @a codepoint1 is in the Basic Multilingual - Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) - @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to - represent a code point above U+FFFF. - - @param[in] codepoint1 the code point (can be high surrogate) - @param[in] codepoint2 the code point (can be low surrogate or 0) - - @return string representation of the code point; the length of the - result string is between 1 and 4 characters. - - @throw std::out_of_range if code point is > 0x10ffff; example: `"code - points above 0x10FFFF are invalid"` - @throw std::invalid_argument if the low surrogate is invalid; example: - `""missing or wrong low surrogate""` - - @complexity Constant. - - @see - */ - static string_t to_unicode(const std::size_t codepoint1, - const std::size_t codepoint2 = 0) - { - // calculate the code point from the given code points - std::size_t codepoint = codepoint1; - - // check if codepoint1 is a high surrogate - if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) - { - // check if codepoint2 is a low surrogate - if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) - { - codepoint = - // high surrogate occupies the most significant 22 bits - (codepoint1 << 10) - // low surrogate occupies the least significant 15 bits - + codepoint2 - // there is still the 0xD800, 0xDC00 and 0x10000 noise - // in the result so we have to subtract with: - // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 - - 0x35FDC00; - } - else - { - throw std::invalid_argument("missing or wrong low surrogate"); - } - } - - string_t result; - - if (codepoint < 0x80) - { - // 1-byte characters: 0xxxxxxx (ASCII) - result.append(1, static_cast(codepoint)); - } - else if (codepoint <= 0x7ff) - { - // 2-byte characters: 110xxxxx 10xxxxxx - result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else if (codepoint <= 0xffff) - { - // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); - result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else if (codepoint <= 0x10ffff) - { - // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); - result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); - result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else - { - throw std::out_of_range("code points above 0x10FFFF are invalid"); - } - - return result; - } - - /// return name of values of type token_type (only used for errors) - static std::string token_type_name(const token_type t) - { - switch (t) - { - case token_type::uninitialized: - return ""; - case token_type::literal_true: - return "true literal"; - case token_type::literal_false: - return "false literal"; - case token_type::literal_null: - return "null literal"; - case token_type::value_string: - return "string literal"; - case token_type::value_number: - return "number literal"; - case token_type::begin_array: - return "'['"; - case token_type::begin_object: - return "'{'"; - case token_type::end_array: - return "']'"; - case token_type::end_object: - return "'}'"; - case token_type::name_separator: - return "':'"; - case token_type::value_separator: - return "','"; - case token_type::parse_error: - return ""; - case token_type::end_of_input: - return "end of input"; - default: - { - // catch non-enum values - return "unknown token"; // LCOV_EXCL_LINE - } - } - } - - /*! - This function implements a scanner for JSON. It is specified using - regular expressions that try to follow RFC 7159 as close as possible. - These regular expressions are then translated into a minimized - deterministic finite automaton (DFA) by the tool - [re2c](http://re2c.org). As a result, the translated code for this - function consists of a large block of code with `goto` jumps. - - @return the class of the next token read from the buffer - - @complexity Linear in the length of the input.\n - - Proposition: The loop below will always terminate for finite input.\n - - Proof (by contradiction): Assume a finite input. To loop forever, the - loop must never hit code with a `break` statement. The only code - snippets without a `break` statement are the continue statements for - whitespace and byte-order-marks. To loop forever, the input must be an - infinite sequence of whitespace or byte-order-marks. This contradicts - the assumption of finite input, q.e.d. - */ - token_type scan() noexcept - { - while (true) - { - // pointer for backtracking information - m_marker = nullptr; - - // remember the begin of the token - m_start = m_cursor; - assert(m_start != nullptr); - - - { - lexer_char_t yych; - unsigned int yyaccept = 0; - static const unsigned char yybm[] = - { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 32, 32, 0, 0, 32, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 160, 128, 0, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 0, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - }; - if ((m_limit - m_cursor) < 5) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 32) - { - goto basic_json_parser_6; - } - if (yych <= '\\') - { - if (yych <= '-') - { - if (yych <= '"') - { - if (yych <= 0x00) - { - goto basic_json_parser_2; - } - if (yych <= '!') - { - goto basic_json_parser_4; - } - goto basic_json_parser_9; - } - else - { - if (yych <= '+') - { - goto basic_json_parser_4; - } - if (yych <= ',') - { - goto basic_json_parser_10; - } - goto basic_json_parser_12; - } - } - else - { - if (yych <= '9') - { - if (yych <= '/') - { - goto basic_json_parser_4; - } - if (yych <= '0') - { - goto basic_json_parser_13; - } - goto basic_json_parser_15; - } - else - { - if (yych <= ':') - { - goto basic_json_parser_17; - } - if (yych == '[') - { - goto basic_json_parser_19; - } - goto basic_json_parser_4; - } - } - } - else - { - if (yych <= 't') - { - if (yych <= 'f') - { - if (yych <= ']') - { - goto basic_json_parser_21; - } - if (yych <= 'e') - { - goto basic_json_parser_4; - } - goto basic_json_parser_23; - } - else - { - if (yych == 'n') - { - goto basic_json_parser_24; - } - if (yych <= 's') - { - goto basic_json_parser_4; - } - goto basic_json_parser_25; - } - } - else - { - if (yych <= '|') - { - if (yych == '{') - { - goto basic_json_parser_26; - } - goto basic_json_parser_4; - } - else - { - if (yych <= '}') - { - goto basic_json_parser_28; - } - if (yych == 0xEF) - { - goto basic_json_parser_30; - } - goto basic_json_parser_4; - } - } - } -basic_json_parser_2: - ++m_cursor; - { - last_token_type = token_type::end_of_input; - break; - } -basic_json_parser_4: - ++m_cursor; -basic_json_parser_5: - { - last_token_type = token_type::parse_error; - break; - } -basic_json_parser_6: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 32) - { - goto basic_json_parser_6; - } - { - continue; - } -basic_json_parser_9: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych <= 0x1F) - { - goto basic_json_parser_5; - } - goto basic_json_parser_32; -basic_json_parser_10: - ++m_cursor; - { - last_token_type = token_type::value_separator; - break; - } -basic_json_parser_12: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_5; - } - if (yych <= '0') - { - goto basic_json_parser_13; - } - if (yych <= '9') - { - goto basic_json_parser_15; - } - goto basic_json_parser_5; -basic_json_parser_13: - yyaccept = 1; - yych = *(m_marker = ++m_cursor); - if (yych <= 'D') - { - if (yych == '.') - { - goto basic_json_parser_37; - } - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - } -basic_json_parser_14: - { - last_token_type = token_type::value_number; - break; - } -basic_json_parser_15: - yyaccept = 1; - m_marker = ++m_cursor; - if ((m_limit - m_cursor) < 3) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 64) - { - goto basic_json_parser_15; - } - if (yych <= 'D') - { - if (yych == '.') - { - goto basic_json_parser_37; - } - goto basic_json_parser_14; - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - goto basic_json_parser_14; - } -basic_json_parser_17: - ++m_cursor; - { - last_token_type = token_type::name_separator; - break; - } -basic_json_parser_19: - ++m_cursor; - { - last_token_type = token_type::begin_array; - break; - } -basic_json_parser_21: - ++m_cursor; - { - last_token_type = token_type::end_array; - break; - } -basic_json_parser_23: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'a') - { - goto basic_json_parser_39; - } - goto basic_json_parser_5; -basic_json_parser_24: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'u') - { - goto basic_json_parser_40; - } - goto basic_json_parser_5; -basic_json_parser_25: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'r') - { - goto basic_json_parser_41; - } - goto basic_json_parser_5; -basic_json_parser_26: - ++m_cursor; - { - last_token_type = token_type::begin_object; - break; - } -basic_json_parser_28: - ++m_cursor; - { - last_token_type = token_type::end_object; - break; - } -basic_json_parser_30: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 0xBB) - { - goto basic_json_parser_42; - } - goto basic_json_parser_5; -basic_json_parser_31: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; -basic_json_parser_32: - if (yybm[0 + yych] & 128) - { - goto basic_json_parser_31; - } - if (yych <= 0x1F) - { - goto basic_json_parser_33; - } - if (yych <= '"') - { - goto basic_json_parser_34; - } - goto basic_json_parser_36; -basic_json_parser_33: - m_cursor = m_marker; - if (yyaccept == 0) - { - goto basic_json_parser_5; - } - else - { - goto basic_json_parser_14; - } -basic_json_parser_34: - ++m_cursor; - { - last_token_type = token_type::value_string; - break; - } -basic_json_parser_36: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= 'e') - { - if (yych <= '/') - { - if (yych == '"') - { - goto basic_json_parser_31; - } - if (yych <= '.') - { - goto basic_json_parser_33; - } - goto basic_json_parser_31; - } - else - { - if (yych <= '\\') - { - if (yych <= '[') - { - goto basic_json_parser_33; - } - goto basic_json_parser_31; - } - else - { - if (yych == 'b') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - } - } - else - { - if (yych <= 'q') - { - if (yych <= 'f') - { - goto basic_json_parser_31; - } - if (yych == 'n') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 's') - { - if (yych <= 'r') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 't') - { - goto basic_json_parser_31; - } - if (yych <= 'u') - { - goto basic_json_parser_43; - } - goto basic_json_parser_33; - } - } - } -basic_json_parser_37: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_44; - } - goto basic_json_parser_33; -basic_json_parser_38: - yych = *++m_cursor; - if (yych <= ',') - { - if (yych == '+') - { - goto basic_json_parser_46; - } - goto basic_json_parser_33; - } - else - { - if (yych <= '-') - { - goto basic_json_parser_46; - } - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_47; - } - goto basic_json_parser_33; - } -basic_json_parser_39: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_49; - } - goto basic_json_parser_33; -basic_json_parser_40: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_50; - } - goto basic_json_parser_33; -basic_json_parser_41: - yych = *++m_cursor; - if (yych == 'u') - { - goto basic_json_parser_51; - } - goto basic_json_parser_33; -basic_json_parser_42: - yych = *++m_cursor; - if (yych == 0xBF) - { - goto basic_json_parser_52; - } - goto basic_json_parser_33; -basic_json_parser_43: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_54; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_54; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_54; - } - goto basic_json_parser_33; - } -basic_json_parser_44: - yyaccept = 1; - m_marker = ++m_cursor; - if ((m_limit - m_cursor) < 3) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= 'D') - { - if (yych <= '/') - { - goto basic_json_parser_14; - } - if (yych <= '9') - { - goto basic_json_parser_44; - } - goto basic_json_parser_14; - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - goto basic_json_parser_14; - } -basic_json_parser_46: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych >= ':') - { - goto basic_json_parser_33; - } -basic_json_parser_47: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '/') - { - goto basic_json_parser_14; - } - if (yych <= '9') - { - goto basic_json_parser_47; - } - goto basic_json_parser_14; -basic_json_parser_49: - yych = *++m_cursor; - if (yych == 's') - { - goto basic_json_parser_55; - } - goto basic_json_parser_33; -basic_json_parser_50: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_56; - } - goto basic_json_parser_33; -basic_json_parser_51: - yych = *++m_cursor; - if (yych == 'e') - { - goto basic_json_parser_58; - } - goto basic_json_parser_33; -basic_json_parser_52: - ++m_cursor; - { - continue; - } -basic_json_parser_54: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_60; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_60; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_60; - } - goto basic_json_parser_33; - } -basic_json_parser_55: - yych = *++m_cursor; - if (yych == 'e') - { - goto basic_json_parser_61; - } - goto basic_json_parser_33; -basic_json_parser_56: - ++m_cursor; - { - last_token_type = token_type::literal_null; - break; - } -basic_json_parser_58: - ++m_cursor; - { - last_token_type = token_type::literal_true; - break; - } -basic_json_parser_60: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_63; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_63; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_63; - } - goto basic_json_parser_33; - } -basic_json_parser_61: - ++m_cursor; - { - last_token_type = token_type::literal_false; - break; - } -basic_json_parser_63: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_31; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - } - - } - - return last_token_type; - } - - /// append data from the stream to the internal buffer - void yyfill() noexcept - { - if (m_stream == nullptr or not * m_stream) - { - return; - } - - const auto offset_start = m_start - m_content; - const auto offset_marker = m_marker - m_start; - const auto offset_cursor = m_cursor - m_start; - - m_buffer.erase(0, static_cast(offset_start)); - std::string line; - assert(m_stream != nullptr); - std::getline(*m_stream, line); - m_buffer += "\n" + line; // add line with newline symbol - - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_content; - m_marker = m_start + offset_marker; - m_cursor = m_start + offset_cursor; - m_limit = m_start + m_buffer.size() - 1; - } - - /// return string representation of last read token - string_t get_token_string() const - { - assert(m_start != nullptr); - return string_t(reinterpret_cast(m_start), - static_cast(m_cursor - m_start)); - } - - /*! - @brief return string value for string tokens - - The function iterates the characters between the opening and closing - quotes of the string value. The complete string is the range - [m_start,m_cursor). Consequently, we iterate from m_start+1 to - m_cursor-1. - - We differentiate two cases: - - 1. Escaped characters. In this case, a new character is constructed - according to the nature of the escape. Some escapes create new - characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied - as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape - `"\\uxxxx"` need special care. In this case, to_unicode takes care - of the construction of the values. - 2. Unescaped characters are copied as is. - - @pre `m_cursor - m_start >= 2`, meaning the length of the last token - is at least 2 bytes which is trivially true for any string (which - consists of at least two quotes). - - " c1 c2 c3 ... " - ^ ^ - m_start m_cursor - - @complexity Linear in the length of the string.\n - - Lemma: The loop body will always terminate.\n - - Proof (by contradiction): Assume the loop body does not terminate. As - the loop body does not contain another loop, one of the called - functions must never return. The called functions are `std::strtoul` - and to_unicode. Neither function can loop forever, so the loop body - will never loop forever which contradicts the assumption that the loop - body does not terminate, q.e.d.\n - - Lemma: The loop condition for the for loop is eventually false.\n - - Proof (by contradiction): Assume the loop does not terminate. Due to - the above lemma, this can only be due to a tautological loop - condition; that is, the loop condition i < m_cursor - 1 must always be - true. Let x be the change of i for any loop iteration. Then - m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This - can be rephrased to m_cursor - m_start - 2 > x. With the - precondition, we x <= 0, meaning that the loop condition holds - indefinitly if i is always decreased. However, observe that the value - of i is strictly increasing with each iteration, as it is incremented - by 1 in the iteration expression and never decremented inside the loop - body. Hence, the loop condition will eventually be false which - contradicts the assumption that the loop condition is a tautology, - q.e.d. - - @return string value of current token without opening and closing - quotes - @throw std::out_of_range if to_unicode fails - */ - string_t get_string() const - { - assert(m_cursor - m_start >= 2); - - string_t result; - result.reserve(static_cast(m_cursor - m_start - 2)); - - // iterate the result between the quotes - for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) - { - // process escaped characters - if (*i == '\\') - { - // read next character - ++i; - - switch (*i) - { - // the default escapes - case 't': - { - result += "\t"; - break; - } - case 'b': - { - result += "\b"; - break; - } - case 'f': - { - result += "\f"; - break; - } - case 'n': - { - result += "\n"; - break; - } - case 'r': - { - result += "\r"; - break; - } - case '\\': - { - result += "\\"; - break; - } - case '/': - { - result += "/"; - break; - } - case '"': - { - result += "\""; - break; - } - - // unicode - case 'u': - { - // get code xxxx from uxxxx - auto codepoint = std::strtoul(std::string(reinterpret_cast(i + 1), - 4).c_str(), nullptr, 16); - - // check if codepoint is a high surrogate - if (codepoint >= 0xD800 and codepoint <= 0xDBFF) - { - // make sure there is a subsequent unicode - if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') - { - throw std::invalid_argument("missing low surrogate"); - } - - // get code yyyy from uxxxx\uyyyy - auto codepoint2 = std::strtoul(std::string(reinterpret_cast - (i + 7), 4).c_str(), nullptr, 16); - result += to_unicode(codepoint, codepoint2); - // skip the next 10 characters (xxxx\uyyyy) - i += 10; - } - else - { - // add unicode character(s) - result += to_unicode(codepoint); - // skip the next four characters (xxxx) - i += 4; - } - break; - } - } - } - else - { - // all other characters are just copied to the end of the - // string - result.append(1, static_cast(*i)); - } - } - - return result; - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - long double str_to_float_t(long double* /* type */, char** endptr) const - { - return std::strtold(reinterpret_cast(m_start), endptr); - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - double str_to_float_t(double* /* type */, char** endptr) const - { - return std::strtod(reinterpret_cast(m_start), endptr); - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } - - /*! - @brief return number value for number tokens - - This function translates the last token into the most appropriate - number type (either integer, unsigned integer or floating point), - which is passed back to the caller via the result parameter. - - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. - - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). - - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. - */ - void get_number(basic_json& result) const - { - assert(m_start != nullptr); - - const lexer::lexer_char_t* curptr = m_start; - - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; - - // maximum absolute value of the relevant integer type - number_unsigned_t max; - - // temporarily store the type to avoid unecessary bitfield access - value_t type; - - // look for sign - if (*curptr == '-') - { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; - } - else - { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); - } - - // count the significant figures - for (; curptr < m_cursor; curptr++) - { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') - { - if (*curptr == '.') - { - // don't count '.' but change to float - type = value_t::number_float; - continue; - } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; - break; - } - - // skip if definitely not an integer - if (type != value_t::number_float) - { - // multiply last value by ten and add the new digit - auto temp = value * 10 + *curptr - '0'; - - // test for overflow - if (temp < value || temp > max) - { - // overflow - type = value_t::number_float; - } - else - { - // no overflow - save it - value = temp; - } - } - } - - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - result.m_value.number_integer = -static_cast(value); - } - else - { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); - } - - // save the type - result.m_type = type; - } - - private: - /// optional input stream - std::istream* m_stream = nullptr; - /// the buffer - string_t m_buffer; - /// the buffer pointer - const lexer_char_t* m_content = nullptr; - /// pointer to the beginning of the current symbol - const lexer_char_t* m_start = nullptr; - /// pointer for backtracking information - const lexer_char_t* m_marker = nullptr; - /// pointer to the current symbol - const lexer_char_t* m_cursor = nullptr; - /// pointer to the end of the buffer - const lexer_char_t* m_limit = nullptr; - /// the last token type - token_type last_token_type = token_type::end_of_input; - }; - - /*! - @brief syntax analysis - - This class implements a recursive decent parser. - */ - class parser - { - public: - /// constructor for strings - parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(s) - { - // read first token - get_token(); - } - - /// a parser reading from an input stream - parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(&_is) - { - // read first token - get_token(); - } - - /// public parser interface - basic_json parse() - { - basic_json result = parse_internal(true); - result.assert_invariant(); - - expect(lexer::token_type::end_of_input); - - // return parser result and replace it with null in case the - // top-level value was discarded by the callback function - return result.is_discarded() ? basic_json() : std::move(result); - } - - private: - /// the actual parser - basic_json parse_internal(bool keep) - { - auto result = basic_json(value_t::discarded); - - switch (last_token) - { - case lexer::token_type::begin_object: - { - if (keep and (not callback or (keep = callback(depth++, parse_event_t::object_start, result)))) - { - // explicitly set result to object to cope with {} - result.m_type = value_t::object; - result.m_value = value_t::object; - } - - // read next token - get_token(); - - // closing } -> we are done - if (last_token == lexer::token_type::end_object) - { - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - // no comma is expected here - unexpect(lexer::token_type::value_separator); - - // otherwise: parse key-value pairs - do - { - // ugly, but could be fixed with loop reorganization - if (last_token == lexer::token_type::value_separator) - { - get_token(); - } - - // store key - expect(lexer::token_type::value_string); - const auto key = m_lexer.get_string(); - - bool keep_tag = false; - if (keep) - { - if (callback) - { - basic_json k(key); - keep_tag = callback(depth, parse_event_t::key, k); - } - else - { - keep_tag = true; - } - } - - // parse separator (:) - get_token(); - expect(lexer::token_type::name_separator); - - // parse and add value - get_token(); - auto value = parse_internal(keep); - if (keep and keep_tag and not value.is_discarded()) - { - result[key] = std::move(value); - } - } - while (last_token == lexer::token_type::value_separator); - - // closing } - expect(lexer::token_type::end_object); - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result = basic_json(value_t::discarded); - } - - return result; - } - - case lexer::token_type::begin_array: - { - if (keep and (not callback or (keep = callback(depth++, parse_event_t::array_start, result)))) - { - // explicitly set result to object to cope with [] - result.m_type = value_t::array; - result.m_value = value_t::array; - } - - // read next token - get_token(); - - // closing ] -> we are done - if (last_token == lexer::token_type::end_array) - { - get_token(); - if (callback and not callback(--depth, parse_event_t::array_end, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - // no comma is expected here - unexpect(lexer::token_type::value_separator); - - // otherwise: parse values - do - { - // ugly, but could be fixed with loop reorganization - if (last_token == lexer::token_type::value_separator) - { - get_token(); - } - - // parse value - auto value = parse_internal(keep); - if (keep and not value.is_discarded()) - { - result.push_back(std::move(value)); - } - } - while (last_token == lexer::token_type::value_separator); - - // closing ] - expect(lexer::token_type::end_array); - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) - { - result = basic_json(value_t::discarded); - } - - return result; - } - - case lexer::token_type::literal_null: - { - get_token(); - result.m_type = value_t::null; - break; - } - - case lexer::token_type::value_string: - { - const auto s = m_lexer.get_string(); - get_token(); - result = basic_json(s); - break; - } - - case lexer::token_type::literal_true: - { - get_token(); - result.m_type = value_t::boolean; - result.m_value = true; - break; - } - - case lexer::token_type::literal_false: - { - get_token(); - result.m_type = value_t::boolean; - result.m_value = false; - break; - } - - case lexer::token_type::value_number: - { - m_lexer.get_number(result); - get_token(); - break; - } - - default: - { - // the last token was unexpected - unexpect(last_token); - } - } - - if (keep and callback and not callback(depth, parse_event_t::value, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - /// get next token from lexer - typename lexer::token_type get_token() noexcept - { - last_token = m_lexer.scan(); - return last_token; - } - - void expect(typename lexer::token_type t) const - { - if (t != last_token) - { - std::string error_msg = "parse error - unexpected "; - error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + - "'") : - lexer::token_type_name(last_token)); - error_msg += "; expected " + lexer::token_type_name(t); - throw std::invalid_argument(error_msg); - } - } - - void unexpect(typename lexer::token_type t) const - { - if (t == last_token) - { - std::string error_msg = "parse error - unexpected "; - error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + - "'") : - lexer::token_type_name(last_token)); - throw std::invalid_argument(error_msg); - } - } - - private: - /// current level of recursion - int depth = 0; - /// callback function - const parser_callback_t callback = nullptr; - /// the type of the last read token - typename lexer::token_type last_token = lexer::token_type::uninitialized; - /// the lexer - lexer m_lexer; - }; - - public: - /*! - @brief JSON Pointer - - A JSON pointer defines a string syntax for identifying a specific value - within a JSON document. It can be used with functions `at` and - `operator[]`. Furthermore, JSON pointers are the base for JSON patches. - - @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - - @since version 2.0.0 - */ - class json_pointer - { - /// allow basic_json to access private members - friend class basic_json; - - public: - /*! - @brief create JSON pointer - - Create a JSON pointer according to the syntax described in - [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). - - @param[in] s string representing the JSON pointer; if omitted, the - empty string is assumed which references the whole JSON - value - - @throw std::domain_error if reference token is nonempty and does not - begin with a slash (`/`); example: `"JSON pointer must be empty or - begin with /"` - @throw std::domain_error if a tilde (`~`) is not followed by `0` - (representing `~`) or `1` (representing `/`); example: `"escape error: - ~ must be followed with 0 or 1"` - - @liveexample{The example shows the construction several valid JSON - pointers as well as the exceptional behavior.,json_pointer} - - @since version 2.0.0 - */ - explicit json_pointer(const std::string& s = "") - : reference_tokens(split(s)) - {} - - /*! - @brief return a string representation of the JSON pointer - - @invariant For each JSON pointer `ptr`, it holds: - @code {.cpp} - ptr == json_pointer(ptr.to_string()); - @endcode - - @return a string representation of the JSON pointer - - @liveexample{The example shows the result of `to_string`., - json_pointer__to_string} - - @since version 2.0.0 - */ - std::string to_string() const noexcept - { - return std::accumulate(reference_tokens.begin(), - reference_tokens.end(), std::string{}, - [](const std::string & a, const std::string & b) - { - return a + "/" + escape(b); - }); - } - - /// @copydoc to_string() - operator std::string() const - { - return to_string(); - } - - private: - /// remove and return last reference pointer - std::string pop_back() - { - if (is_root()) - { - throw std::domain_error("JSON pointer has no parent"); - } - - auto last = reference_tokens.back(); - reference_tokens.pop_back(); - return last; - } - - /// return whether pointer points to the root document - bool is_root() const - { - return reference_tokens.empty(); - } - - json_pointer top() const - { - if (is_root()) - { - throw std::domain_error("JSON pointer has no parent"); - } - - json_pointer result = *this; - result.reference_tokens = {reference_tokens[0]}; - return result; - } - - /*! - @brief create and return a reference to the pointed to value - - @complexity Linear in the number of reference tokens. - */ - reference get_and_create(reference j) const - { - pointer result = &j; - - // in case no reference tokens exist, return a reference to the - // JSON value j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) - { - switch (result->m_type) - { - case value_t::null: - { - if (reference_token == "0") - { - // start a new array if reference token is 0 - result = &result->operator[](0); - } - else - { - // start a new object otherwise - result = &result->operator[](reference_token); - } - break; - } - - case value_t::object: - { - // create an entry in the object - result = &result->operator[](reference_token); - break; - } - - case value_t::array: - { - // create an entry in the array - result = &result->operator[](static_cast(std::stoi(reference_token))); - break; - } - - /* - The following code is only reached if there exists a - reference token _and_ the current value is primitive. In - this case, we have an error situation, because primitive - values may only occur as single value; that is, with an - empty list of reference tokens. - */ - default: - { - throw std::domain_error("invalid value to unflatten"); - } - } - } - - return *result; - } - - /*! - @brief return a reference to the pointed to value - - @param[in] ptr a JSON value - - @return reference to the JSON value pointed to by the JSON pointer - - @complexity Linear in the length of the JSON pointer. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - */ - reference get_unchecked(pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case value_t::array: - { - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - if (reference_token == "-") - { - // explicityly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); - } - else - { - // convert array index to number; unchecked access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); - } - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - reference get_checked(pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - /*! - @brief return a const reference to the pointed to value - - @param[in] ptr a JSON value - - @return const reference to the JSON value pointed to by the JSON - pointer - */ - const_reference get_unchecked(const_pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" cannot be used for const access - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // use unchecked array access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - const_reference get_checked(const_pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - /// split the string input to reference tokens - static std::vector split(std::string reference_string) - { - std::vector result; - - // special case: empty reference string -> no reference tokens - if (reference_string.empty()) - { - return result; - } - - // check if nonempty reference string begins with slash - if (reference_string[0] != '/') - { - throw std::domain_error("JSON pointer must be empty or begin with '/'"); - } - - // extract the reference tokens: - // - slash: position of the last read slash (or end of string) - // - start: position after the previous slash - for ( - // search for the first slash after the first character - size_t slash = reference_string.find_first_of("/", 1), - // set the beginning of the first reference token - start = 1; - // we can stop if start == string::npos+1 = 0 - start != 0; - // set the beginning of the next reference token - // (will eventually be 0 if slash == std::string::npos) - start = slash + 1, - // find next slash - slash = reference_string.find_first_of("/", start)) - { - // use the text between the beginning of the reference token - // (start) and the last slash (slash). - auto reference_token = reference_string.substr(start, slash - start); - - // check reference tokens are properly escaped - for (size_t pos = reference_token.find_first_of("~"); - pos != std::string::npos; - pos = reference_token.find_first_of("~", pos + 1)) - { - assert(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (pos == reference_token.size() - 1 or - (reference_token[pos + 1] != '0' and - reference_token[pos + 1] != '1')) - { - throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); - } - } - - // finally, store the reference token - unescape(reference_token); - result.push_back(reference_token); - } - - return result; - } - - private: - /*! - @brief replace all occurrences of a substring by another string - - @param[in,out] s the string to manipulate - @param[in] f the substring to replace with @a t - @param[in] t the string to replace @a f - - @return The string @a s where all occurrences of @a f are replaced - with @a t. - - @pre The search string @a f must not be empty. - - @since version 2.0.0 - */ - static void replace_substring(std::string& s, - const std::string& f, - const std::string& t) - { - assert(not f.empty()); - - for ( - size_t pos = s.find(f); // find first occurrence of f - pos != std::string::npos; // make sure f was found - s.replace(pos, f.size(), t), // replace with t - pos = s.find(f, pos + t.size()) // find next occurrence of f - ); - } - - /// escape tilde and slash - static std::string escape(std::string s) - { - // escape "~"" to "~0" and "/" to "~1" - replace_substring(s, "~", "~0"); - replace_substring(s, "/", "~1"); - return s; - } - - /// unescape tilde and slash - static void unescape(std::string& s) - { - // first transform any occurrence of the sequence '~1' to '/' - replace_substring(s, "~1", "/"); - // then transform any occurrence of the sequence '~0' to '~' - replace_substring(s, "~0", "~"); - } - - /*! - @param[in] reference_string the reference string to the current value - @param[in] value the value to consider - @param[in,out] result the result object to insert values to - - @note Empty objects or arrays are flattened to `null`. - */ - static void flatten(const std::string& reference_string, - const basic_json& value, - basic_json& result) - { - switch (value.m_type) - { - case value_t::array: - { - if (value.m_value.array->empty()) - { - // flatten empty array as null - result[reference_string] = nullptr; - } - else - { - // iterate array and use index as reference string - for (size_t i = 0; i < value.m_value.array->size(); ++i) - { - flatten(reference_string + "/" + std::to_string(i), - value.m_value.array->operator[](i), result); - } - } - break; - } - - case value_t::object: - { - if (value.m_value.object->empty()) - { - // flatten empty object as null - result[reference_string] = nullptr; - } - else - { - // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) - { - flatten(reference_string + "/" + escape(element.first), - element.second, result); - } - } - break; - } - - default: - { - // add primitive value with its reference string - result[reference_string] = value; - break; - } - } - } - - /*! - @param[in] value flattened JSON - - @return unflattened JSON - */ - static basic_json unflatten(const basic_json& value) - { - if (not value.is_object()) - { - throw std::domain_error("only objects can be unflattened"); - } - - basic_json result; - - // iterate the JSON object values - for (const auto& element : *value.m_value.object) - { - if (not element.second.is_primitive()) - { - throw std::domain_error("values in object must be primitive"); - } - - // assign value to reference pointed to by JSON pointer; Note - // that if the JSON pointer is "" (i.e., points to the whole - // value), function get_and_create returns a reference to - // result itself. An assignment will then create a primitive - // value. - json_pointer(element.first).get_and_create(result) = element.second; - } - - return result; - } - - private: - /// the reference tokens - std::vector reference_tokens {}; - }; - - ////////////////////////// - // JSON Pointer support // - ////////////////////////// - - /// @name JSON Pointer functions - /// @{ - - /*! - @brief access specified element via JSON Pointer - - Uses a JSON pointer to retrieve a reference to the respective JSON value. - No bound checking is performed. Similar to @ref operator[](const typename - object_t::key_type&), `null` values are created in arrays and objects if - necessary. - - In particular: - - If the JSON pointer points to an object key that does not exist, it - is created an filled with a `null` value before a reference to it - is returned. - - If the JSON pointer points to an array index that does not exist, it - is created an filled with a `null` value before a reference to it - is returned. All indices between the current maximum and the given - index are also filled with `null`. - - The special value `-` is treated as a synonym for the index past the - end. - - @param[in] ptr a JSON pointer - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,operatorjson_pointer} - - @since version 2.0.0 - */ - reference operator[](const json_pointer& ptr) - { - return ptr.get_unchecked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Uses a JSON pointer to retrieve a reference to the respective JSON value. - No bound checking is performed. The function does not change the JSON - value; no `null` values are created. In particular, the the special value - `-` yields an exception. - - @param[in] ptr JSON pointer to the desired element - - @return const reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} - - @since version 2.0.0 - */ - const_reference operator[](const json_pointer& ptr) const - { - return ptr.get_unchecked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Returns a reference to the element at with specified JSON pointer @a ptr, - with bounds checking. - - @param[in] ptr JSON pointer to the desired element - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,at_json_pointer} - - @since version 2.0.0 - */ - reference at(const json_pointer& ptr) - { - return ptr.get_checked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Returns a const reference to the element at with specified JSON pointer @a - ptr, with bounds checking. - - @param[in] ptr JSON pointer to the desired element - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,at_json_pointer_const} - - @since version 2.0.0 - */ - const_reference at(const json_pointer& ptr) const - { - return ptr.get_checked(this); - } - - /*! - @brief return flattened JSON value - - The function creates a JSON object whose keys are JSON pointers (see [RFC - 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all - primitive. The original JSON value can be restored using the @ref - unflatten() function. - - @return an object that maps JSON pointers to primitve values - - @note Empty objects and arrays are flattened to `null` and will not be - reconstructed correctly by the @ref unflatten() function. - - @complexity Linear in the size the JSON value. - - @liveexample{The following code shows how a JSON object is flattened to an - object whose keys consist of JSON pointers.,flatten} - - @sa @ref unflatten() for the reverse function - - @since version 2.0.0 - */ - basic_json flatten() const - { - basic_json result(value_t::object); - json_pointer::flatten("", *this, result); - return result; - } - - /*! - @brief unflatten a previously flattened JSON value - - The function restores the arbitrary nesting of a JSON value that has been - flattened before using the @ref flatten() function. The JSON value must - meet certain constraints: - 1. The value must be an object. - 2. The keys must be JSON pointers (see - [RFC 6901](https://tools.ietf.org/html/rfc6901)) - 3. The mapped values must be primitive JSON types. - - @return the original JSON from a flattened version - - @note Empty objects and arrays are flattened by @ref flatten() to `null` - values and can not unflattened to their original type. Apart from - this example, for a JSON value `j`, the following is always true: - `j == j.flatten().unflatten()`. - - @complexity Linear in the size the JSON value. - - @liveexample{The following code shows how a flattened JSON object is - unflattened into the original nested JSON object.,unflatten} - - @sa @ref flatten() for the reverse function - - @since version 2.0.0 - */ - basic_json unflatten() const - { - return json_pointer::unflatten(*this); - } - - /// @} - - ////////////////////////// - // JSON Patch functions // - ////////////////////////// - - /// @name JSON Patch functions - /// @{ - - /*! - @brief applies a JSON patch - - [JSON Patch](http://jsonpatch.com) defines a JSON document structure for - expressing a sequence of operations to apply to a JSON) document. With - this funcion, a JSON Patch is applied to the current JSON value by - executing all operations from the patch. - - @param[in] json_patch JSON patch document - @return patched document - - @note The application of a patch is atomic: Either all operations succeed - and the patched document is returned or an exception is thrown. In - any case, the original value is not changed: the patch is applied - to a copy of the value. - - @throw std::out_of_range if a JSON pointer inside the patch could not - be resolved successfully in the current JSON value; example: `"key baz - not found"` - @throw invalid_argument if the JSON patch is malformed (e.g., mandatory - attributes are missing); example: `"operation add must have member path"` - - @complexity Linear in the size of the JSON value and the length of the - JSON patch. As usually only a fraction of the JSON value is affected by - the patch, the complexity can usually be neglected. - - @liveexample{The following code shows how a JSON patch is applied to a - value.,patch} - - @sa @ref diff -- create a JSON patch by comparing two JSON values - - @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) - @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) - - @since version 2.0.0 - */ - basic_json patch(const basic_json& json_patch) const - { - // make a working copy to apply the patch to - basic_json result = *this; - - // the valid JSON Patch operations - enum class patch_operations {add, remove, replace, move, copy, test, invalid}; - - const auto get_op = [](const std::string op) - { - if (op == "add") - { - return patch_operations::add; - } - if (op == "remove") - { - return patch_operations::remove; - } - if (op == "replace") - { - return patch_operations::replace; - } - if (op == "move") - { - return patch_operations::move; - } - if (op == "copy") - { - return patch_operations::copy; - } - if (op == "test") - { - return patch_operations::test; - } - - return patch_operations::invalid; - }; - - // wrapper for "add" operation; add value at ptr - const auto operation_add = [&result](json_pointer & ptr, basic_json val) - { - // adding to the root of the target document means replacing it - if (ptr.is_root()) - { - result = val; - } - else - { - // make sure the top element of the pointer exists - json_pointer top_pointer = ptr.top(); - if (top_pointer != ptr) - { - basic_json& x = result.at(top_pointer); - } - - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result[ptr]; - - switch (parent.m_type) - { - case value_t::null: - case value_t::object: - { - // use operator[] to add value - parent[last_path] = val; - break; - } - - case value_t::array: - { - if (last_path == "-") - { - // special case: append to back - parent.push_back(val); - } - else - { - const auto idx = std::stoi(last_path); - if (static_cast(idx) > parent.size()) - { - // avoid undefined behavior - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - else - { - // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); - } - } - break; - } - - default: - { - // if there exists a parent it cannot be primitive - assert(false); // LCOV_EXCL_LINE - } - } - } - }; - - // wrapper for "remove" operation; remove value at ptr - const auto operation_remove = [&result](json_pointer & ptr) - { - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result.at(ptr); - - // remove child - if (parent.is_object()) - { - // perform range check - auto it = parent.find(last_path); - if (it != parent.end()) - { - parent.erase(it); - } - else - { - throw std::out_of_range("key '" + last_path + "' not found"); - } - } - else if (parent.is_array()) - { - // note erase performs range check - parent.erase(static_cast(std::stoi(last_path))); - } - }; - - // type check - if (not json_patch.is_array()) - { - // a JSON patch must be an array of objects - throw std::invalid_argument("JSON patch must be an array of objects"); - } - - // iterate and apply th eoperations - for (const auto& val : json_patch) - { - // wrapper to get a value for an operation - const auto get_value = [&val](const std::string & op, - const std::string & member, - bool string_type) -> basic_json& - { - // find value - auto it = val.m_value.object->find(member); - - // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; - - // check if desired value is present - if (it == val.m_value.object->end()) - { - throw std::invalid_argument(error_msg + " must have member '" + member + "'"); - } - - // check if result is of type string - if (string_type and not it->second.is_string()) - { - throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); - } - - // no error: return value - return it->second; - }; - - // type check - if (not val.is_object()) - { - throw std::invalid_argument("JSON patch must be an array of objects"); - } - - // collect mandatory members - const std::string op = get_value("op", "op", true); - const std::string path = get_value(op, "path", true); - json_pointer ptr(path); - - switch (get_op(op)) - { - case patch_operations::add: - { - operation_add(ptr, get_value("add", "value", false)); - break; - } - - case patch_operations::remove: - { - operation_remove(ptr); - break; - } - - case patch_operations::replace: - { - // the "path" location must exist - use at() - result.at(ptr) = get_value("replace", "value", false); - break; - } - - case patch_operations::move: - { - const std::string from_path = get_value("move", "from", true); - json_pointer from_ptr(from_path); - - // the "from" location must exist - use at() - basic_json v = result.at(from_ptr); - - // The move operation is functionally identical to a - // "remove" operation on the "from" location, followed - // immediately by an "add" operation at the target - // location with the value that was just removed. - operation_remove(from_ptr); - operation_add(ptr, v); - break; - } - - case patch_operations::copy: - { - const std::string from_path = get_value("copy", "from", true);; - const json_pointer from_ptr(from_path); - - // the "from" location must exist - use at() - result[ptr] = result.at(from_ptr); - break; - } - - case patch_operations::test: - { - bool success = false; - try - { - // check if "value" matches the one at "path" - // the "path" location must exist - use at() - success = (result.at(ptr) == get_value("test", "value", false)); - } - catch (std::out_of_range&) - { - // ignore out of range errors: success remains false - } - - // throw an exception if test fails - if (not success) - { - throw std::domain_error("unsuccessful: " + val.dump()); - } - - break; - } - - case patch_operations::invalid: - { - // op must be "add", "remove", "replace", "move", "copy", or - // "test" - throw std::invalid_argument("operation value '" + op + "' is invalid"); - } - } - } - - return result; - } - - /*! - @brief creates a diff as a JSON patch - - Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can - be changed into the value @a target by calling @ref patch function. - - @invariant For two JSON values @a source and @a target, the following code - yields always `true`: - @code {.cpp} - source.patch(diff(source, target)) == target; - @endcode - - @note Currently, only `remove`, `add`, and `replace` operations are - generated. - - @param[in] source JSON value to copare from - @param[in] target JSON value to copare against - @param[in] path helper value to create JSON pointers - - @return a JSON patch to convert the @a source to @a target - - @complexity Linear in the lengths of @a source and @a target. - - @liveexample{The following code shows how a JSON patch is created as a - diff for two JSON values.,diff} - - @sa @ref patch -- apply a JSON patch - - @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) - - @since version 2.0.0 - */ - static basic_json diff(const basic_json& source, - const basic_json& target, - std::string path = "") - { - // the patch - basic_json result(value_t::array); - - // if the values are the same, return empty patch - if (source == target) - { - return result; - } - - if (source.type() != target.type()) - { - // different types: replace value - result.push_back( - { - {"op", "replace"}, - {"path", path}, - {"value", target} - }); - } - else - { - switch (source.type()) - { - case value_t::array: - { - // first pass: traverse common elements - size_t i = 0; - while (i < source.size() and i < target.size()) - { - // recursive call to compare array values at index i - auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - ++i; - } - - // i now reached the end of at least one array - // in a second pass, traverse the remaining elements - - // remove my remaining elements - const auto end_index = static_cast(result.size()); - while (i < source.size()) - { - // add operations in reverse order to avoid invalid - // indices - result.insert(result.begin() + end_index, object( - { - {"op", "remove"}, - {"path", path + "/" + std::to_string(i)} - })); - ++i; - } - - // add other remaining elements - while (i < target.size()) - { - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + std::to_string(i)}, - {"value", target[i]} - }); - ++i; - } - - break; - } - - case value_t::object: - { - // first pass: traverse this object's elements - for (auto it = source.begin(); it != source.end(); ++it) - { - // escape the key name to be used in a JSON patch - const auto key = json_pointer::escape(it.key()); - - if (target.find(it.key()) != target.end()) - { - // recursive call to compare object values at key it - auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - } - else - { - // found a key that is not in o -> remove it - result.push_back(object( - { - {"op", "remove"}, - {"path", path + "/" + key} - })); - } - } - - // second pass: traverse other object's elements - for (auto it = target.begin(); it != target.end(); ++it) - { - if (source.find(it.key()) == source.end()) - { - // found a key that is not in this -> add it - const auto key = json_pointer::escape(it.key()); - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + key}, - {"value", it.value()} - }); - } - } - - break; - } - - default: - { - // both primitive type: replace value - result.push_back( - { - {"op", "replace"}, - {"path", path}, - {"value", target} - }); - break; - } - } - } - - return result; - } - - /// @} -}; - - -///////////// -// presets // -///////////// - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; -} - - -/////////////////////// -// nonmember support // -/////////////////////// - -// specialization of std::swap, and std::hash -namespace std -{ -/*! -@brief exchanges the values of two JSON objects - -@since version 1.0.0 -*/ -template <> -inline void swap(nlohmann::json& j1, - nlohmann::json& j2) noexcept( - is_nothrow_move_constructible::value and - is_nothrow_move_assignable::value - ) -{ - j1.swap(j2); -} - -/// hash value for JSON objects -template <> -struct hash -{ - /*! - @brief return a hash value for a JSON object - - @since version 1.0.0 - */ - std::size_t operator()(const nlohmann::json& j) const - { - // a naive hashing via the string representation - const auto& h = hash(); - return h(j.dump()); - } -}; -} - -/*! -@brief user-defined string literal for JSON values - -This operator implements a user-defined string literal for JSON objects. It -can be used by adding `"_json"` to a string literal and returns a JSON object -if no parse error occurred. - -@param[in] s a string representation of a JSON object -@return a JSON object - -@since version 1.0.0 -*/ -inline nlohmann::json operator "" _json(const char* s, std::size_t) -{ - return nlohmann::json::parse(reinterpret_cast(s)); -} - -/*! -@brief user-defined string literal for JSON pointer - -This operator implements a user-defined string literal for JSON Pointers. It -can be used by adding `"_json"` to a string literal and returns a JSON pointer -object if no parse error occurred. - -@param[in] s a string representation of a JSON Pointer -@return a JSON pointer object - -@since version 2.0.0 -*/ -inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t) -{ - return nlohmann::json::json_pointer(s); -} - -// restore GCC/clang diagnostic settings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic pop -#endif - -#endif diff --git a/ext/offbase/README.md b/ext/offbase/README.md new file mode 100644 index 00000000..6668b878 --- /dev/null +++ b/ext/offbase/README.md @@ -0,0 +1,59 @@ +A super-minimal in-filesystem persistent JSON object +====== + +[We](https://www.zerotier.com/) like minimalism. + +Offbase is an extension of the excellent [nlohmann/json](https://github.com/nlohmann/json) C++11 JSON class that adds simple object persistence to/from the filesystem. Objects are stored into a directory hierarchy in fully "expanded" form with each field/value being represented by a separate file. + +Features: + + - Very easy to use + - Minimal! + - Easy to understand and maintain + - Trivial to implement in other languages + - No dependencies beyond standard libraries + - Small code footprint in both source and binary form + - Easy to port to other platforms + - Exactly reproduces JSON object hierarchies including all JSON type information + - Database can be explored from the shell, browsed in a web browser or file explorer, scanned with `find` and `grep`, etc. + - Database can be backed up, restored, versioned, etc. with tools like `git`, `rsync`, `duplicity`, etc. + - Alien files like `.git` or `.DS_Store` are harmlessly ignored if present + - Saving only changes what's changed to reduce I/O overhead and SSD wear + +Limitations and shortcomings: + + - This creates a lot of tiny files, which is inefficient on some filesystems and might run into inode limits in extreme cases. For data sets with more than, say, a million items we recommend a filesystem like `btrfs` or `reiserfs`. Things like [redisfs](https://steve.fi/Software/redisfs/) are also worth exploring. On Linux another alternative is to put the database into `/dev/shm` (RAM disk) and then regularly back it up with `duplicity` or similar. + - The whole JSON object is held in memory *twice* for diffing purposes. + - Diffing traverses the whole tree and then updates the shadow copy, which makes `commit()` slow for huge data sets. This is not suitable for "big" data where "big" here is probably more than a few hundred megabytes. + - Recursion is used, so if you have object hierarchies that are incredibly deep (hundreds or more) it might be possible to overflow your stack and crash your app. + - This is not thread safe and must be guarded by a mutex if used in a threaded app. + +Caveats: + + - Key names are escaped for safety in the filesystem, but we still don't recommend allowing external users to set just anything into your JSON store. See the point about recursion under limitations. + +Future: + + - It would not be too hard to tie this into a filesystem change monitoring API and automatically read changes from disk if they are detected. This would allow the database to be edited "live" in the filesystem. + - In theory this could provide replication or clustering through distributed filesystems, file syncing, or things like [Amazon Elastic Filesystem](https://aws.amazon.com/efs/). + - Recursion could be factored out to get rid of any object hierarchy depth constraints. + - Mutexes could be integrated somehow to allow for finer grained locking in multithreaded apps. + - Diffing and selective updates could be made more memory and CPU efficient using hashes, etc. + +## How to Use + +The `offbase` class just extends [nlohmann::json](https://github.com/nlohmann/json) and gives you a JSON object. Take care to make sure you don't change the type of the 'root' object represented by the 'offbase' instance from JSON 'object'. Anything under it can of course be any JSON type, including any object. + +Just put data into the object and then periodically call `commit()` to persist changes to disk. The `commit()` method diffs the current contents of the object with what it knows to have been previously persisted to disk and modifies the representation on disk to match. This can be done after writes or periodically in a background thread. + +See comments in `offbase.hpp` for full documentation including details about error handling, etc. + +## Persistence format + +The base object represented by the `offbase` instance is persisted into a directory hierarchy under its base path. Files and directories are named according to a simple convention of `keyname.typecode` where `keyname` is an escaped key name (or hex array index in the case of arrays) and `typecode` is a single character indicating whether the item is a JSON value, array, or object. + + - `*.V`: JSON values (actual value type is inferred during JSON parse) + - `*.O`: JSON objects (these are subdirectories) + - `*.A`: JSON arrays (also subdirectories containing items by hex array index) + +There are in theory simpler ways to represent JSON in a filesystem, such as the "flattened" JSON "pointer" format, but this has the disadvantage of not disambiguating objects vs. arrays. Offbase's persistence format is designed to perfectly reproduce the exact same JSON tree on load as was most recently committed. diff --git a/ext/offbase/json/LICENSE.MIT b/ext/offbase/json/LICENSE.MIT new file mode 100644 index 00000000..e2ac4891 --- /dev/null +++ b/ext/offbase/json/LICENSE.MIT @@ -0,0 +1,22 @@ +The library is licensed under the MIT License +: + +Copyright (c) 2013-2016 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ext/offbase/json/README.md b/ext/offbase/json/README.md new file mode 100644 index 00000000..c0bb61b1 --- /dev/null +++ b/ext/offbase/json/README.md @@ -0,0 +1,511 @@ +[![JSON for Modern C++](https://raw.githubusercontent.com/nlohmann/json/master/doc/json.gif)](https://github.com/nlohmann/json/releases) + +[![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) +[![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk?svg=true)](https://ci.appveyor.com/project/nlohmann/json) +[![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) +[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) +[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/p5o4znPnGHJpDVqN) +[![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) +[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) +[![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) +[![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) + +## Design goals + +There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: + +- **Intuitive syntax**. In languages such as Python, JSON feels like a first class data type. We used all the operator magic of modern C++ to achieve the same feeling in your code. Check out the [examples below](#examples) and you know, what I mean. + +- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/src/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. + +- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/blob/master/test/src/unit.cpp) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) that there are no memory leaks. + +Other aspects were not so important to us: + +- **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. + +- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). + +See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. + + +## Integration + +The single required source, file `json.hpp` is in the `src` directory or [released here](https://github.com/nlohmann/json/releases). All you need to do is add + +```cpp +#include "json.hpp" + +// for convenience +using json = nlohmann::json; +``` + +to the files you want to use JSON objects. That's it. Do not forget to set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). + +:beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. + + +## Examples + +Here are some examples to give you an idea how to use the class. + +Assume you want to create the JSON object + +```json +{ + "pi": 3.141, + "happy": true, + "name": "Niels", + "nothing": null, + "answer": { + "everything": 42 + }, + "list": [1, 0, 2], + "object": { + "currency": "USD", + "value": 42.99 + } +} +``` + +With the JSON class, you could write: + +```cpp +// create an empty structure (null) +json j; + +// add a number that is stored as double (note the implicit conversion of j to an object) +j["pi"] = 3.141; + +// add a Boolean that is stored as bool +j["happy"] = true; + +// add a string that is stored as std::string +j["name"] = "Niels"; + +// add another null object by passing nullptr +j["nothing"] = nullptr; + +// add an object inside the object +j["answer"]["everything"] = 42; + +// add an array that is stored as std::vector (using an initializer list) +j["list"] = { 1, 0, 2 }; + +// add another object (using an initializer list of pairs) +j["object"] = { {"currency", "USD"}, {"value", 42.99} }; + +// instead, you could also write (which looks very similar to the JSON above) +json j2 = { + {"pi", 3.141}, + {"happy", true}, + {"name", "Niels"}, + {"nothing", nullptr}, + {"answer", { + {"everything", 42} + }}, + {"list", {1, 0, 2}}, + {"object", { + {"currency", "USD"}, + {"value", 42.99} + }} +}; +``` + +Note that in all these cases, you never need to "tell" the compiler which JSON value you want to use. If you want to be explicit or express some edge cases, the functions `json::array` and `json::object` will help: + +```cpp +// a way to express the empty array [] +json empty_array_explicit = json::array(); + +// ways to express the empty object {} +json empty_object_implicit = json({}); +json empty_object_explicit = json::object(); + +// a way to express an _array_ of key/value pairs [["currency", "USD"], ["value", 42.99]] +json array_not_object = { json::array({"currency", "USD"}), json::array({"value", 42.99}) }; +``` + + +### Serialization / Deserialization + +You can create an object (deserialization) by appending `_json` to a string literal: + +```cpp +// create object from string literal +json j = "{ \"happy\": true, \"pi\": 3.141 }"_json; + +// or even nicer with a raw string literal +auto j2 = R"( + { + "happy": true, + "pi": 3.141 + } +)"_json; + +// or explicitly +auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); +``` + +You can also get a string representation (serialize): + +```cpp +// explicit conversion to string +std::string s = j.dump(); // {\"happy\":true,\"pi\":3.141} + +// serialization with pretty printing +// pass in the amount of spaces to indent +std::cout << j.dump(4) << std::endl; +// { +// "happy": true, +// "pi": 3.141 +// } +``` + +You can also use streams to serialize and deserialize: + +```cpp +// deserialize from standard input +json j; +std::cin >> j; + +// serialize to standard output +std::cout << j; + +// the setw manipulator was overloaded to set the indentation for pretty printing +std::cout << std::setw(4) << j << std::endl; +``` + +These operators work for any subclasses of `std::istream` or `std::ostream`. + +Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. + + +### STL-like access + +We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) requirement. + +```cpp +// create an array using push_back +json j; +j.push_back("foo"); +j.push_back(1); +j.push_back(true); + +// iterate the array +for (json::iterator it = j.begin(); it != j.end(); ++it) { + std::cout << *it << '\n'; +} + +// range-based for +for (auto& element : j) { + std::cout << element << '\n'; +} + +// getter/setter +const std::string tmp = j[0]; +j[1] = 42; +bool foo = j.at(2); + +// other stuff +j.size(); // 3 entries +j.empty(); // false +j.type(); // json::value_t::array +j.clear(); // the array is empty again + +// convenience type checkers +j.is_null(); +j.is_boolean(); +j.is_number(); +j.is_object(); +j.is_array(); +j.is_string(); + +// comparison +j == "[\"foo\", 1, true]"_json; // true + +// create an object +json o; +o["foo"] = 23; +o["bar"] = false; +o["baz"] = 3.141; + +// special iterator member functions for objects +for (json::iterator it = o.begin(); it != o.end(); ++it) { + std::cout << it.key() << " : " << it.value() << "\n"; +} + +// find an entry +if (o.find("foo") != o.end()) { + // there is an entry with key "foo" +} + +// or simpler using count() +int foo_present = o.count("foo"); // 1 +int fob_present = o.count("fob"); // 0 + +// delete an entry +o.erase("foo"); +``` + + +### Conversion from STL containers + +Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON types (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends how the elements are ordered in the respective STL container. + +```cpp +std::vector c_vector {1, 2, 3, 4}; +json j_vec(c_vector); +// [1, 2, 3, 4] + +std::deque c_deque {1.2, 2.3, 3.4, 5.6}; +json j_deque(c_deque); +// [1.2, 2.3, 3.4, 5.6] + +std::list c_list {true, true, false, true}; +json j_list(c_list); +// [true, true, false, true] + +std::forward_list c_flist {12345678909876, 23456789098765, 34567890987654, 45678909876543}; +json j_flist(c_flist); +// [12345678909876, 23456789098765, 34567890987654, 45678909876543] + +std::array c_array {{1, 2, 3, 4}}; +json j_array(c_array); +// [1, 2, 3, 4] + +std::set c_set {"one", "two", "three", "four", "one"}; +json j_set(c_set); // only one entry for "one" is used +// ["four", "one", "three", "two"] + +std::unordered_set c_uset {"one", "two", "three", "four", "one"}; +json j_uset(c_uset); // only one entry for "one" is used +// maybe ["two", "three", "four", "one"] + +std::multiset c_mset {"one", "two", "one", "four"}; +json j_mset(c_mset); // only one entry for "one" is used +// maybe ["one", "two", "four"] + +std::unordered_multiset c_umset {"one", "two", "one", "four"}; +json j_umset(c_umset); // both entries for "one" are used +// maybe ["one", "two", "one", "four"] +``` + +Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON types (see examples above) can be used to to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. + +```cpp +std::map c_map { {"one", 1}, {"two", 2}, {"three", 3} }; +json j_map(c_map); +// {"one": 1, "three": 3, "two": 2 } + +std::unordered_map c_umap { {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; +json j_umap(c_umap); +// {"one": 1.2, "two": 2.3, "three": 3.4} + +std::multimap c_mmap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; +json j_mmap(c_mmap); // only one entry for key "three" is used +// maybe {"one": true, "two": true, "three": true} + +std::unordered_multimap c_ummap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; +json j_ummap(c_ummap); // only one entry for key "three" is used +// maybe {"one": true, "two": true, "three": true} +``` + +### JSON Pointer and JSON Patch + +The library supports **JSON Pointer** ([RFC 6901](https://tools.ietf.org/html/rfc6901)) as alternative means to address structured values. On top of this, **JSON Patch** ([RFC 6902](https://tools.ietf.org/html/rfc6902)) allows to describe differences between two JSON values - effectively allowing patch and diff operations known from Unix. + +```cpp +// a JSON value +json j_original = R"({ + "baz": ["one", "two", "three"], + "foo": "bar" +})"_json; + +// access members with a JSON pointer (RFC 6901) +j_original["/baz/1"_json_pointer]; +// "two" + +// a JSON patch (RFC 6902) +json j_patch = R"([ + { "op": "replace", "path": "/baz", "value": "boo" }, + { "op": "add", "path": "/hello", "value": ["world"] }, + { "op": "remove", "path": "/foo"} +])"_json; + +// apply the patch +json j_result = j_original.patch(j_patch); +// { +// "baz": "boo", +// "hello": ["world"] +// } + +// calculate a JSON patch from two JSON values +json::diff(j_result, j_original); +// [ +// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, +// { "op": "remove","path": "/hello" }, +// { "op": "add", "path": "/foo", "value": "bar" } +// ] +``` + + +### Implicit conversions + +The type of the JSON object is determined automatically by the expression to store. Likewise, the stored value is implicitly converted. + +```cpp +// strings +std::string s1 = "Hello, world!"; +json js = s1; +std::string s2 = js; + +// Booleans +bool b1 = true; +json jb = b1; +bool b2 = jb; + +// numbers +int i = 42; +json jn = i; +double f = jn; + +// etc. +``` + +You can also explicitly ask for the value: + +```cpp +std::string vs = js.get(); +bool vb = jb.get(); +int vi = jn.get(); + +// etc. +``` + + +## Supported compilers + +Though it's 2016 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: + +- GCC 4.9 - 6.0 (and possibly later) +- Clang 3.4 - 3.9 (and possibly later) +- Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) + +I would be happy to learn about other compilers/versions. + +Please note: + +- GCC 4.8 does not work because of two bugs ([55817](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55817) and [57824](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57824)) in the C++11 support. Note there is a [pull request](https://github.com/nlohmann/json/pull/212) to fix some of the issues. +- Android defaults to using very old compilers and C++ libraries. To fix this, add the following to your `Application.mk`. This will switch to the LLVM C++ library, the Clang compiler, and enable C++11 and other features disabled by default. + + ``` + APP_STL := c++_shared + NDK_TOOLCHAIN_VERSION := clang3.6 + APP_CPPFLAGS += -frtti -fexceptions + ``` + + The code compiles successfully with [Android NDK](https://developer.android.com/ndk/index.html?hl=ml), Revision 9 - 11 (and possibly later) and [CrystaX's Android NDK](https://www.crystax.net/en/android/ndk) version 10. + +- For GCC running on MinGW or Android SDK, the error `'to_string' is not a member of 'std'` (or similarly, for `strtod`) may occur. Note this is not an issue with the code, but rather with the compiler itself. On Android, see above to build with a newer environment. For MinGW, please refer to [this site](http://tehsausage.com/mingw-to-string) and [this discussion](https://github.com/nlohmann/json/issues/136) for information on how to fix this bug. For Android NDK using `APP_STL := gnustl_static`, please refer to [this discussion](https://github.com/nlohmann/json/issues/219). + +The following compilers are currently used in continuous integration at [Travis](https://travis-ci.org/nlohmann/json) and [AppVeyor](https://ci.appveyor.com/project/nlohmann/json): + +| Compiler | Operating System | Version String | +|-----------------|------------------------------|----------------| +| GCC 4.9.3 | Ubuntu 14.04.4 LTS | g++-4.9 (Ubuntu 4.9.3-8ubuntu2~14.04) 4.9.3 | +| GCC 5.3.0 | Ubuntu 14.04.4 LTS | g++-5 (Ubuntu 5.3.0-3ubuntu1~14.04) 5.3.0 20151204 | +| GCC 6.1.1 | Ubuntu 14.04.4 LTS | g++-6 (Ubuntu 6.1.1-3ubuntu11~14.04.1) 6.1.1 20160511 | +| Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | +| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | +| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | +| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | +| Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | +| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | +| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | +| Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | +| Clang Xcode 8.0 | Darwin Kernel Version 15.5.0 (OSX 10.11.5) | Apple LLVM version 8.0.0 (clang-800.0.24.1) | +| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | + + +## License + + + +The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): + +Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +## Thanks + +I deeply appreciate the help of the following people. + +- [Teemperor](https://github.com/Teemperor) implemented CMake support and lcov integration, realized escape and Unicode handling in the string parser, and fixed the JSON serialization. +- [elliotgoodrich](https://github.com/elliotgoodrich) fixed an issue with double deletion in the iterator classes. +- [kirkshoop](https://github.com/kirkshoop) made the iterators of the class composable to other libraries. +- [wancw](https://github.com/wanwc) fixed a bug that hindered the class to compile with Clang. +- Tomas Åblad found a bug in the iterator implementation. +- [Joshua C. Randall](https://github.com/jrandall) fixed a bug in the floating-point serialization. +- [Aaron Burghardt](https://github.com/aburgh) implemented code to parse streams incrementally. Furthermore, he greatly improved the parser class by allowing the definition of a filter function to discard undesired elements while parsing. +- [Daniel Kopeček](https://github.com/dkopecek) fixed a bug in the compilation with GCC 5.0. +- [Florian Weber](https://github.com/Florianjw) fixed a bug in and improved the performance of the comparison operators. +- [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. +- [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. +- [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. +- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. +- [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. +- [dariomt](https://github.com/dariomt) fixed some typos in the examples. +- [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. +- [Colin Hirsch](https://github.com/ColinH) took care of a small namespace issue. +- [Huu Nguyen](https://github.com/whoshuu) correct a variable name in the documentation. +- [Silverweed](https://github.com/silverweed) overloaded `parse()` to accept an rvalue reference. +- [dariomt](https://github.com/dariomt) fixed a subtlety in MSVC type support and implemented the `get_ref()` function to get a reference to stored values. +- [ZahlGraf](https://github.com/ZahlGraf) added a workaround that allows compilation using Android NDK. +- [whackashoe](https://github.com/whackashoe) replaced a function that was marked as unsafe by Visual Studio. +- [406345](https://github.com/406345) fixed two small warnings. +- [Glen Fernandes](https://github.com/glenfe) noted a potential portability problem in the `has_mapped_type` function. +- [Corbin Hughes](https://github.com/nibroc) fixed some typos in the contribution guidelines. +- [twelsby](https://github.com/twelsby) fixed the array subscript operator, an issue that failed the MSVC build, and floating-point parsing/dumping. He further added support for unsigned integer numbers and implemented better roundtrip support for parsed numbers. +- [Volker Diels-Grabsch](https://github.com/vog) fixed a link in the README file. +- [msm-](https://github.com/msm-) added support for american fuzzy lop. +- [Annihil](https://github.com/Annihil) fixed an example in the README file. +- [Themercee](https://github.com/Themercee) noted a wrong URL in the README file. +- [Lv Zheng](https://github.com/lv-zheng) fixed a namespace issue with `int64_t` and `uint64_t`. +- [abc100m](https://github.com/abc100m) analyzed the issues with GCC 4.8 and proposed a [partial solution](https://github.com/nlohmann/json/pull/212). +- [zewt](https://github.com/zewt) added useful notes to the README file about Android. +- [Róbert Márki](https://github.com/robertmrk) added a fix to use move iterators and improved the integration via CMake. +- [Chris Kitching](https://github.com/ChrisKitching) cleaned up the CMake files. +- [Tom Needham](https://github.com/06needhamt) fixed a subtle bug with MSVC 2015 which was also proposed by [Michael K.](https://github.com/Epidal). +- [Mário Feroldi](https://github.com/thelostt) fixed a small typo. +- [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. +- [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. + +Thanks a lot for helping out! + + +## Notes + +- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). +- As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions. + + +## Execute unit tests + +To compile and run the tests, you need to execute + +```sh +$ make +$ ./json_unit "*" + +=============================================================================== +All tests passed (8905012 assertions in 32 test cases) +``` + +For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). diff --git a/ext/offbase/json/json.hpp b/ext/offbase/json/json.hpp new file mode 100644 index 00000000..878fb899 --- /dev/null +++ b/ext/offbase/json/json.hpp @@ -0,0 +1,10435 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 2.0.2 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +Copyright (c) 2013-2016 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// exclude unsupported compilers +#if defined(__clang__) + #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) + #if CLANG_VERSION < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#elif defined(__GNUC__) + #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) + #if GCC_VERSION < 40900 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + + +/*! +@brief unnamed namespace with internal helper functions +@since version 1.0.0 +*/ +namespace +{ +/*! +@brief Helper to determine whether there's a key_type for T. + +Thus helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. + +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0 +*/ +template +struct has_mapped_type +{ + private: + template static char test(typename C::mapped_type*); + template static char (&test(...))[2]; + public: + static constexpr bool value = sizeof(test(0)) == 1; +}; + +/*! +@brief helper class to create locales with decimal point + +This struct is used a default locale during the JSON serialization. JSON +requires the decimal point to be `.`, so this function overloads the +`do_decimal_point()` function to return `.`. This function is called by +float-to-string conversions to retrieve the decimal separator between integer +and fractional parts. + +@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 +@since version 2.0.0 +*/ +struct DecimalSeparator : std::numpunct +{ + char do_decimal_point() const + { + return '.'; + } +}; + +} + +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null value. + - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + JSON values have + [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the class + has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](http://en.cppreference.com/w/cpp/concept/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. + +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 + +@nosubgrouping +*/ +template < + template class ObjectType = std::map, + template class ArrayType = std::vector, + class StringType = std::string, + class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator + > +class basic_json +{ + private: + /// workaround type for MSVC + using basic_json_t = basic_json; + + public: + // forward declarations + template class json_reverse_iterator; + class json_pointer; + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + class iterator; + /// a const iterator for a basic_json container + class const_iterator; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() + { + return allocator_type(); + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /*! + @brief a type for an object + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. + + To store objects in C++, a type is defined by the template parameters + described below. + + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) + + #### Default type + + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: + + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less, // key_compare + std::allocator> // allocator_type + > + @endcode + + #### Behavior + + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: + + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, later stored name/value + pairs overwrite previously stored name/value pairs, leaving the used + names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will + be treated as equal and both stored as `{"key": 1}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType, + AllocatorType>>; + + /*! + @brief a type for an array + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. + + To store objects in C++, a type is defined by the template parameters + explained below. + + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + + #### Default type + + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: + + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator // allocator_type + > + @endcode + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the array's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. + + #### Storage + + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value + + @since version 1.0.0 + */ + using array_t = ArrayType>; + + /*! + @brief a type for a string + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. + + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. + + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. + + #### Default type + + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: + + @code {.cpp} + std::string + @endcode + + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage + + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. + + @since version 1.0.0 + */ + using string_t = StringType; + + /*! + @brief a type for a boolean + + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. + + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. + + #### Default type + + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. + + @since version 1.0.0 + */ + using boolean_t = BooleanType; + + /*! + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. + + #### Default type + + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: + + @code {.cpp} + int64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_integer_t = NumberIntegerType; + + /*! + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. + + #### Default type + + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: + + @code {.cpp} + uint64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) + + @since version 2.0.0 + */ + using number_unsigned_t = NumberUnsignedType; + + /*! + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. + + #### Default type + + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: + + @code {.cpp} + double + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. + + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. + + #### Storage + + Floating-point number values are stored directly inside a @ref basic_json + type. + + @sa @ref number_integer_t -- type for number values (integer) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_float_t = NumberFloatType; + + /// @} + + + /////////////////////////// + // JSON type enumeration // + /////////////////////////// + + /*! + @brief the JSON type enumeration + + This enumeration collects the different JSON types. It is internally used + to distinguish the stored values, and the functions @ref is_null(), @ref + is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref + is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and + @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and + @ref is_structured() rely on it. + + @note There are three enumeration entries (number_integer, + number_unsigned, and number_float), because the library distinguishes + these three types for numbers: @ref number_unsigned_t is used for unsigned + integers, @ref number_integer_t is used for signed integers, and @ref + number_float_t is used for floating-point numbers or to approximate + integers which do not fit in the limits of their respective type. + + @sa @ref basic_json(const value_t value_type) -- create a JSON value with + the default value for a given type + + @since version 1.0.0 + */ + enum class value_t : uint8_t + { + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function + }; + + + private: + + /// helper for exception-safe object creation + template + static T* create(Args&& ... args) + { + AllocatorType alloc; + auto deleter = [&](T * object) + { + alloc.deallocate(object, 1); + }; + std::unique_ptr object(alloc.allocate(1), deleter); + alloc.construct(object.get(), std::forward(args)...); + assert(object.get() != nullptr); + return object.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } + + case value_t::array: + { + array = create(); + break; + } + + case value_t::string: + { + string = create(""); + break; + } + + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } + + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } + + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } + + default: + { + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) + { + string = create(value); + } + + /// constructor for objects + json_value(const object_t& value) + { + object = create(value); + } + + /// constructor for arrays + json_value(const array_t& value) + { + array = create(value); + } + }; + + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + */ + void assert_invariant() const + { + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /*! + @brief JSON callback events + + This enumeration lists the parser events that can trigger calling a + callback function of type @ref parser_callback_t during parsing. + + @image html callback_events.png "Example when certain parse events are triggered" + + @since version 1.0.0 + */ + enum class parse_event_t : uint8_t + { + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; + + /*! + @brief per-element parser callback type + + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse(std::istream&, const + parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), + it is called on certain events (passed as @ref parse_event_t via parameter + @a event) with a set recursion depth @a depth and context JSON value + @a parsed. The return value of the callback function is a boolean + indicating whether the element that emitted the callback shall be kept or + not. + + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. + + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: + + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. + + @param[in] depth the depth of the recursion during parsing + + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called + + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events + + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. + + @sa @ref parse(std::istream&, parser_callback_t) or + @ref parse(const string_t&, parser_callback_t) for examples + + @since version 1.0.0 + */ + using parser_callback_t = std::function; + + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /*! + @brief create an empty value with a given type + + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @param[in] value_type the type of the value to create + + @complexity Constant. + + @throw std::bad_alloc if allocation for object, array, or string value + fails + + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} + + @sa @ref basic_json(std::nullptr_t) -- create a `null` value + @sa @ref basic_json(boolean_t value) -- create a boolean value + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const object_t&) -- create a object value + @sa @ref basic_json(const array_t&) -- create a array value + @sa @ref basic_json(const number_float_t) -- create a number + (floating-point) value + @sa @ref basic_json(const number_integer_t) -- create a number (integer) + value + @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) + value + + @since version 1.0.0 + */ + basic_json(const value_t value_type) + : m_type(value_type), m_value(value_type) + { + assert_invariant(); + } + + /*! + @brief create a null object (implicitly) + + Create a `null` JSON value. This is the implicit version of the `null` + value constructor as it takes no parameters. + + @note The class invariant is satisfied, because it poses no requirements + for null values. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - As postcondition, it holds: `basic_json().empty() == true`. + + @liveexample{The following code shows the constructor for a `null` JSON + value.,basic_json} + + @sa @ref basic_json(std::nullptr_t) -- create a `null` value + + @since version 1.0.0 + */ + basic_json() = default; + + /*! + @brief create a null object (explicitly) + + Create a `null` JSON value. This is the explicitly version of the `null` + value constructor as it takes a null pointer as parameter. It allows to + create `null` values by explicitly assigning a `nullptr` to a JSON value. + The passed null pointer itself is not read -- it is only used to choose + the right constructor. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @liveexample{The following code shows the constructor with null pointer + parameter.,basic_json__nullptr_t} + + @sa @ref basic_json() -- default constructor (implicitly creating a `null` + value) + + @since version 1.0.0 + */ + basic_json(std::nullptr_t) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /*! + @brief create an object (explicit) + + Create an object JSON value with a given content. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with an @ref + object_t parameter.,basic_json__object_t} + + @sa @ref basic_json(const CompatibleObjectType&) -- create an object value + from a compatible STL container + + @since version 1.0.0 + */ + basic_json(const object_t& val) + : m_type(value_t::object), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an object (implicit) + + Create an object JSON value with a given content. This constructor allows + any type @a CompatibleObjectType that can be used to construct values of + type @ref object_t. + + @tparam CompatibleObjectType An object type whose `key_type` and + `value_type` is compatible to @ref object_t. Examples include `std::map`, + `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with + a `key_type` of `std::string`, and a `value_type` from which a @ref + basic_json value can be constructed. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with several + compatible object type parameters.,basic_json__CompatibleObjectType} + + @sa @ref basic_json(const object_t&) -- create an object value + + @since version 1.0.0 + */ + template ::value and + std::is_constructible::value, int>::type + = 0> + basic_json(const CompatibleObjectType& val) + : m_type(value_t::object) + { + using std::begin; + using std::end; + m_value.object = create(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create an array (explicit) + + Create an array JSON value with a given content. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with an @ref array_t + parameter.,basic_json__array_t} + + @sa @ref basic_json(const CompatibleArrayType&) -- create an array value + from a compatible STL containers + + @since version 1.0.0 + */ + basic_json(const array_t& val) + : m_type(value_t::array), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an array (implicit) + + Create an array JSON value with a given content. This constructor allows + any type @a CompatibleArrayType that can be used to construct values of + type @ref array_t. + + @tparam CompatibleArrayType An object type whose `value_type` is + compatible to @ref array_t. Examples include `std::vector`, `std::deque`, + `std::list`, `std::forward_list`, `std::array`, `std::set`, + `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a + `value_type` from which a @ref basic_json value can be constructed. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with several + compatible array type parameters.,basic_json__CompatibleArrayType} + + @sa @ref basic_json(const array_t&) -- create an array value + + @since version 1.0.0 + */ + template ::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + std::is_constructible::value, int>::type + = 0> + basic_json(const CompatibleArrayType& val) + : m_type(value_t::array) + { + using std::begin; + using std::end; + m_value.array = create(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create an string JSON value with a given content. + + @param[in] val a value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with an @ref + string_t parameter.,basic_json__string_t} + + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const string_t& val) + : m_type(value_t::string), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create a string JSON value with a given content. + + @param[in] val a literal value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with string literal + parameter.,basic_json__string_t_value_type} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const typename string_t::value_type* val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a string (implicit) + + Create a string JSON value with a given content. + + @param[in] val a value for the string + + @tparam CompatibleStringType an string type which is compatible to @ref + string_t, for instance `std::string`. + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the construction of a string value + from a compatible type.,basic_json__CompatibleStringType} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + + @since version 1.0.0 + */ + template ::value, int>::type + = 0> + basic_json(const CompatibleStringType& val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a boolean (explicit) + + Creates a JSON boolean type from a given value. + + @param[in] val a boolean value to store + + @complexity Constant. + + @liveexample{The example below demonstrates boolean + values.,basic_json__boolean_t} + + @since version 1.0.0 + */ + basic_json(boolean_t val) noexcept + : m_type(value_t::boolean), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number (explicit) + + Create an integer number JSON value with a given content. + + @tparam T A helper type to remove this function via SFINAE in case @ref + number_integer_t is the same as `int`. In this case, this constructor + would have the same signature as @ref basic_json(const int value). Note + the helper type @a T is not visible in this constructor's interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value.,basic_json__number_integer_t} + + @sa @ref basic_json(const int) -- create a number value (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + template::value) + and std::is_same::value + , int>::type + = 0> + basic_json(const number_integer_t val) noexcept + : m_type(value_t::number_integer), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number from an enum type (explicit) + + Create an integer number JSON value with a given content. + + @param[in] val an integer to create a JSON number from + + @note This constructor allows to pass enums directly to a constructor. As + C++ has no way of specifying the type of an anonymous enum explicitly, we + can only rely on the fact that such values implicitly convert to int. As + int may already be the same type of number_integer_t, we may need to + switch off the constructor @ref basic_json(const number_integer_t). + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value from an anonymous enum.,basic_json__const_int} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const int val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create an integer number (implicit) + + Create an integer number JSON value with a given content. This constructor + allows any type @a CompatibleNumberIntegerType that can be used to + construct values of type @ref number_integer_t. + + @tparam CompatibleNumberIntegerType An integer type which is compatible to + @ref number_integer_t. Examples include the types `int`, `int32_t`, + `long`, and `short`. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of several integer + number values from compatible + types.,basic_json__CompatibleIntegerNumberType} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const int) -- create a number value (integer) + + @since version 1.0.0 + */ + template::value and + std::numeric_limits::is_integer and + std::numeric_limits::is_signed, + CompatibleNumberIntegerType>::type + = 0> + basic_json(const CompatibleNumberIntegerType val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create an unsigned integer number (explicit) + + Create an unsigned integer number JSON value with a given content. + + @tparam T helper type to compare number_unsigned_t and unsigned int (not + visible in) the interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number + value (unsigned integer) from a compatible number type + + @since version 2.0.0 + */ + template::value) + and std::is_same::value + , int>::type + = 0> + basic_json(const number_unsigned_t val) noexcept + : m_type(value_t::number_unsigned), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an unsigned number (implicit) + + Create an unsigned number JSON value with a given content. This + constructor allows any type @a CompatibleNumberUnsignedType that can be + used to construct values of type @ref number_unsigned_t. + + @tparam CompatibleNumberUnsignedType An integer type which is compatible + to @ref number_unsigned_t. Examples may include the types `unsigned int`, + `uint32_t`, or `unsigned short`. + + @param[in] val an unsigned integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const number_unsigned_t) -- create a number value + (unsigned) + + @since version 2.0.0 + */ + template ::value and + std::numeric_limits::is_integer and + not std::numeric_limits::is_signed, + CompatibleNumberUnsignedType>::type + = 0> + basic_json(const CompatibleNumberUnsignedType val) noexcept + : m_type(value_t::number_unsigned), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create a floating-point number (explicit) + + Create a floating-point number JSON value with a given content. + + @param[in] val a floating-point value to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is created + instead. + + @complexity Constant. + + @liveexample{The following example creates several floating-point + values.,basic_json__number_float_t} + + @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number + value (floating-point) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const number_float_t val) noexcept + : m_type(value_t::number_float), m_value(val) + { + // replace infinity and NAN by null + if (not std::isfinite(val)) + { + m_type = value_t::null; + m_value = json_value(); + } + + assert_invariant(); + } + + /*! + @brief create an floating-point number (implicit) + + Create an floating-point number JSON value with a given content. This + constructor allows any type @a CompatibleNumberFloatType that can be used + to construct values of type @ref number_float_t. + + @tparam CompatibleNumberFloatType A floating-point type which is + compatible to @ref number_float_t. Examples may include the types `float` + or `double`. + + @param[in] val a floating-point to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is + created instead. + + @complexity Constant. + + @liveexample{The example below shows the construction of several + floating-point number values from compatible + types.,basic_json__CompatibleNumberFloatType} + + @sa @ref basic_json(const number_float_t) -- create a number value + (floating-point) + + @since version 1.0.0 + */ + template::value and + std::is_floating_point::value>::type + > + basic_json(const CompatibleNumberFloatType val) noexcept + : basic_json(number_float_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a container (array or object) from an initializer list + + Creates a JSON value of type array or object from the passed initializer + list @a init. In case @a type_deduction is `true` (default), the type of + the JSON value to be created is deducted from the initializer list @a init + according to the following rules: + + 1. If the list is empty, an empty JSON object value `{}` is created. + 2. If the list consists of pairs whose first element is a string, a JSON + object value is created where the first elements of the pairs are + treated as keys and the second elements are as values. + 3. In all other cases, an array is created. + + The rules aim to create the best fit between a C++ initializer list and + JSON values. The rationale is as follows: + + 1. The empty initializer list is written as `{}` which is exactly an empty + JSON object. + 2. C++ has now way of describing mapped types other than to list a list of + pairs. As JSON requires that keys must be of type string, rule 2 is the + weakest constraint one can pose on initializer lists to interpret them + as an object. + 3. In all other cases, the initializer list could not be interpreted as + JSON object type, so interpreting it as JSON array type is safe. + + With the rules described above, the following JSON values cannot be + expressed by an initializer list: + + - the empty array (`[]`): use @ref array(std::initializer_list) + with an empty initializer list in this case + - arrays whose elements satisfy rule 2: use @ref + array(std::initializer_list) with the same initializer list + in this case + + @note When used without parentheses around an empty initializer list, @ref + basic_json() is called instead of this function, yielding the JSON null + value. + + @param[in] init initializer list with JSON values + + @param[in] type_deduction internal parameter; when set to `true`, the type + of the JSON value is deducted from the initializer list @a init; when set + to `false`, the type provided via @a manual_type is forced. This mode is + used by the functions @ref array(std::initializer_list) and + @ref object(std::initializer_list). + + @param[in] manual_type internal parameter; when @a type_deduction is set + to `false`, the created JSON value will use the provided type (only @ref + value_t::array and @ref value_t::object are valid); when @a type_deduction + is set to `true`, this parameter has no effect + + @throw std::domain_error if @a type_deduction is `false`, @a manual_type + is `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string; example: `"cannot create object from + initializer list"` + + @complexity Linear in the size of the initializer list @a init. + + @liveexample{The example below shows how JSON values are created from + initializer lists.,basic_json__list_init_t} + + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + basic_json(std::initializer_list init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const basic_json & element) + { + return element.is_array() and element.size() == 2 and element[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (not type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (manual_type == value_t::object and not is_an_object) + { + throw std::domain_error("cannot create object from initializer list"); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + std::for_each(init.begin(), init.end(), [this](const basic_json & element) + { + m_value.object->emplace(*(element[0].m_value.string), element[1]); + }); + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create(init); + } + + assert_invariant(); + } + + /*! + @brief explicitly create an array from an initializer list + + Creates a JSON array value from a given initializer list. That is, given a + list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the + initializer list is empty, the empty array `[]` is created. + + @note This function is only needed to express two edge cases that cannot + be realized with the initializer list constructor (@ref + basic_json(std::initializer_list, bool, value_t)). These cases + are: + 1. creating an array whose elements are all pairs whose first element is a + string -- in this case, the initializer list constructor would create an + object, taking the first elements as keys + 2. creating an empty array -- passing the empty initializer list to the + initializer list constructor yields an empty object + + @param[in] init initializer list with JSON values to create an array from + (optional) + + @return JSON array value + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `array` + function.,array} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + static basic_json array(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::array); + } + + /*! + @brief explicitly create an object from an initializer list + + Creates a JSON object value from a given initializer list. The initializer + lists elements must be pairs, and their first elements must be strings. If + the initializer list is empty, the empty object `{}` is created. + + @note This function is only added for symmetry reasons. In contrast to the + related function @ref array(std::initializer_list), there are + no cases which can only be expressed by this function. That is, any + initializer list @a init can also be passed to the initializer list + constructor @ref basic_json(std::initializer_list, bool, + value_t). + + @param[in] init initializer list to create an object from (optional) + + @return JSON object value + + @throw std::domain_error if @a init is not a pair whose first elements are + strings; thrown by + @ref basic_json(std::initializer_list, bool, value_t) + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `object` + function.,object} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + + @since version 1.0.0 + */ + static basic_json object(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::object); + } + + /*! + @brief construct an array with count copies of given value + + Constructs a JSON array value by creating @a cnt copies of a passed value. + In case @a cnt is `0`, an empty array is created. As postcondition, + `std::distance(begin(),end()) == cnt` holds. + + @param[in] cnt the number of JSON copies of @a val to create + @param[in] val the JSON value to copy + + @complexity Linear in @a cnt. + + @liveexample{The following code shows examples for the @ref + basic_json(size_type\, const basic_json&) + constructor.,basic_json__size_type_basic_json} + + @since version 1.0.0 + */ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + assert_invariant(); + } + + /*! + @brief construct a JSON container given an iterator range + + Constructs the JSON value with the contents of the range `[first, last)`. + The semantics depends on the different types a JSON value can have: + - In case of primitive types (number, boolean, or string), @a first must + be `begin()` and @a last must be `end()`. In this case, the value is + copied. Otherwise, std::out_of_range is thrown. + - In case of structured types (array, object), the constructor behaves as + similar versions for `std::vector`. + - In case of a null type, std::domain_error is thrown. + + @tparam InputIT an input iterator type (@ref iterator or @ref + const_iterator) + + @param[in] first begin of the range to copy from (included) + @param[in] last end of the range to copy from (excluded) + + @pre Iterators @a first and @a last must be initialized. + + @throw std::domain_error if iterators are not compatible; that is, do not + belong to the same JSON value; example: `"iterators are not compatible"` + @throw std::out_of_range if iterators are for a primitive type (number, + boolean, or string) where an out of range error can be detected easily; + example: `"iterators out of range"` + @throw std::bad_alloc if allocation for object, array, or string fails + @throw std::domain_error if called with a null value; example: `"cannot + use construct with iterators from null"` + + @complexity Linear in distance between @a first and @a last. + + @liveexample{The example below shows several ways to create JSON values by + specifying a subrange with iterators.,basic_json__InputIt_InputIt} + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + basic_json(InputIT first, InputIT last) + { + assert(first.m_object != nullptr); + assert(last.m_object != nullptr); + + // make sure iterator fits the current value + if (first.m_object != last.m_object) + { + throw std::domain_error("iterators are not compatible"); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + throw std::out_of_range("iterators out of range"); + } + break; + } + + default: + { + break; + } + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); + } + } + + assert_invariant(); + } + + /*! + @brief construct a JSON value given an input stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates constructing a JSON value from + a `std::stringstream` with and without callback + function.,basic_json__istream} + + @since version 2.0.0 + */ + explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) + { + *this = parser(i, cb).parse(); + assert_invariant(); + } + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + /*! + @brief copy constructor + + Creates a copy of a given JSON value. + + @param[in] other the JSON value to copy + + @complexity Linear in the size of @a other. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - As postcondition, it holds: `other == basic_json(other)`. + + @throw std::bad_alloc if allocation for object, array, or string fails. + + @liveexample{The following code shows an example for the copy + constructor.,basic_json__basic_json} + + @since version 1.0.0 + */ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + default: + { + break; + } + } + + assert_invariant(); + } + + /*! + @brief move constructor + + Move constructor. Constructs a JSON value with the contents of the given + value @a other using move semantics. It "steals" the resources from @a + other and leaves it as JSON null value. + + @param[in,out] other value to move to this object + + @post @a other is a JSON null value + + @complexity Constant. + + @liveexample{The code below shows the move constructor explicitly called + via std::move.,basic_json__moveconstructor} + + @since version 1.0.0 + */ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + assert_invariant(); + } + + /*! + @brief copy assignment + + Copy assignment operator. Copies a JSON value via the "copy and swap" + strategy: It is expressed in terms of the copy constructor, destructor, + and the swap() member function. + + @param[in] other value to copy from + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + + @liveexample{The code below shows and example for the copy assignment. It + creates a copy of value `a` which is then swapped with `b`. Finally\, the + copy of `a` (which is the null value after the swap) is + destroyed.,basic_json__copyassignment} + + @since version 1.0.0 + */ + reference& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + assert_invariant(); + return *this; + } + + /*! + @brief destructor + + Destroys the JSON value and frees all allocated memory. + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - All stored elements are destroyed and all memory is freed. + + @since version 1.0.0 + */ + ~basic_json() + { + assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + AllocatorType alloc; + alloc.destroy(m_value.object); + alloc.deallocate(m_value.object, 1); + break; + } + + case value_t::array: + { + AllocatorType alloc; + alloc.destroy(m_value.array); + alloc.deallocate(m_value.array, 1); + break; + } + + case value_t::string: + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + break; + } + + default: + { + // all other types need no specific destructor + break; + } + } + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /*! + @brief serialization + + Serialization function for JSON values. The function tries to mimic + Python's `json.dumps()` function, and currently supports its @a indent + parameter. + + @param[in] indent If indent is nonnegative, then array elements and object + members will be pretty-printed with that indent level. An indent level of + `0` will only insert newlines. `-1` (the default) selects the most compact + representation. + + @return string containing the serialization of the JSON value + + @complexity Linear. + + @liveexample{The following example shows the effect of different @a indent + parameters to the result of the serialization.,dump} + + @see https://docs.python.org/2/library/json.html#json.dump + + @since version 1.0.0 + */ + string_t dump(const int indent = -1) const + { + std::stringstream ss; + // fix locale problems + ss.imbue(std::locale(std::locale(), new DecimalSeparator)); + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits::digits10 + ss.precision(std::numeric_limits::digits10); + + if (indent >= 0) + { + dump(ss, true, static_cast(indent)); + } + else + { + dump(ss, false, 0); + } + + return ss.str(); + } + + /*! + @brief return the type of the JSON value (explicit) + + Return the type of the JSON value as a value from the @ref value_t + enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `type()` for all JSON + types.,type} + + @since version 1.0.0 + */ + constexpr value_t type() const noexcept + { + return m_type; + } + + /*! + @brief return whether type is primitive + + This function returns true iff the JSON type is primitive (string, number, + boolean, or null). + + @return `true` if type is primitive (string, number, boolean, or null), + `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_primitive()` for all JSON + types.,is_primitive} + + @sa @ref is_structured() -- returns whether JSON value is structured + @sa @ref is_null() -- returns whether JSON value is `null` + @sa @ref is_string() -- returns whether JSON value is a string + @sa @ref is_boolean() -- returns whether JSON value is a boolean + @sa @ref is_number() -- returns whether JSON value is a number + + @since version 1.0.0 + */ + constexpr bool is_primitive() const noexcept + { + return is_null() or is_string() or is_boolean() or is_number(); + } + + /*! + @brief return whether type is structured + + This function returns true iff the JSON type is structured (array or + object). + + @return `true` if type is structured (array or object), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_structured()` for all JSON + types.,is_structured} + + @sa @ref is_primitive() -- returns whether value is primitive + @sa @ref is_array() -- returns whether value is an array + @sa @ref is_object() -- returns whether value is an object + + @since version 1.0.0 + */ + constexpr bool is_structured() const noexcept + { + return is_array() or is_object(); + } + + /*! + @brief return whether value is null + + This function returns true iff the JSON value is null. + + @return `true` if type is null, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_null()` for all JSON + types.,is_null} + + @since version 1.0.0 + */ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /*! + @brief return whether value is a boolean + + This function returns true iff the JSON value is a boolean. + + @return `true` if type is boolean, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_boolean()` for all JSON + types.,is_boolean} + + @since version 1.0.0 + */ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /*! + @brief return whether value is a number + + This function returns true iff the JSON value is a number. This includes + both integer and floating-point values. + + @return `true` if type is number (regardless whether integer, unsigned + integer or floating-type), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number()` for all JSON + types.,is_number} + + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number() const noexcept + { + return is_number_integer() or is_number_float(); + } + + /*! + @brief return whether value is an integer number + + This function returns true iff the JSON value is an integer or unsigned + integer number. This excludes floating-point values. + + @return `true` if type is an integer or unsigned integer number, `false` + otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_integer()` for all + JSON types.,is_number_integer} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer or m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is an unsigned integer number + + This function returns true iff the JSON value is an unsigned integer + number. This excludes floating-point and (signed) integer values. + + @return `true` if type is an unsigned integer number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_unsigned()` for all + JSON types.,is_number_unsigned} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 2.0.0 + */ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is a floating-point number + + This function returns true iff the JSON value is a floating-point number. + This excludes integer and unsigned integer values. + + @return `true` if type is a floating-point number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_float()` for all + JSON types.,is_number_float} + + @sa @ref is_number() -- check if value is number + @sa @ref is_number_integer() -- check if value is an integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + + @since version 1.0.0 + */ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /*! + @brief return whether value is an object + + This function returns true iff the JSON value is an object. + + @return `true` if type is object, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_object()` for all JSON + types.,is_object} + + @since version 1.0.0 + */ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /*! + @brief return whether value is an array + + This function returns true iff the JSON value is an array. + + @return `true` if type is array, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_array()` for all JSON + types.,is_array} + + @since version 1.0.0 + */ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /*! + @brief return whether value is a string + + This function returns true iff the JSON value is a string. + + @return `true` if type is string, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_string()` for all JSON + types.,is_string} + + @since version 1.0.0 + */ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /*! + @brief return whether value is discarded + + This function returns true iff the JSON value was discarded during parsing + with a callback function (see @ref parser_callback_t). + + @note This function will always be `false` for JSON values after parsing. + That is, discarded values can only occur during parsing, but will be + removed when inside a structured value or replaced by null in other cases. + + @return `true` if type is discarded, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_discarded()` for all JSON + types.,is_discarded} + + @since version 1.0.0 + */ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /*! + @brief return the type of the JSON value (implicit) + + Implicitly return the type of the JSON value as a value from the @ref + value_t enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies the @ref value_t operator for + all JSON types.,operator__value_t} + + @since version 1.0.0 + */ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get an object (explicit) + template ::value and + std::is_convertible::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_object()) + { + return T(m_value.object->begin(), m_value.object->end()); + } + else + { + throw std::domain_error("type must be object, but is " + type_name()); + } + } + + /// get an object (explicit) + object_t get_impl(object_t*) const + { + if (is_object()) + { + return *(m_value.object); + } + else + { + throw std::domain_error("type must be object, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not std::is_same::value and + not std::is_arithmetic::value and + not std::is_convertible::value and + not has_mapped_type::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + T to_vector; + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get(); + }); + return to_vector; + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not std::is_same::value + , int>::type = 0> + std::vector get_impl(std::vector*) const + { + if (is_array()) + { + std::vector to_vector; + to_vector.reserve(m_value.array->size()); + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get(); + }); + return to_vector; + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not has_mapped_type::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + return T(m_value.array->begin(), m_value.array->end()); + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + array_t get_impl(array_t*) const + { + if (is_array()) + { + return *(m_value.array); + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get a string (explicit) + template ::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_string()) + { + return *m_value.string; + } + else + { + throw std::domain_error("type must be string, but is " + type_name()); + } + } + + /// get a number (explicit) + template::value + , int>::type = 0> + T get_impl(T*) const + { + switch (m_type) + { + case value_t::number_integer: + { + return static_cast(m_value.number_integer); + } + + case value_t::number_unsigned: + { + return static_cast(m_value.number_unsigned); + } + + case value_t::number_float: + { + return static_cast(m_value.number_float); + } + + default: + { + throw std::domain_error("type must be number, but is " + type_name()); + } + } + } + + /// get a boolean (explicit) + constexpr boolean_t get_impl(boolean_t*) const + { + return is_boolean() + ? m_value.boolean + : throw std::domain_error("type must be boolean, but is " + type_name()); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t*) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t*) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t*) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t*) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t*) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t*) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t*) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t*) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t*) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This funcion helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw std::domain_error if ReferenceType does not match underlying value + type of the current JSON + */ + template + static ReferenceType get_ref_impl(ThisType& obj) + { + // helper type + using PointerType = typename std::add_pointer::type; + + // delegate the call to get_ptr<>() + auto ptr = obj.template get_ptr(); + + if (ptr != nullptr) + { + return *ptr; + } + else + { + throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + + obj.type_name()); + } + } + + public: + + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON; example: `"type must be object, but is null"` + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,get__ValueType_const} + + @internal + The idea of using a casted null pointer to choose the correct + implementation is from . + @endinternal + + @sa @ref operator ValueType() const for implicit conversion + @sa @ref get() for pointer-member access + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + ValueType get() const + { + return get_impl(static_cast(nullptr)); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + PointerType get() noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template::value + , int>::type = 0> + constexpr const PointerType get() const noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are + made. + + @warning Writing data to the pointee of the result yields an undefined + state. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get_ptr} + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + PointerType get_ptr() noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a pointer value (implicit) + @copydoc get_ptr() + */ + template::value + and std::is_const::type>::value + , int>::type = 0> + constexpr const PointerType get_ptr() const noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a reference value (implicit) + + Implict reference access to the internally stored JSON value. No copies + are made. + + @warning Writing data to the referee of the result yields an undefined + state. + + @tparam ReferenceType reference type; must be a reference to @ref array_t, + @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or + @ref number_float_t. Enforced by static assertion. + + @return reference to the internally stored JSON value if the requested + reference type @a ReferenceType fits to the JSON value; throws + std::domain_error otherwise + + @throw std::domain_error in case passed type @a ReferenceType is + incompatible with the stored JSON value + + @complexity Constant. + + @liveexample{The example shows several calls to `get_ref()`.,get_ref} + + @since version 1.1.0 + */ + template::value + , int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a reference value (implicit) + @copydoc get_ref() + */ + template::value + and std::is_const::type>::value + , int>::type = 0> + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON, thrown by @ref get() const + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename + std::enable_if < + not std::is_pointer::value + and not std::is_same::value +#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 + and not std::is_same>::value +#endif + , int >::type = 0 > + operator ValueType() const + { + // delegate the call to get<>() const + return get(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /*! + @brief access specified array element with bounds checking + + Returns a reference to the element at specified location @a idx, with + bounds checking. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read and + written using `at()`.,at__size_type} + + @since version 1.0.0 + */ + reference at(size_type idx) + { + // at only works for arrays + if (is_array()) + { + try + { + return m_value.array->at(idx); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element with bounds checking + + Returns a const reference to the element at specified location @a idx, + with bounds checking. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + `at()`.,at__size_type_const} + + @since version 1.0.0 + */ + const_reference at(size_type idx) const + { + // at only works for arrays + if (is_array()) + { + try + { + return m_value.array->at(idx); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a reference to the element at with specified key @a key, with + bounds checking. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using `at()`.,at__object_t_key_type} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (is_object()) + { + try + { + return m_value.object->at(key); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("key '" + key + "' not found"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a const reference to the element at with specified key @a key, + with bounds checking. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + `at()`.,at__object_t_key_type_const} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (is_object()) + { + try + { + return m_value.object->at(key); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("key '" + key + "' not found"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a reference to the element at specified location @a idx. + + @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), + then the array is silently filled up with `null` values to make `idx` a + valid reference to the last stored element. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array or null; example: + `"cannot use operator[] with string"` + + @complexity Constant if @a idx is in the range of the array. Otherwise + linear in `idx - size()`. + + @liveexample{The example below shows how array elements can be read and + written using `[]` operator. Note the addition of `null` + values.,operatorarray__size_type} + + @since version 1.0.0 + */ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create(); + assert_invariant(); + } + + // operator[] only works for arrays + if (is_array()) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { + m_value.array->insert(m_value.array->end(), + idx - m_value.array->size() + 1, + basic_json()); + } + + return m_value.array->operator[](idx); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a const reference to the element at specified location @a idx. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array; example: `"cannot use + operator[] with null"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + the `[]` operator.,operatorarray__size_type_const} + + @since version 1.0.0 + */ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (is_array()) + { + return m_value.array->operator[](idx); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + // operator[] only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + reference operator[](T * (&key)[n]) + { + return operator[](static_cast(key)); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @note This function is required for compatibility reasons with Clang. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + const_reference operator[](T * (&key)[n]) const + { + return operator[](static_cast(key)); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + const_reference operator[](T* key) const + { + // at only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(key); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const typename object_t::key_type&), this function + does not throw if the given key @a key was not found. + + @note Unlike @ref operator[](const typename object_t::key_type& key), this + function does not implicitly add an element to the position defined by @a + key. This function is furthermore also applicable to const objects. + + @param[in] key key of the element to access + @param[in] default_value the value to return if @a key is not found + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + + @since version 1.0.0 + */ + template ::value + , int>::type = 0> + ValueType value(const typename object_t::key_type& key, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return *it; + } + else + { + return default_value; + } + } + else + { + throw std::domain_error("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const + */ + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /*! + @brief access specified object element via JSON Pointer with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(ptr); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const json_pointer&), this function does not throw + if the given key @a key was not found. + + @param[in] ptr a JSON pointer to the element to access + @param[in] default_value the value to return if @a ptr found no value + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value_ptr} + + @sa @ref operator[](const json_pointer&) for unchecked access by reference + + @since version 2.0.2 + */ + template ::value + , int>::type = 0> + ValueType value(const json_pointer& ptr, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if pointer resolves a value, return it or use default value + try + { + return ptr.get_checked(this); + } + catch (std::out_of_range&) + { + return default_value; + } + } + else + { + throw std::domain_error("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const json_pointer&, ValueType) const + */ + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /*! + @brief access the first element + + Returns a reference to the first element in the container. For a JSON + container `c`, the expression `c.front()` is equivalent to `*c.begin()`. + + @return In case of a structured type (array or object), a reference to the + first element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, guarded by assertions). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value + + @liveexample{The following code shows an example for `front()`.,front} + + @sa @ref back() -- access the last element + + @since version 1.0.0 + */ + reference front() + { + return *begin(); + } + + /*! + @copydoc basic_json::front() + */ + const_reference front() const + { + return *cbegin(); + } + + /*! + @brief access the last element + + Returns a reference to the last element in the container. For a JSON + container `c`, the expression `c.back()` is equivalent to + @code {.cpp} + auto tmp = c.end(); + --tmp; + return *tmp; + @endcode + + @return In case of a structured type (array or object), a reference to the + last element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, guarded by assertions). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value. + + @liveexample{The following code shows an example for `back()`.,back} + + @sa @ref front() -- access the first element + + @since version 1.0.0 + */ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /*! + @copydoc basic_json::back() + */ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /*! + @brief remove element given an iterator + + Removes the element specified by iterator @a pos. The iterator @a pos must + be valid and dereferenceable. Thus the `end()` iterator (which is valid, + but is not dereferenceable) cannot be used as a value for @a pos. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] pos iterator to the element to remove + @return Iterator following the last removed element. If the iterator @a + pos refers to the last element, the `end()` iterator is returned. + + @tparam InteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on an iterator which does not belong to + the current JSON value; example: `"iterator does not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterator (i.e., any iterator which is not `begin()`); example: `"iterator + out of range"` + + @complexity The complexity depends on the type: + - objects: amortized constant + - arrays: linear in distance between pos and the end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType} + + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + InteratorType erase(InteratorType pos) + { + // make sure iterator fits the current value + if (this != pos.m_object) + { + throw std::domain_error("iterator does not fit current value"); + } + + InteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not pos.m_it.primitive_iterator.is_begin()) + { + throw std::out_of_range("iterator out of range"); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove elements given an iterator range + + Removes the element specified by the range `[first; last)`. The iterator + @a first does not need to be dereferenceable if `first == last`: erasing + an empty range is a no-op. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] first iterator to the beginning of the range to remove + @param[in] last iterator past the end of the range to remove + @return Iterator following the last removed element. If the iterator @a + second refers to the last element, the `end()` iterator is returned. + + @tparam InteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on iterators which does not belong to + the current JSON value; example: `"iterators do not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterators (i.e., if `first != begin()` and `last != end()`); example: + `"iterators out of range"` + + @complexity The complexity depends on the type: + - objects: `log(size()) + std::distance(first, last)` + - arrays: linear in the distance between @a first and @a last, plus linear + in the distance between @a last and end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType_IteratorType} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + InteratorType erase(InteratorType first, InteratorType last) + { + // make sure iterator fits the current value + if (this != first.m_object or this != last.m_object) + { + throw std::domain_error("iterators do not fit current value"); + } + + InteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + throw std::out_of_range("iterators out of range"); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove element from a JSON object given a key + + Removes elements from a JSON object with the key value @a key. + + @param[in] key value of the elements to remove + + @return Number of elements removed. If @a ObjectType is the default + `std::map` type, the return value will always be `0` (@a key was not + found) or `1` (@a key was found). + + @post References and iterators to the erased elements are invalidated. + Other references and iterators are not affected. + + @throw std::domain_error when called on a type other than JSON object; + example: `"cannot use erase() with null"` + + @complexity `log(size()) + count(key)` + + @liveexample{The example shows the effect of `erase()`.,erase__key_type} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (is_object()) + { + return m_value.object->erase(key); + } + else + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + /*! + @brief remove element from a JSON array given an index + + Removes element from a JSON array at the index @a idx. + + @param[in] idx index of the element to remove + + @throw std::domain_error when called on a type other than JSON array; + example: `"cannot use erase() with null"` + @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + is out of range"` + + @complexity Linear in distance between @a idx and the end of the container. + + @liveexample{The example shows the effect of `erase()`.,erase__size_type} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + + @since version 1.0.0 + */ + void erase(const size_type idx) + { + // this erase only works for arrays + if (is_array()) + { + if (idx >= size()) + { + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + + m_value.array->erase(m_value.array->begin() + static_cast(idx)); + } + else + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /*! + @brief find an element in a JSON object + + Finds an element in a JSON object with key equivalent to @a key. If the + element is not found or the JSON value is not an object, end() is + returned. + + @param[in] key key value of the element to search for + + @return Iterator to an element with key equivalent to @a key. If no such + element is found, past-the-end (see end()) iterator is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `find()` is used.,find__key_type} + + @since version 1.0.0 + */ + iterator find(typename object_t::key_type key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief find an element in a JSON object + @copydoc find(typename object_t::key_type) + */ + const_iterator find(typename object_t::key_type key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief returns the number of occurrences of a key in a JSON object + + Returns the number of elements with key @a key. If ObjectType is the + default `std::map` type, the return value will always be `0` (@a key was + not found) or `1` (@a key was found). + + @param[in] key key value of the element to count + + @return Number of elements with key @a key. If the JSON value is not an + object, the return value will be `0`. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `count()` is used.,count} + + @since version 1.0.0 + */ + size_type count(typename object_t::key_type key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(key) : 0; + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /*! + @brief returns an iterator to the first element + + Returns an iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `begin()`.,begin} + + @sa @ref cbegin() -- returns a const iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /*! + @copydoc basic_json::cbegin() + */ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /*! + @brief returns a const iterator to the first element + + Returns a const iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).begin()`. + + @liveexample{The following code shows an example for `cbegin()`.,cbegin} + + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /*! + @brief returns an iterator to one past the last element + + Returns an iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `end()`.,end} + + @sa @ref cend() -- returns a const iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /*! + @copydoc basic_json::cend() + */ + const_iterator end() const noexcept + { + return cend(); + } + + /*! + @brief returns a const iterator to one past the last element + + Returns a const iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).end()`. + + @liveexample{The following code shows an example for `cend()`.,cend} + + @sa @ref end() -- returns an iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /*! + @brief returns an iterator to the reverse-beginning + + Returns an iterator to the reverse-beginning; that is, the last element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(end())`. + + @liveexample{The following code shows an example for `rbegin()`.,rbegin} + + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /*! + @copydoc basic_json::crbegin() + */ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /*! + @brief returns an iterator to the reverse-end + + Returns an iterator to the reverse-end; that is, one before the first + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(begin())`. + + @liveexample{The following code shows an example for `rend()`.,rend} + + @sa @ref crend() -- returns a const reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /*! + @copydoc basic_json::crend() + */ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /*! + @brief returns a const reverse iterator to the last element + + Returns a const iterator to the reverse-beginning; that is, the last + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rbegin()`. + + @liveexample{The following code shows an example for `crbegin()`.,crbegin} + + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /*! + @brief returns a const reverse iterator to one before the first + + Returns a const reverse iterator to the reverse-end; that is, one before + the first element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rend()`. + + @liveexample{The following code shows an example for `crend()`.,crend} + + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + private: + // forward declaration + template class iteration_proxy; + + public: + /*! + @brief wrapper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + @note The name of this function is not yet final and may change in the + future. + */ + static iteration_proxy iterator_wrapper(reference cont) + { + return iteration_proxy(cont); + } + + /*! + @copydoc iterator_wrapper(reference) + */ + static iteration_proxy iterator_wrapper(const_reference cont) + { + return iteration_proxy(cont); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /*! + @brief checks whether the container is empty + + Checks if a JSON value has no elements. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `true` + boolean | `false` + string | `false` + number | `false` + object | result of function `object_t::empty()` + array | result of function `array_t::empty()` + + @note This function does not return whether a string stored as JSON value + is empty - it returns whether the JSON container itself is empty which is + false in the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `empty()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `begin() == end()`. + + @liveexample{The following code uses `empty()` to check if a JSON + object contains any elements.,empty} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + default: + { + // all other types are nonempty + return false; + } + } + } + + /*! + @brief returns the number of elements + + Returns the number of elements in a JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` + boolean | `1` + string | `1` + number | `1` + object | result of function object_t::size() + array | result of function array_t::size() + + @note This function does not return the length of a string stored as JSON + value - it returns the number of elements in the JSON value which is 1 in + the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their size() functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `std::distance(begin(), end())`. + + @liveexample{The following code calls `size()` on the different value + types.,size} + + @sa @ref empty() -- checks whether the container is empty + @sa @ref max_size() -- returns the maximal number of elements + + @since version 1.0.0 + */ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + default: + { + // all other types have size 1 + return 1; + } + } + } + + /*! + @brief returns the maximum possible number of elements + + Returns the maximum number of elements a JSON value is able to hold due to + system or library implementation limitations, i.e. `std::distance(begin(), + end())` for the JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` (same as `size()`) + boolean | `1` (same as `size()`) + string | `1` (same as `size()`) + number | `1` (same as `size()`) + object | result of function `object_t::max_size()` + array | result of function `array_t::max_size()` + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `max_size()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of returning `b.size()` where `b` is the largest + possible JSON value. + + @liveexample{The following code calls `max_size()` on the different value + types. Note the output is implementation specific.,max_size} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /*! + @brief clears the contents + + Clears the content of a JSON value and resets it to the default value as + if @ref basic_json(value_t) would have been called: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @note Floating-point numbers are set to `0.0` which will be serialized to + `0`. The vale type remains @ref number_float_t. + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows the effect of `clear()` to different + JSON types.,clear} + + @since version 1.0.0 + */ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + default: + { + break; + } + } + } + + /*! + @brief add an object to an array + + Appends the given element @a val to the end of the JSON value. If the + function is called on a JSON null value, an empty array is created before + appending @a val. + + @param[in] val the value to add to the JSON array + + @throw std::domain_error when called on a type other than JSON array or + null; example: `"cannot use push_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON array. Note how the `null` value was silently + converted to a JSON array.,push_back} + + @since version 1.0.0 + */ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + m_value.array->push_back(std::move(val)); + // invalidate object + val.m_type = value_t::null; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + m_value.array->push_back(val); + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + Inserts the given element @a val to the JSON object. If the function is + called on a JSON null value, an empty object is created before inserting + @a val. + + @param[in] val the value to add to the JSON object + + @throw std::domain_error when called on a type other than JSON object or + null; example: `"cannot use push_back() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON object. Note how the `null` value was silently + converted to a JSON object.,push_back__object_t__value} + + @since version 1.0.0 + */ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (not(is_null() or is_object())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array + m_value.object->insert(val); + } + + /*! + @brief add an object to an object + @copydoc push_back(const typename object_t::value_type&) + */ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + This function allows to use `push_back` with an initializer list. In case + + 1. the current value is an object, + 2. the initializer list @a init contains only two elements, and + 3. the first element of @a init is a string, + + @a init is converted into an object element and added using + @ref push_back(const typename object_t::value_type&). Otherwise, @a init + is converted to a JSON value and added using @ref push_back(basic_json&&). + + @param init an initializer list + + @complexity Linear in the size of the initializer list @a init. + + @note This function is required to resolve an ambiguous overload error, + because pairs like `{"key", "value"}` can be both interpreted as + `object_t::value_type` or `std::initializer_list`, see + https://github.com/nlohmann/json/issues/235 for more information. + + @liveexample{The example shows how initializer lists are treated as + objects when possible.,push_back__initializer_list} + */ + void push_back(std::initializer_list init) + { + if (is_object() and init.size() == 2 and init.begin()->is_string()) + { + const string_t key = *init.begin(); + push_back(typename object_t::value_type(key, *(init.begin() + 1))); + } + else + { + push_back(basic_json(init)); + } + } + + /*! + @brief add an object to an object + @copydoc push_back(std::initializer_list) + */ + reference operator+=(std::initializer_list init) + { + push_back(init); + return *this; + } + + /*! + @brief inserts element + + Inserts element @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] val element to insert + @return iterator pointing to the inserted @a val. + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Constant plus linear in the distance between pos and end of the + container. + + @liveexample{The example shows how `insert()` is used.,insert} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); + return result; + } + else + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts element + @copydoc insert(const_iterator, const basic_json&) + */ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /*! + @brief inserts elements + + Inserts @a cnt copies of @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] cnt number of copies of @a val to insert + @param[in] val element to insert + @return iterator pointing to the first element inserted, or @a pos if + `cnt==0` + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Linear in @a cnt plus linear in the distance between @a pos + and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__count} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + return result; + } + else + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts elements + + Inserts elements from range `[first, last)` before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + @throw std::domain_error if @a first and @a last do not belong to the same + JSON value; example: `"iterators do not fit"` + @throw std::domain_error if @a first or @a last are iterators into + container for which insert is called; example: `"passed iterators may not + belong to container"` + + @return iterator pointing to the first element inserted, or @a pos if + `first==last` + + @complexity Linear in `std::distance(first, last)` plus linear in the + distance between @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__range} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (not is_array()) + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + throw std::domain_error("iterators do not fit"); + } + + if (first.m_object == this or last.m_object == this) + { + throw std::domain_error("passed iterators may not belong to container"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert( + pos.m_it.array_iterator, + first.m_it.array_iterator, + last.m_it.array_iterator); + return result; + } + + /*! + @brief inserts elements + + Inserts elements from initializer list @a ilist before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] ilist initializer list to insert the values from + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @return iterator pointing to the first element inserted, or @a pos if + `ilist` is empty + + @complexity Linear in `ilist.size()` plus linear in the distance between + @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__ilist} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, std::initializer_list ilist) + { + // insert only works for arrays + if (not is_array()) + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); + return result; + } + + /*! + @brief exchanges the values + + Exchanges the contents of the JSON value with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other JSON value to exchange the contents with + + @complexity Constant. + + @liveexample{The example below shows how JSON values can be swapped with + `swap()`.,swap__reference} + + @since version 1.0.0 + */ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + assert_invariant(); + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON array with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other array to exchange the contents with + + @throw std::domain_error when JSON value is not an array; example: `"cannot + use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how arrays can be swapped with + `swap()`.,swap__array_t} + + @since version 1.0.0 + */ + void swap(array_t& other) + { + // swap only works for arrays + if (is_array()) + { + std::swap(*(m_value.array), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON object with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other object to exchange the contents with + + @throw std::domain_error when JSON value is not an object; example: + `"cannot use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how objects can be swapped with + `swap()`.,swap__object_t} + + @since version 1.0.0 + */ + void swap(object_t& other) + { + // swap only works for objects + if (is_object()) + { + std::swap(*(m_value.object), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON string with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other string to exchange the contents with + + @throw std::domain_error when JSON value is not a string; example: `"cannot + use swap() with boolean"` + + @complexity Constant. + + @liveexample{The example below shows how strings can be swapped with + `swap()`.,swap__string_t} + + @since version 1.0.0 + */ + void swap(string_t& other) + { + // swap only works for strings + if (is_string()) + { + std::swap(*(m_value.string), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /// @} + + + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + private: + /*! + @brief comparison operator for JSON types + + Returns an ordering that is similar to Python: + - order: null < boolean < number < object < array < string + - furthermore, each type is not smaller than itself + + @since version 1.0.0 + */ + friend bool operator<(const value_t lhs, const value_t rhs) noexcept + { + static constexpr std::array order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) + { + return false; + } + + return order[static_cast(lhs)] < order[static_cast(rhs)]; + } + + public: + /*! + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same. + - Integer and floating-point numbers are automatically converted before + comparison. Floating-point numbers are compared indirectly: two + floating-point numbers `f1` and `f2` are considered equal if neither + `f1 > f2` nor `f2 > f1` holds. + - Two JSON null values are equal. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array == *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object == *rhs.m_value.object; + } + case value_t::null: + { + return true; + } + case value_t::string: + { + return *lhs.m_value.string == *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean == rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer == rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float == rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; + } + + /*! + @brief comparison: equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__equal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator==(const_reference v, std::nullptr_t) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, std::nullptr_t) + */ + friend bool operator==(std::nullptr_t, const_reference v) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs == rhs); + } + + /*! + @brief comparison: not equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `not v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is not null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__notequal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference v, std::nullptr_t) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, std::nullptr_t) + */ + friend bool operator!=(std::nullptr_t, const_reference v) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: less than + + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. + - Integer and floating-point numbers are automatically converted before + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__less} + + @since version 1.0.0 + */ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array < *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object < *rhs.m_value.object; + } + case value_t::null: + { + return false; + } + case value_t::string: + { + return *lhs.m_value.string < *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean < rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer < rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float < rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /*! + @brief comparison: less than or equal + + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} + + @since version 1.0.0 + */ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return not (rhs < lhs); + } + + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__lessequal} + + @since version 1.0.0 + */ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs <= rhs); + } + + /*! + @brief comparison: greater than or equal + + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} + + @since version 1.0.0 + */ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs < rhs); + } + + /// @} + + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ + + /*! + @brief serialize to stream + + Serialize the given JSON value @a j to the output stream @a o. The JSON + value will be serialized using the @ref dump member function. The + indentation of the output can be controlled with the member variable + `width` of the output stream @a o. For instance, using the manipulator + `std::setw(4)` on @a o sets the indentation level to `4` and the + serialization result is the same as calling `dump(4)`. + + @note During serializaion, the locale and the precision of the output + stream @a o are changed. The original values are restored when the + function returns. + + @param[in,out] o stream to serialize to + @param[in] j JSON value to serialize + + @return the stream @a o + + @complexity Linear. + + @liveexample{The example below shows the serialization with different + parameters to `width` to adjust the indentation level.,operator_serialize} + + @since version 1.0.0 + */ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = (o.width() > 0); + const auto indentation = (pretty_print ? o.width() : 0); + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // fix locale problems + const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator)); + // set precision + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits::digits10 + const auto old_precision = o.precision(std::numeric_limits::digits10); + + // do the actual serialization + j.dump(o, pretty_print, static_cast(indentation)); + + // reset locale and precision + o.imbue(old_locale); + o.precision(old_precision); + return o; + } + + /*! + @brief serialize to stream + @copydoc operator<<(std::ostream&, const basic_json&) + */ + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } + + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /*! + @brief deserialize from string + + @param[in] s string to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @sa @ref parse(std::istream&, const parser_callback_t) for a version that + reads from an input stream + + @since version 1.0.0 + */ + static basic_json parse(const string_t& s, + const parser_callback_t cb = nullptr) + { + return parser(s, cb).parse(); + } + + /*! + @brief deserialize from stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__istream__parser_callback_t} + + @sa @ref parse(const string_t&, const parser_callback_t) for a version + that reads from a string + + @since version 1.0.0 + */ + static basic_json parse(std::istream& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @copydoc parse(std::istream&, const parser_callback_t) + */ + static basic_json parse(std::istream&& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @brief deserialize from stream + + Deserializes an input stream to a JSON value. + + @param[in,out] i input stream to read a serialized JSON value from + @param[in,out] j JSON value to write the deserialized input to + + @throw std::invalid_argument in case of parse errors + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below shows how a JSON value is constructed by + reading a serialization from a stream.,operator_deserialize} + + @sa parse(std::istream&, const parser_callback_t) for a variant with a + parser callback function to filter values while parsing + + @since version 1.0.0 + */ + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; + } + + /*! + @brief deserialize from stream + @copydoc operator<<(basic_json&, std::istream&) + */ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + j = parser(i).parse(); + return i; + } + + /// @} + + + private: + /////////////////////////// + // convenience functions // + /////////////////////////// + + /*! + @brief return the type as string + + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. + + @return basically a string representation of a the @ref m_type member + + @complexity Constant. + + @since version 1.0.0 + */ + std::string type_name() const + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; + } + } + + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + else + { + return res; + } + } + } + }); + } + + /*! + @brief escape a string + + Escape a string by replacing certain special characters by a sequence of + an escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. + + @param[in] s the string to escape + @return the escaped string + + @complexity Linear in the length of string @a s. + */ + static string_t escape_string(const string_t& s) + { + const auto space = extra_space(s); + if (space == 0) + { + return s; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + } + else + { + // all other characters are added as-is + result[pos++] = c; + } + break; + } + } + } + + return result; + } + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. Note that + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[out] o stream to write to + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(std::ostream& o, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) const + { + // variable to hold indentation for recursive calls + unsigned int new_indent = current_indent; + + switch (m_type) + { + case value_t::object: + { + if (m_value.object->empty()) + { + o << "{}"; + return; + } + + o << "{"; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) + { + if (i != m_value.object->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' ') << "\"" + << escape_string(i->first) << "\":" + << (pretty_print ? " " : ""); + i->second.dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') + "}"; + return; + } + + case value_t::array: + { + if (m_value.array->empty()) + { + o << "[]"; + return; + } + + o << "["; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) + { + if (i != m_value.array->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' '); + i->dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') << "]"; + return; + } + + case value_t::string: + { + o << string_t("\"") << escape_string(*m_value.string) << "\""; + return; + } + + case value_t::boolean: + { + o << (m_value.boolean ? "true" : "false"); + return; + } + + case value_t::number_integer: + { + o << m_value.number_integer; + return; + } + + case value_t::number_unsigned: + { + o << m_value.number_unsigned; + return; + } + + case value_t::number_float: + { + if (m_value.number_float == 0) + { + // special case for zero to get "0.0"/"-0.0" + o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); + } + else + { + o << m_value.number_float; + } + return; + } + + case value_t::discarded: + { + o << ""; + return; + } + + case value_t::null: + { + o << "null"; + return; + } + } + } + + private: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + + + private: + /////////////// + // iterators // + /////////////// + + /*! + @brief an iterator for primitive JSON types + + This class models an iterator for primitive JSON types (boolean, number, + string). It's only purpose is to allow the iterator/const_iterator classes + to "iterate" over primitive values. Internally, the iterator is modeled by + a `difference_type` variable. Value begin_value (`0`) models the begin, + end_value (`1`) models past the end. + */ + class primitive_iterator_t + { + public: + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return (m_it == begin_value); + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return (m_it == end_value); + } + + /// return reference to the value to change and compare + operator difference_type& () noexcept + { + return m_it; + } + + /// return value to compare + constexpr operator difference_type () const noexcept + { + return m_it; + } + + private: + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + /// iterator as signed integer type + difference_type m_it = std::numeric_limits::denorm_min(); + }; + + /*! + @brief an iterator value + + @note This structure could easily be a union, but MSVC currently does not + allow unions members with complex constructors, see + https://github.com/nlohmann/json/pull/105. + */ + struct internal_iterator + { + /// iterator for JSON objects + typename object_t::iterator object_iterator; + /// iterator for JSON arrays + typename array_t::iterator array_iterator; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator; + + /// create an uninitialized internal_iterator + internal_iterator() noexcept + : object_iterator(), array_iterator(), primitive_iterator() + {} + }; + + /// proxy class for the iterator_wrapper functions + template + class iteration_proxy + { + private: + /// helper class for iteration + class iteration_proxy_internal + { + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + size_t array_index = 0; + + public: + explicit iteration_proxy_internal(IteratorType it) noexcept + : anchor(it) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_internal& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_internal& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// inequality operator (needed for range-based for) + bool operator!= (const iteration_proxy_internal& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + typename basic_json::string_t key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + return std::to_string(array_index); + } + + // use key from the object + case value_t::object: + { + return anchor.key(); + } + + // use an empty key for all primitive types + default: + { + return ""; + } + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } + }; + + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) + : container(cont) + {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_internal begin() noexcept + { + return iteration_proxy_internal(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_internal end() noexcept + { + return iteration_proxy_internal(container.end()); + } + }; + + public: + /*! + @brief a const random access iterator for the @ref basic_json class + + This class implements a const iterator for the @ref basic_json class. From + this class, the @ref iterator class is derived. + + @note An iterator is called *initialized* when a pointer to a JSON value + has been set (e.g., by a constructor or a copy assignment). If the + iterator is default-constructed, it is *uninitialized* and most + methods are undefined. The library uses assertions to detect calls + on uninitialized iterators. + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + + @since version 1.0.0 + */ + class const_iterator : public std::iterator + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /// the type of the values when the iterator is dereferenced + using value_type = typename basic_json::value_type; + /// a type to represent differences between iterators + using difference_type = typename basic_json::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename basic_json::const_pointer; + /// defines a reference to the type iterated over (value_type) + using reference = typename basic_json::const_reference; + /// the category of the iterator + using iterator_category = std::bidirectional_iterator_tag; + + /// default constructor + const_iterator() = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit const_iterator(pointer object) noexcept + : m_object(object) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @brief copy constructor given a non-const iterator + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + explicit const_iterator(const iterator& other) noexcept + : m_object(other.m_object) + { + if (m_object != nullptr) + { + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = other.m_it.object_iterator; + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = other.m_it.array_iterator; + break; + } + + default: + { + m_it.primitive_iterator = other.m_it.primitive_iterator; + break; + } + } + } + } + + /*! + @brief copy constructor + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator(const const_iterator& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief copy assignment + @param[in,out] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator& operator=(const_iterator other) noexcept( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_object, other.m_object); + std::swap(m_it, other.m_it); + return *this; + } + + private: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case basic_json::value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case basic_json::value_t::null: + { + throw std::out_of_range("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return *m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator++(int) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator++() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator--(int) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator--() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator==(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + throw std::domain_error("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator == other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator!=(const const_iterator& other) const + { + return not operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + throw std::domain_error("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot compare order of object iterators"); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator < other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const const_iterator& other) const + { + return not other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const const_iterator& other) const + { + return not operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const const_iterator& other) const + { + return not operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator+=(difference_type i) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const const_iterator& other) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + return m_it.array_iterator - other.m_it.array_iterator; + } + + default: + { + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use operator[] for object iterators"); + } + + case basic_json::value_t::array: + { + return *std::next(m_it.array_iterator, n); + } + + case basic_json::value_t::null: + { + throw std::out_of_range("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator == -n) + { + return *m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + typename object_t::key_type key() const + { + assert(m_object != nullptr); + + if (m_object->is_object()) + { + return m_it.object_iterator->first; + } + else + { + throw std::domain_error("cannot use key() for non-object iterators"); + } + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + private: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator m_it = internal_iterator(); + }; + + /*! + @brief a mutable random access iterator for the @ref basic_json class + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element. + + @since version 1.0.0 + */ + class iterator : public const_iterator + { + public: + using base_iterator = const_iterator; + using pointer = typename basic_json::pointer; + using reference = typename basic_json::reference; + + /// default constructor + iterator() = default; + + /// constructor for a given JSON instance + explicit iterator(pointer object) noexcept + : base_iterator(object) + {} + + /// copy constructor + iterator(const iterator& other) noexcept + : base_iterator(other) + {} + + /// copy assignment + iterator& operator=(iterator other) noexcept( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + base_iterator::operator=(other); + return *this; + } + + /// return a reference to the value pointed to by the iterator + reference operator*() const + { + return const_cast(base_iterator::operator*()); + } + + /// dereference the iterator + pointer operator->() const + { + return const_cast(base_iterator::operator->()); + } + + /// post-increment (it++) + iterator operator++(int) + { + iterator result = *this; + base_iterator::operator++(); + return result; + } + + /// pre-increment (++it) + iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + iterator operator--(int) + { + iterator result = *this; + base_iterator::operator--(); + return result; + } + + /// pre-decrement (--it) + iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// subtract from iterator + iterator& operator-=(difference_type i) + { + base_iterator::operator-=(i); + return *this; + } + + /// add to iterator + iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const iterator& other) const + { + return base_iterator::operator-(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return const_cast(base_iterator::operator[](n)); + } + + /// return the value of an iterator + reference value() const + { + return const_cast(base_iterator::value()); + } + }; + + /*! + @brief a template for a reverse iterator class + + @tparam Base the base iterator type to reverse. Valid types are @ref + iterator (to create @ref reverse_iterator) and @ref const_iterator (to + create @ref const_reverse_iterator). + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + + @since version 1.0.0 + */ + template + class json_reverse_iterator : public std::reverse_iterator + { + public: + /// shortcut to the reverse iterator adaptor + using base_iterator = std::reverse_iterator; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) + {} + + /// create reverse iterator from base class + json_reverse_iterator(const base_iterator& it) noexcept + : base_iterator(it) + {} + + /// post-increment (it++) + json_reverse_iterator operator++(int) + { + return base_iterator::operator++(1); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + json_reverse_iterator operator--(int) + { + return base_iterator::operator--(1); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return this->base() - other.base(); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + typename object_t::key_type key() const + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } + }; + + + private: + ////////////////////// + // lexer and parser // + ////////////////////// + + /*! + @brief lexical analysis + + This class organizes the lexical analysis during JSON deserialization. The + core of it is a scanner generated by [re2c](http://re2c.org) that + processes a buffer and recognizes tokens according to RFC 7159. + */ + class lexer + { + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_number, ///< a number -- use get_number() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input ///< indicating the end of the input buffer + }; + + /// the char type to use in the lexer + using lexer_char_t = unsigned char; + + /// constructor with a given buffer + explicit lexer(const string_t& s) noexcept + : m_stream(nullptr), m_buffer(s) + { + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + s.size(); + } + + /// constructor with a given stream + explicit lexer(std::istream* s) noexcept + : m_stream(s), m_buffer() + { + assert(m_stream != nullptr); + std::getline(*m_stream, m_buffer); + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + m_buffer.size(); + } + + /// default constructor + lexer() = default; + + // switch off unwanted functions + lexer(const lexer&) = delete; + lexer operator=(const lexer&) = delete; + + /*! + @brief create a string from one or two Unicode code points + + There are two cases: (1) @a codepoint1 is in the Basic Multilingual + Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) + @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to + represent a code point above U+FFFF. + + @param[in] codepoint1 the code point (can be high surrogate) + @param[in] codepoint2 the code point (can be low surrogate or 0) + + @return string representation of the code point; the length of the + result string is between 1 and 4 characters. + + @throw std::out_of_range if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` + @throw std::invalid_argument if the low surrogate is invalid; example: + `""missing or wrong low surrogate""` + + @complexity Constant. + + @see + */ + static string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) + { + // calculate the code point from the given code points + std::size_t codepoint = codepoint1; + + // check if codepoint1 is a high surrogate + if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) + { + // check if codepoint2 is a low surrogate + if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) + { + codepoint = + // high surrogate occupies the most significant 22 bits + (codepoint1 << 10) + // low surrogate occupies the least significant 15 bits + + codepoint2 + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00; + } + else + { + throw std::invalid_argument("missing or wrong low surrogate"); + } + } + + string_t result; + + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + result.append(1, static_cast(codepoint)); + } + else if (codepoint <= 0x7ff) + { + // 2-byte characters: 110xxxxx 10xxxxxx + result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xffff) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0x10ffff) + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else + { + throw std::out_of_range("code points above 0x10FFFF are invalid"); + } + + return result; + } + + /// return name of values of type token_type (only used for errors) + static std::string token_type_name(const token_type t) + { + switch (t) + { + case token_type::uninitialized: + return ""; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case token_type::value_number: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return ""; + case token_type::end_of_input: + return "end of input"; + default: + { + // catch non-enum values + return "unknown token"; // LCOV_EXCL_LINE + } + } + } + + /*! + This function implements a scanner for JSON. It is specified using + regular expressions that try to follow RFC 7159 as close as possible. + These regular expressions are then translated into a minimized + deterministic finite automaton (DFA) by the tool + [re2c](http://re2c.org). As a result, the translated code for this + function consists of a large block of code with `goto` jumps. + + @return the class of the next token read from the buffer + + @complexity Linear in the length of the input.\n + + Proposition: The loop below will always terminate for finite input.\n + + Proof (by contradiction): Assume a finite input. To loop forever, the + loop must never hit code with a `break` statement. The only code + snippets without a `break` statement are the continue statements for + whitespace and byte-order-marks. To loop forever, the input must be an + infinite sequence of whitespace or byte-order-marks. This contradicts + the assumption of finite input, q.e.d. + */ + token_type scan() noexcept + { + while (true) + { + // pointer for backtracking information + m_marker = nullptr; + + // remember the begin of the token + m_start = m_cursor; + assert(m_start != nullptr); + + + { + lexer_char_t yych; + unsigned int yyaccept = 0; + static const unsigned char yybm[] = + { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 32, 0, 0, 32, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + }; + if ((m_limit - m_cursor) < 5) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + if (yych <= '\\') + { + if (yych <= '-') + { + if (yych <= '"') + { + if (yych <= 0x00) + { + goto basic_json_parser_2; + } + if (yych <= '!') + { + goto basic_json_parser_4; + } + goto basic_json_parser_9; + } + else + { + if (yych <= '+') + { + goto basic_json_parser_4; + } + if (yych <= ',') + { + goto basic_json_parser_10; + } + goto basic_json_parser_12; + } + } + else + { + if (yych <= '9') + { + if (yych <= '/') + { + goto basic_json_parser_4; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + goto basic_json_parser_15; + } + else + { + if (yych <= ':') + { + goto basic_json_parser_17; + } + if (yych == '[') + { + goto basic_json_parser_19; + } + goto basic_json_parser_4; + } + } + } + else + { + if (yych <= 't') + { + if (yych <= 'f') + { + if (yych <= ']') + { + goto basic_json_parser_21; + } + if (yych <= 'e') + { + goto basic_json_parser_4; + } + goto basic_json_parser_23; + } + else + { + if (yych == 'n') + { + goto basic_json_parser_24; + } + if (yych <= 's') + { + goto basic_json_parser_4; + } + goto basic_json_parser_25; + } + } + else + { + if (yych <= '|') + { + if (yych == '{') + { + goto basic_json_parser_26; + } + goto basic_json_parser_4; + } + else + { + if (yych <= '}') + { + goto basic_json_parser_28; + } + if (yych == 0xEF) + { + goto basic_json_parser_30; + } + goto basic_json_parser_4; + } + } + } +basic_json_parser_2: + ++m_cursor; + { + last_token_type = token_type::end_of_input; + break; + } +basic_json_parser_4: + ++m_cursor; +basic_json_parser_5: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_6: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + { + continue; + } +basic_json_parser_9: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych <= 0x1F) + { + goto basic_json_parser_5; + } + goto basic_json_parser_32; +basic_json_parser_10: + ++m_cursor; + { + last_token_type = token_type::value_separator; + break; + } +basic_json_parser_12: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_5; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + if (yych <= '9') + { + goto basic_json_parser_15; + } + goto basic_json_parser_5; +basic_json_parser_13: + yyaccept = 1; + yych = *(m_marker = ++m_cursor); + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_37; + } + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + } +basic_json_parser_14: + { + last_token_type = token_type::value_number; + break; + } +basic_json_parser_15: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 64) + { + goto basic_json_parser_15; + } + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_37; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + goto basic_json_parser_14; + } +basic_json_parser_17: + ++m_cursor; + { + last_token_type = token_type::name_separator; + break; + } +basic_json_parser_19: + ++m_cursor; + { + last_token_type = token_type::begin_array; + break; + } +basic_json_parser_21: + ++m_cursor; + { + last_token_type = token_type::end_array; + break; + } +basic_json_parser_23: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'a') + { + goto basic_json_parser_39; + } + goto basic_json_parser_5; +basic_json_parser_24: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'u') + { + goto basic_json_parser_40; + } + goto basic_json_parser_5; +basic_json_parser_25: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'r') + { + goto basic_json_parser_41; + } + goto basic_json_parser_5; +basic_json_parser_26: + ++m_cursor; + { + last_token_type = token_type::begin_object; + break; + } +basic_json_parser_28: + ++m_cursor; + { + last_token_type = token_type::end_object; + break; + } +basic_json_parser_30: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 0xBB) + { + goto basic_json_parser_42; + } + goto basic_json_parser_5; +basic_json_parser_31: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; +basic_json_parser_32: + if (yybm[0 + yych] & 128) + { + goto basic_json_parser_31; + } + if (yych <= 0x1F) + { + goto basic_json_parser_33; + } + if (yych <= '"') + { + goto basic_json_parser_34; + } + goto basic_json_parser_36; +basic_json_parser_33: + m_cursor = m_marker; + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } +basic_json_parser_34: + ++m_cursor; + { + last_token_type = token_type::value_string; + break; + } +basic_json_parser_36: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= 'e') + { + if (yych <= '/') + { + if (yych == '"') + { + goto basic_json_parser_31; + } + if (yych <= '.') + { + goto basic_json_parser_33; + } + goto basic_json_parser_31; + } + else + { + if (yych <= '\\') + { + if (yych <= '[') + { + goto basic_json_parser_33; + } + goto basic_json_parser_31; + } + else + { + if (yych == 'b') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + } + } + else + { + if (yych <= 'q') + { + if (yych <= 'f') + { + goto basic_json_parser_31; + } + if (yych == 'n') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 's') + { + if (yych <= 'r') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 't') + { + goto basic_json_parser_31; + } + if (yych <= 'u') + { + goto basic_json_parser_43; + } + goto basic_json_parser_33; + } + } + } +basic_json_parser_37: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_44; + } + goto basic_json_parser_33; +basic_json_parser_38: + yych = *++m_cursor; + if (yych <= ',') + { + if (yych == '+') + { + goto basic_json_parser_46; + } + goto basic_json_parser_33; + } + else + { + if (yych <= '-') + { + goto basic_json_parser_46; + } + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_47; + } + goto basic_json_parser_33; + } +basic_json_parser_39: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_49; + } + goto basic_json_parser_33; +basic_json_parser_40: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_50; + } + goto basic_json_parser_33; +basic_json_parser_41: + yych = *++m_cursor; + if (yych == 'u') + { + goto basic_json_parser_51; + } + goto basic_json_parser_33; +basic_json_parser_42: + yych = *++m_cursor; + if (yych == 0xBF) + { + goto basic_json_parser_52; + } + goto basic_json_parser_33; +basic_json_parser_43: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_54; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_54; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_54; + } + goto basic_json_parser_33; + } +basic_json_parser_44: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= 'D') + { + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_44; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + goto basic_json_parser_14; + } +basic_json_parser_46: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych >= ':') + { + goto basic_json_parser_33; + } +basic_json_parser_47: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_47; + } + goto basic_json_parser_14; +basic_json_parser_49: + yych = *++m_cursor; + if (yych == 's') + { + goto basic_json_parser_55; + } + goto basic_json_parser_33; +basic_json_parser_50: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_56; + } + goto basic_json_parser_33; +basic_json_parser_51: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_58; + } + goto basic_json_parser_33; +basic_json_parser_52: + ++m_cursor; + { + continue; + } +basic_json_parser_54: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_60; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_60; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_60; + } + goto basic_json_parser_33; + } +basic_json_parser_55: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_61; + } + goto basic_json_parser_33; +basic_json_parser_56: + ++m_cursor; + { + last_token_type = token_type::literal_null; + break; + } +basic_json_parser_58: + ++m_cursor; + { + last_token_type = token_type::literal_true; + break; + } +basic_json_parser_60: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_63; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_63; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_63; + } + goto basic_json_parser_33; + } +basic_json_parser_61: + ++m_cursor; + { + last_token_type = token_type::literal_false; + break; + } +basic_json_parser_63: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_31; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + } + + } + + return last_token_type; + } + + /// append data from the stream to the internal buffer + void yyfill() noexcept + { + if (m_stream == nullptr or not * m_stream) + { + return; + } + + const auto offset_start = m_start - m_content; + const auto offset_marker = m_marker - m_start; + const auto offset_cursor = m_cursor - m_start; + + m_buffer.erase(0, static_cast(offset_start)); + std::string line; + assert(m_stream != nullptr); + std::getline(*m_stream, line); + m_buffer += "\n" + line; // add line with newline symbol + + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_content; + m_marker = m_start + offset_marker; + m_cursor = m_start + offset_cursor; + m_limit = m_start + m_buffer.size() - 1; + } + + /// return string representation of last read token + string_t get_token_string() const + { + assert(m_start != nullptr); + return string_t(reinterpret_cast(m_start), + static_cast(m_cursor - m_start)); + } + + /*! + @brief return string value for string tokens + + The function iterates the characters between the opening and closing + quotes of the string value. The complete string is the range + [m_start,m_cursor). Consequently, we iterate from m_start+1 to + m_cursor-1. + + We differentiate two cases: + + 1. Escaped characters. In this case, a new character is constructed + according to the nature of the escape. Some escapes create new + characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied + as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape + `"\\uxxxx"` need special care. In this case, to_unicode takes care + of the construction of the values. + 2. Unescaped characters are copied as is. + + @pre `m_cursor - m_start >= 2`, meaning the length of the last token + is at least 2 bytes which is trivially true for any string (which + consists of at least two quotes). + + " c1 c2 c3 ... " + ^ ^ + m_start m_cursor + + @complexity Linear in the length of the string.\n + + Lemma: The loop body will always terminate.\n + + Proof (by contradiction): Assume the loop body does not terminate. As + the loop body does not contain another loop, one of the called + functions must never return. The called functions are `std::strtoul` + and to_unicode. Neither function can loop forever, so the loop body + will never loop forever which contradicts the assumption that the loop + body does not terminate, q.e.d.\n + + Lemma: The loop condition for the for loop is eventually false.\n + + Proof (by contradiction): Assume the loop does not terminate. Due to + the above lemma, this can only be due to a tautological loop + condition; that is, the loop condition i < m_cursor - 1 must always be + true. Let x be the change of i for any loop iteration. Then + m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This + can be rephrased to m_cursor - m_start - 2 > x. With the + precondition, we x <= 0, meaning that the loop condition holds + indefinitly if i is always decreased. However, observe that the value + of i is strictly increasing with each iteration, as it is incremented + by 1 in the iteration expression and never decremented inside the loop + body. Hence, the loop condition will eventually be false which + contradicts the assumption that the loop condition is a tautology, + q.e.d. + + @return string value of current token without opening and closing + quotes + @throw std::out_of_range if to_unicode fails + */ + string_t get_string() const + { + assert(m_cursor - m_start >= 2); + + string_t result; + result.reserve(static_cast(m_cursor - m_start - 2)); + + // iterate the result between the quotes + for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) + { + // process escaped characters + if (*i == '\\') + { + // read next character + ++i; + + switch (*i) + { + // the default escapes + case 't': + { + result += "\t"; + break; + } + case 'b': + { + result += "\b"; + break; + } + case 'f': + { + result += "\f"; + break; + } + case 'n': + { + result += "\n"; + break; + } + case 'r': + { + result += "\r"; + break; + } + case '\\': + { + result += "\\"; + break; + } + case '/': + { + result += "/"; + break; + } + case '"': + { + result += "\""; + break; + } + + // unicode + case 'u': + { + // get code xxxx from uxxxx + auto codepoint = std::strtoul(std::string(reinterpret_cast(i + 1), + 4).c_str(), nullptr, 16); + + // check if codepoint is a high surrogate + if (codepoint >= 0xD800 and codepoint <= 0xDBFF) + { + // make sure there is a subsequent unicode + if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') + { + throw std::invalid_argument("missing low surrogate"); + } + + // get code yyyy from uxxxx\uyyyy + auto codepoint2 = std::strtoul(std::string(reinterpret_cast + (i + 7), 4).c_str(), nullptr, 16); + result += to_unicode(codepoint, codepoint2); + // skip the next 10 characters (xxxx\uyyyy) + i += 10; + } + else + { + // add unicode character(s) + result += to_unicode(codepoint); + // skip the next four characters (xxxx) + i += 4; + } + break; + } + } + } + else + { + // all other characters are just copied to the end of the + // string + result.append(1, static_cast(*i)); + } + } + + return result; + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + long double str_to_float_t(long double* /* type */, char** endptr) const + { + return std::strtold(reinterpret_cast(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + double str_to_float_t(double* /* type */, char** endptr) const + { + return std::strtod(reinterpret_cast(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + float str_to_float_t(float* /* type */, char** endptr) const + { + return std::strtof(reinterpret_cast(m_start), endptr); + } + + /*! + @brief return number value for number tokens + + This function translates the last token into the most appropriate + number type (either integer, unsigned integer or floating point), + which is passed back to the caller via the result parameter. + + This function parses the integer component up to the radix point or + exponent while collecting information about the 'floating point + representation', which it stores in the result parameter. If there is + no radix point or exponent, and the number can fit into a @ref + number_integer_t or @ref number_unsigned_t then it sets the result + parameter accordingly. + + If the number is a floating point number the number is then parsed + using @a std:strtod (or @a std:strtof or @a std::strtold). + + @param[out] result @ref basic_json object to receive the number, or + NAN if the conversion read past the current token. The latter case + needs to be treated by the caller function. + */ + void get_number(basic_json& result) const + { + assert(m_start != nullptr); + + const lexer::lexer_char_t* curptr = m_start; + + // accumulate the integer conversion result (unsigned for now) + number_unsigned_t value = 0; + + // maximum absolute value of the relevant integer type + number_unsigned_t max; + + // temporarily store the type to avoid unecessary bitfield access + value_t type; + + // look for sign + if (*curptr == '-') + { + type = value_t::number_integer; + max = static_cast((std::numeric_limits::max)()) + 1; + curptr++; + } + else + { + type = value_t::number_unsigned; + max = static_cast((std::numeric_limits::max)()); + } + + // count the significant figures + for (; curptr < m_cursor; curptr++) + { + // quickly skip tests if a digit + if (*curptr < '0' || *curptr > '9') + { + if (*curptr == '.') + { + // don't count '.' but change to float + type = value_t::number_float; + continue; + } + // assume exponent (if not then will fail parse): change to + // float, stop counting and record exponent details + type = value_t::number_float; + break; + } + + // skip if definitely not an integer + if (type != value_t::number_float) + { + // multiply last value by ten and add the new digit + auto temp = value * 10 + *curptr - '0'; + + // test for overflow + if (temp < value || temp > max) + { + // overflow + type = value_t::number_float; + } + else + { + // no overflow - save it + value = temp; + } + } + } + + // save the value (if not a float) + if (type == value_t::number_unsigned) + { + result.m_value.number_unsigned = value; + } + else if (type == value_t::number_integer) + { + result.m_value.number_integer = -static_cast(value); + } + else + { + // parse with strtod + result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); + } + + // save the type + result.m_type = type; + } + + private: + /// optional input stream + std::istream* m_stream = nullptr; + /// the buffer + string_t m_buffer; + /// the buffer pointer + const lexer_char_t* m_content = nullptr; + /// pointer to the beginning of the current symbol + const lexer_char_t* m_start = nullptr; + /// pointer for backtracking information + const lexer_char_t* m_marker = nullptr; + /// pointer to the current symbol + const lexer_char_t* m_cursor = nullptr; + /// pointer to the end of the buffer + const lexer_char_t* m_limit = nullptr; + /// the last token type + token_type last_token_type = token_type::end_of_input; + }; + + /*! + @brief syntax analysis + + This class implements a recursive decent parser. + */ + class parser + { + public: + /// constructor for strings + parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(s) + { + // read first token + get_token(); + } + + /// a parser reading from an input stream + parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(&_is) + { + // read first token + get_token(); + } + + /// public parser interface + basic_json parse() + { + basic_json result = parse_internal(true); + result.assert_invariant(); + + expect(lexer::token_type::end_of_input); + + // return parser result and replace it with null in case the + // top-level value was discarded by the callback function + return result.is_discarded() ? basic_json() : std::move(result); + } + + private: + /// the actual parser + basic_json parse_internal(bool keep) + { + auto result = basic_json(value_t::discarded); + + switch (last_token) + { + case lexer::token_type::begin_object: + { + if (keep and (not callback or (keep = callback(depth++, parse_event_t::object_start, result)))) + { + // explicitly set result to object to cope with {} + result.m_type = value_t::object; + result.m_value = value_t::object; + } + + // read next token + get_token(); + + // closing } -> we are done + if (last_token == lexer::token_type::end_object) + { + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse key-value pairs + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // store key + expect(lexer::token_type::value_string); + const auto key = m_lexer.get_string(); + + bool keep_tag = false; + if (keep) + { + if (callback) + { + basic_json k(key); + keep_tag = callback(depth, parse_event_t::key, k); + } + else + { + keep_tag = true; + } + } + + // parse separator (:) + get_token(); + expect(lexer::token_type::name_separator); + + // parse and add value + get_token(); + auto value = parse_internal(keep); + if (keep and keep_tag and not value.is_discarded()) + { + result[key] = std::move(value); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing } + expect(lexer::token_type::end_object); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::begin_array: + { + if (keep and (not callback or (keep = callback(depth++, parse_event_t::array_start, result)))) + { + // explicitly set result to object to cope with [] + result.m_type = value_t::array; + result.m_value = value_t::array; + } + + // read next token + get_token(); + + // closing ] -> we are done + if (last_token == lexer::token_type::end_array) + { + get_token(); + if (callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse values + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // parse value + auto value = parse_internal(keep); + if (keep and not value.is_discarded()) + { + result.push_back(std::move(value)); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing ] + expect(lexer::token_type::end_array); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::literal_null: + { + get_token(); + result.m_type = value_t::null; + break; + } + + case lexer::token_type::value_string: + { + const auto s = m_lexer.get_string(); + get_token(); + result = basic_json(s); + break; + } + + case lexer::token_type::literal_true: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = true; + break; + } + + case lexer::token_type::literal_false: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = false; + break; + } + + case lexer::token_type::value_number: + { + m_lexer.get_number(result); + get_token(); + break; + } + + default: + { + // the last token was unexpected + unexpect(last_token); + } + } + + if (keep and callback and not callback(depth, parse_event_t::value, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + /// get next token from lexer + typename lexer::token_type get_token() noexcept + { + last_token = m_lexer.scan(); + return last_token; + } + + void expect(typename lexer::token_type t) const + { + if (t != last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + error_msg += "; expected " + lexer::token_type_name(t); + throw std::invalid_argument(error_msg); + } + } + + void unexpect(typename lexer::token_type t) const + { + if (t == last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + throw std::invalid_argument(error_msg); + } + } + + private: + /// current level of recursion + int depth = 0; + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + typename lexer::token_type last_token = lexer::token_type::uninitialized; + /// the lexer + lexer m_lexer; + }; + + public: + /*! + @brief JSON Pointer + + A JSON pointer defines a string syntax for identifying a specific value + within a JSON document. It can be used with functions `at` and + `operator[]`. Furthermore, JSON pointers are the base for JSON patches. + + @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + class json_pointer + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /*! + @brief create JSON pointer + + Create a JSON pointer according to the syntax described in + [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). + + @param[in] s string representing the JSON pointer; if omitted, the + empty string is assumed which references the whole JSON + value + + @throw std::domain_error if reference token is nonempty and does not + begin with a slash (`/`); example: `"JSON pointer must be empty or + begin with /"` + @throw std::domain_error if a tilde (`~`) is not followed by `0` + (representing `~`) or `1` (representing `/`); example: `"escape error: + ~ must be followed with 0 or 1"` + + @liveexample{The example shows the construction several valid JSON + pointers as well as the exceptional behavior.,json_pointer} + + @since version 2.0.0 + */ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /*! + @brief return a string representation of the JSON pointer + + @invariant For each JSON pointer `ptr`, it holds: + @code {.cpp} + ptr == json_pointer(ptr.to_string()); + @endcode + + @return a string representation of the JSON pointer + + @liveexample{The example shows the result of `to_string`., + json_pointer__to_string} + + @since version 2.0.0 + */ + std::string to_string() const noexcept + { + return std::accumulate(reference_tokens.begin(), + reference_tokens.end(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + escape(b); + }); + } + + /// @copydoc to_string() + operator std::string() const + { + return to_string(); + } + + private: + /// remove and return last reference pointer + std::string pop_back() + { + if (is_root()) + { + throw std::domain_error("JSON pointer has no parent"); + } + + auto last = reference_tokens.back(); + reference_tokens.pop_back(); + return last; + } + + /// return whether pointer points to the root document + bool is_root() const + { + return reference_tokens.empty(); + } + + json_pointer top() const + { + if (is_root()) + { + throw std::domain_error("JSON pointer has no parent"); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + */ + reference get_and_create(reference j) const + { + pointer result = &j; + + // in case no reference tokens exist, return a reference to the + // JSON value j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case value_t::array: + { + // create an entry in the array + result = &result->operator[](static_cast(std::stoi(reference_token))); + break; + } + + /* + The following code is only reached if there exists a + reference token _and_ the current value is primitive. In + this case, we have an error situation, because primitive + values may only occur as single value; that is, with an + empty list of reference tokens. + */ + default: + { + throw std::domain_error("invalid value to unflatten"); + } + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + */ + reference get_unchecked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + if (reference_token == "-") + { + // explicityly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + reference get_checked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + */ + const_reference get_unchecked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" cannot be used for const access + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // use unchecked array access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + const_reference get_checked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /// split the string input to reference tokens + static std::vector split(std::string reference_string) + { + std::vector result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (reference_string[0] != '/') + { + throw std::domain_error("JSON pointer must be empty or begin with '/'"); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + size_t slash = reference_string.find_first_of("/", 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == string::npos+1 = 0 + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = slash + 1, + // find next slash + slash = reference_string.find_first_of("/", start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (size_t pos = reference_token.find_first_of("~"); + pos != std::string::npos; + pos = reference_token.find_first_of("~", pos + 1)) + { + assert(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (pos == reference_token.size() - 1 or + (reference_token[pos + 1] != '0' and + reference_token[pos + 1] != '1')) + { + throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); + } + } + + // finally, store the reference token + unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @brief replace all occurrences of a substring by another string + + @param[in,out] s the string to manipulate + @param[in] f the substring to replace with @a t + @param[in] t the string to replace @a f + + @return The string @a s where all occurrences of @a f are replaced + with @a t. + + @pre The search string @a f must not be empty. + + @since version 2.0.0 + */ + static void replace_substring(std::string& s, + const std::string& f, + const std::string& t) + { + assert(not f.empty()); + + for ( + size_t pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t + pos = s.find(f, pos + t.size()) // find next occurrence of f + ); + } + + /// escape tilde and slash + static std::string escape(std::string s) + { + // escape "~"" to "~0" and "/" to "~1" + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; + } + + /// unescape tilde and slash + static void unescape(std::string& s) + { + // first transform any occurrence of the sequence '~1' to '/' + replace_substring(s, "~1", "/"); + // then transform any occurrence of the sequence '~0' to '~' + replace_substring(s, "~0", "~"); + } + + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const basic_json& value, + basic_json& result) + { + switch (value.m_type) + { + case value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), + element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + */ + static basic_json unflatten(const basic_json& value) + { + if (not value.is_object()) + { + throw std::domain_error("only objects can be unflattened"); + } + + basic_json result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (not element.second.is_primitive()) + { + throw std::domain_error("values in object must be primitive"); + } + + // assign value to reference pointed to by JSON pointer; Note + // that if the JSON pointer is "" (i.e., points to the whole + // value), function get_and_create returns a reference to + // result itself. An assignment will then create a primitive + // value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + private: + /// the reference tokens + std::vector reference_tokens {}; + }; + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. Similar to @ref operator[](const typename + object_t::key_type&), `null` values are created in arrays and objects if + necessary. + + In particular: + - If the JSON pointer points to an object key that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. + - If the JSON pointer points to an array index that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. All indices between the current maximum and the given + index are also filled with `null`. + - The special value `-` is treated as a synonym for the index past the + end. + + @param[in] ptr a JSON pointer + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer} + + @since version 2.0.0 + */ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. The function does not change the JSON + value; no `null` values are created. In particular, the the special value + `-` yields an exception. + + @param[in] ptr JSON pointer to the desired element + + @return const reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} + + @since version 2.0.0 + */ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a reference to the element at with specified JSON pointer @a ptr, + with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer} + + @since version 2.0.0 + */ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a const reference to the element at with specified JSON pointer @a + ptr, with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} + + @since version 2.0.0 + */ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /*! + @brief return flattened JSON value + + The function creates a JSON object whose keys are JSON pointers (see [RFC + 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all + primitive. The original JSON value can be restored using the @ref + unflatten() function. + + @return an object that maps JSON pointers to primitve values + + @note Empty objects and arrays are flattened to `null` and will not be + reconstructed correctly by the @ref unflatten() function. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a JSON object is flattened to an + object whose keys consist of JSON pointers.,flatten} + + @sa @ref unflatten() for the reverse function + + @since version 2.0.0 + */ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /*! + @brief unflatten a previously flattened JSON value + + The function restores the arbitrary nesting of a JSON value that has been + flattened before using the @ref flatten() function. The JSON value must + meet certain constraints: + 1. The value must be an object. + 2. The keys must be JSON pointers (see + [RFC 6901](https://tools.ietf.org/html/rfc6901)) + 3. The mapped values must be primitive JSON types. + + @return the original JSON from a flattened version + + @note Empty objects and arrays are flattened by @ref flatten() to `null` + values and can not unflattened to their original type. Apart from + this example, for a JSON value `j`, the following is always true: + `j == j.flatten().unflatten()`. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a flattened JSON object is + unflattened into the original nested JSON object.,unflatten} + + @sa @ref flatten() for the reverse function + + @since version 2.0.0 + */ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /*! + @brief applies a JSON patch + + [JSON Patch](http://jsonpatch.com) defines a JSON document structure for + expressing a sequence of operations to apply to a JSON) document. With + this funcion, a JSON Patch is applied to the current JSON value by + executing all operations from the patch. + + @param[in] json_patch JSON patch document + @return patched document + + @note The application of a patch is atomic: Either all operations succeed + and the patched document is returned or an exception is thrown. In + any case, the original value is not changed: the patch is applied + to a copy of the value. + + @throw std::out_of_range if a JSON pointer inside the patch could not + be resolved successfully in the current JSON value; example: `"key baz + not found"` + @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + attributes are missing); example: `"operation add must have member path"` + + @complexity Linear in the size of the JSON value and the length of the + JSON patch. As usually only a fraction of the JSON value is affected by + the patch, the complexity can usually be neglected. + + @liveexample{The following code shows how a JSON patch is applied to a + value.,patch} + + @sa @ref diff -- create a JSON patch by comparing two JSON values + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.is_root()) + { + result = val; + } + else + { + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + basic_json& x = result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = std::stoi(last_path); + if (static_cast(idx) > parent.size()) + { + // avoid undefined behavior + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + else + { + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); + } + } + break; + } + + default: + { + // if there exists a parent it cannot be primitive + assert(false); // LCOV_EXCL_LINE + } + } + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [&result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (it != parent.end()) + { + parent.erase(it); + } + else + { + throw std::out_of_range("key '" + last_path + "' not found"); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(static_cast(std::stoi(last_path))); + } + }; + + // type check + if (not json_patch.is_array()) + { + // a JSON patch must be an array of objects + throw std::invalid_argument("JSON patch must be an array of objects"); + } + + // iterate and apply th eoperations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json& + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (it == val.m_value.object->end()) + { + throw std::invalid_argument(error_msg + " must have member '" + member + "'"); + } + + // check if result is of type string + if (string_type and not it->second.is_string()) + { + throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); + } + + // no error: return value + return it->second; + }; + + // type check + if (not val.is_object()) + { + throw std::invalid_argument("JSON patch must be an array of objects"); + } + + // collect mandatory members + const std::string op = get_value("op", "op", true); + const std::string path = get_value(op, "path", true); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const std::string from_path = get_value("move", "from", true); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const std::string from_path = get_value("copy", "from", true);; + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + result[ptr] = result.at(from_ptr); + break; + } + + case patch_operations::test: + { + bool success = false; + try + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + catch (std::out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (not success) + { + throw std::domain_error("unsuccessful: " + val.dump()); + } + + break; + } + + case patch_operations::invalid: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + throw std::invalid_argument("operation value '" + op + "' is invalid"); + } + } + } + + return result; + } + + /*! + @brief creates a diff as a JSON patch + + Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can + be changed into the value @a target by calling @ref patch function. + + @invariant For two JSON values @a source and @a target, the following code + yields always `true`: + @code {.cpp} + source.patch(diff(source, target)) == target; + @endcode + + @note Currently, only `remove`, `add`, and `replace` operations are + generated. + + @param[in] source JSON value to copare from + @param[in] target JSON value to copare against + @param[in] path helper value to create JSON pointers + + @return a JSON patch to convert the @a source to @a target + + @complexity Linear in the lengths of @a source and @a target. + + @liveexample{The following code shows how a JSON patch is created as a + diff for two JSON values.,diff} + + @sa @ref patch -- apply a JSON patch + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + + @since version 2.0.0 + */ + static basic_json diff(const basic_json& source, + const basic_json& target, + std::string path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + } + else + { + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + size_t i = 0; + while (i < source.size() and i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.begin(); it != source.end(); ++it) + { + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, + {"path", path + "/" + key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.begin(); it != target.end(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + key}, + {"value", it.value()} + }); + } + } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + break; + } + } + } + + return result; + } + + /// @} +}; + + +///////////// +// presets // +///////////// + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} + + +/////////////////////// +// nonmember support // +/////////////////////// + +// specialization of std::swap, and std::hash +namespace std +{ +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template <> +inline void swap(nlohmann::json& j1, + nlohmann::json& j2) noexcept( + is_nothrow_move_constructible::value and + is_nothrow_move_assignable::value + ) +{ + j1.swap(j2); +} + +/// hash value for JSON objects +template <> +struct hash +{ + /*! + @brief return a hash value for a JSON object + + @since version 1.0.0 + */ + std::size_t operator()(const nlohmann::json& j) const + { + // a naive hashing via the string representation + const auto& h = hash(); + return h(j.dump()); + } +}; +} + +/*! +@brief user-defined string literal for JSON values + +This operator implements a user-defined string literal for JSON objects. It +can be used by adding `"_json"` to a string literal and returns a JSON object +if no parse error occurred. + +@param[in] s a string representation of a JSON object +@return a JSON object + +@since version 1.0.0 +*/ +inline nlohmann::json operator "" _json(const char* s, std::size_t) +{ + return nlohmann::json::parse(reinterpret_cast(s)); +} + +/*! +@brief user-defined string literal for JSON pointer + +This operator implements a user-defined string literal for JSON Pointers. It +can be used by adding `"_json"` to a string literal and returns a JSON pointer +object if no parse error occurred. + +@param[in] s a string representation of a JSON Pointer +@return a JSON pointer object + +@since version 2.0.0 +*/ +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t) +{ + return nlohmann::json::json_pointer(s); +} + +// restore GCC/clang diagnostic settings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic pop +#endif + +#endif diff --git a/ext/offbase/offbase.hpp b/ext/offbase/offbase.hpp new file mode 100644 index 00000000..acce8c4a --- /dev/null +++ b/ext/offbase/offbase.hpp @@ -0,0 +1,393 @@ +/* + * Offbase: a super-minimal in-filesystem JSON object persistence store + */ + +#ifndef OFFBASE_HPP__ +#define OFFBASE_HPP__ + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "json/json.hpp" + +#define OFFBASE_PATH_SEP "/" + +/** + * A super-minimal in-filesystem JSON object persistence store + */ +class offbase : public nlohmann::json +{ +public: + offbase(const char *p) : + nlohmann::json(nlohmann::json::object()), + _path(p), + _saved(nlohmann::json::object()) + { + this->load(); + } + + ~offbase() + { + this->commit(); + } + + /** + * Load this instance from disk, clearing any existing contents first + * + * If the 'errors' vector is NULL, false is returned and reading aborts + * on any error. If this parameter is non-NULL the paths of errors will + * be added to the vector and reading will continue. False will only be + * returned on really big errors like no path being defined. + * + * @param errors If specified, fill this vector with the paths to any objects that fail read + * @return True on success, false on fatal error + */ + inline bool load(std::vector *errors = (std::vector *)0) + { + if (!_path.length()) + return false; + *this = nlohmann::json::object(); + if (!_loadObj(_path,*this,errors)) + return false; + _saved = *(reinterpret_cast(this)); + return true; + } + + /** + * Commit any pending changes to this object to disk + * + * @return True on success or false if an I/O error occurred + */ + inline bool commit(std::vector *errors = (std::vector *)0) + { + if (!_path.length()) + return false; + if (!_commitObj(_path,*this,&_saved,errors)) + return false; + _saved = *(reinterpret_cast(this)); + return true; + } + + static inline std::string escapeKey(const std::string &k) + { + std::string e; + const char *ptr = k.data(); + const char *eof = ptr + k.length(); + char tmp[8]; + while (ptr != eof) { + if ( ((*ptr >= 'a')&&(*ptr <= 'z')) || ((*ptr >= 'A')&&(*ptr <= 'Z')) || ((*ptr >= '0')&&(*ptr <= '9')) || (*ptr == '.') || (*ptr == '_') || (*ptr == '-') || (*ptr == ',') ) + e.push_back(*ptr); + else { + snprintf(tmp,sizeof(tmp),"~%.2x",(unsigned int)*ptr); + e.append(tmp); + } + ++ptr; + } + return e; + } + + static inline std::string unescapeKey(const std::string &k) + { + std::string u; + const char *ptr = k.data(); + const char *eof = ptr + k.length(); + char tmp[8]; + while (ptr != eof) { + if (*ptr == '~') { + if (++ptr == eof) break; + tmp[0] = *ptr; + if (++ptr == eof) break; + tmp[1] = *(ptr++); + tmp[2] = (char)0; + u.push_back((char)strtol(tmp,(char **)0,16)); + } else { + u.push_back(*(ptr++)); + } + } + return u; + } + +private: + static inline bool _readFile(const char *path,std::string &buf) + { + char tmp[4096]; + FILE *f = fopen(path,"rb"); + if (f) { + for(;;) { + long n = (long)fread(tmp,1,sizeof(tmp),f); + if (n > 0) + buf.append(tmp,n); + else break; + } + fclose(f); + return true; + } + return false; + } + + static inline bool _loadArr(const std::string &path,nlohmann::json &arr,std::vector *errors) + { + std::map atmp; // place into an ordered container first because filesystem does not guarantee this + + struct dirent dbuf; + struct dirent *de; + DIR *d = opendir(path.c_str()); + if (d) { + while (!readdir_d(d,&dbuf,&de)) { + if (!de) break; + const std::string name(de->d_name); + if (name.length() != 12) continue; // array entries are XXXXXXXXXX.T + if (name[name.length()-2] == '.') { + if (name[name.length()-1] == 'V') { + std::string buf; + if (_readFile((path + OFFBASE_PATH_SEP + name).c_str(),buf)) { + try { + atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::parse(buf); + } catch ( ... ) { + if (errors) { + errors->push_back(path + OFFBASE_PATH_SEP + name); + } else { + return false; + } + } + } else if (errors) { + errors->push_back(path + OFFBASE_PATH_SEP + name); + } else return false; + } else if (name[name.length()-1] == 'O') { + if (!_loadObj(path + OFFBASE_PATH_SEP + name,atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::object(),errors)) + return false; + } else if (name[name.length()-1] == 'A') { + if (!_loadArr(path + OFFBASE_PATH_SEP + name,atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::array(),errors)) + return false; + } + } + } + closedir(d); + } else if (errors) { + errors->push_back(path); + } else return false; + + if (atmp.size() > 0) { + unsigned long lasti = 0; + for(std::map::iterator i(atmp.begin());i!=atmp.end();++i) { + for(unsigned long k=lasti;kfirst;++k) // fill any gaps with nulls + arr.push_back(nlohmann::json(std::nullptr_t)); + lasti = i->first; + arr.push_back(i->second); + } + } + + return true; + } + + static inline bool _loadObj(const std::string &path,nlohmann::json &obj,std::vector *errors) + { + struct dirent dbuf; + struct dirent *de; + DIR *d = opendir(path.c_str()); + if (d) { + while (!readdir_d(d,&dbuf,&de)) { + if (!de) break; + if ((strcmp(de->d_name,".") == 0)||(strcmp(de->d_name,"..") == 0)) continue; // sanity check + const std::string name(de->d_name); + if (name.length() <= 2) continue; + if (name[name.length()-2] == '.') { + if (name[name.length()-1] == 'V') { + std::string buf; + if (_readFile((path + OFFBASE_PATH_SEP + name).c_str(),buf)) { + try { + obj[unescapeKey(name)] = nlohmann::json::parse(buf); + } catch ( ... ) { + if (errors) { + errors->push_back(path + OFFBASE_PATH_SEP + name); + } else { + return false; + } + } + } else if (errors) { + errors->push_back(path + OFFBASE_PATH_SEP + name); + } else return false; + } else if (name[name.length()-1] == 'O') { + if (!_loadObj(path + OFFBASE_PATH_SEP + name,obj[unescapeKey(name)] = nlohmann::json::object(),errors)) + return false; + } else if (name[name.length()-1] == 'A') { + if (!_loadArr(path + OFFBASE_PATH_SEP + name,obj[unescapeKey(name)] = nlohmann::json::array(),errors)) + return false; + } + } + } + closedir(d); + } else if (errors) { + errors->push_back(path); + } else return false; + return true; + } + + static inline void _rmDashRf(const std::string &path) + { + struct dirent dbuf; + struct dirent *de; + DIR *d = opendir(path.c_str()); + if (d) { + while (!readdir_r(d,&dbuf,&de)) { + if (!de) break; + if ((strcmp(de->d_name,".") == 0)||(strcmp(de->d_name,"..") == 0)) continue; // sanity check + const std::string full(path + OFFBASE_PATH_SEP + de->d_name); + if (unlink(full.c_str())) { + _rmDashRf(full); + rmdir(full.c_str()); + } + } + closedir(d); + } + rmdir(path.c_str()); + } + + static inline bool _commitArr(const std::string &path,const nlohmann::json &arr,const nlohmann::json *previous,std::vector *errors) + { + char tmp[32]; + + if (!arr.is_array()) + return false; + + mkdir(path.c_str(),0755); + + for(unsigned long i=0;i<(unsigned long)arr.size();++i) { + const nlohmann::json &value = arr[i]; + + const nlohmann::json *next = (const nlohmann::json *)0; + if ((previous)&&(previous->is_array())&&(i < previous->size())) { + next = &((*previous)[i]); + if (*next == value) + continue; + } + + if (value.is_object()) { + snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); + if (!_commitObj(path + tmp,value,next,errors)) + return false; + snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); + unlink((path + tmp).c_str()); + snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); + _rmDashRf(path + tmp); + } else if (value.is_array()) { + snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); + if (!_commitArr(path + tmp,value,next,errors)) + return false; + snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); + _rmDashRf(path + tmp); + snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); + unlink((path + tmp).c_str()); + } else { + snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); + FILE *f = fopen((path + tmp).c_str(),"w"); + if (f) { + const std::string v(value.dump()); + if (fwrite(v.c_str(),v.length(),1,f) != 1) { + fclose(f); + return false; + } else { + fclose(f); + } + } else { + return false; + } + snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); + _rmDashRf(path + tmp); + snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); + _rmDashRf(path + tmp); + } + } + + if ((previous)&&(previous->is_array())) { + for(unsigned long i=(unsigned long)arr.size();i<(unsigned long)previous->size();++i) { + snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); + unlink((path + tmp).c_str()); + snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); + _rmDashRf(path + tmp); + snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); + _rmDashRf(path + tmp); + } + } + + return true; + } + + static inline bool _commitObj(const std::string &path,const nlohmann::json &obj,const nlohmann::json *previous,std::vector *errors) + { + if (!obj.is_object()) + return false; + + mkdir(path.c_str(),0755); + + for(nlohmann::json::const_iterator i(obj.begin());i!=obj.end();++i) { + if (i.key().length() == 0) + continue; + + const nlohmann::json *next = (const nlohmann::json *)0; + if ((previous)&&(previous->is_object())) { + nlohmann::json::const_iterator saved(previous->find(i.key())); + if (saved != previous->end()) { + next = &(saved.value()); + if (i.value() == *next) + continue; + } + } + + const std::string keyp(path + OFFBASE_PATH_SEP + escapeKey(i.key())); + if (i.value().is_object()) { + if (!_commitObj(keyp + ".O",i.value(),next,errors)) + return false; + unlink((keyp + ".V").c_str()); + _rmDashRf(keyp + ".A"); + } else if (i.value().is_array()) { + if (!_commitArr(keyp + ".A",i.value(),next,errors)) + return false; + unlink((keyp + ".V").c_str()); + _rmDashRf(keyp + ".O"); + } else { + FILE *f = fopen((keyp + ".V").c_str(),"w"); + if (f) { + const std::string v(i.value().dump()); + if (fwrite(v.c_str(),v.length(),1,f) != 1) { + fclose(f); + return false; + } else { + fclose(f); + } + } else { + return false; + } + _rmDashRf(keyp + ".A"); + _rmDashRf(keyp + ".O"); + } + } + + if ((previous)&&(previous->is_object())) { + for(nlohmann::json::const_iterator i(previous->begin());i!=previous->end();++i) { + if ((i.key().length() > 0)&&(obj.find(i.key()) == obj.end())) { + const std::string keyp(path + OFFBASE_PATH_SEP + escapeKey(i.key())); + unlink((keyp + ".V").c_str()); + _rmDashRf(keyp + ".A"); + _rmDashRf(keyp + ".O"); + } + } + } + + return true; + } + + std::string _path; + nlohmann::json _saved; +}; + +#endif -- cgit v1.2.3 From 3cb2e1197ff7ba1371ae69a09e631a7b87a880bc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 12 Aug 2016 15:32:45 -0700 Subject: . --- controller/SqliteNetworkController.cpp | 104 +++++++++++++++------------------ 1 file changed, 47 insertions(+), 57 deletions(-) (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 56bddbc3..226da657 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -819,15 +819,16 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( return 404; Mutex::Lock _l(_lock); - _backupNeeded = true; - if (path[0] == "network") { + json &networks = _db["network"]; + if (!networks.is_object()) networks = json::object(); if ((path.size() >= 2)&&(path[1].length() == 16)) { uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + /* int64_t revision = 0; sqlite3_reset(_sGetNetworkRevision); sqlite3_bind_text(_sGetNetworkRevision,1,nwids,16,SQLITE_STATIC); @@ -836,17 +837,23 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( networkExists = true; revision = sqlite3_column_int64(_sGetNetworkRevision,0); } + */ if (path.size() >= 3) { - - if (!networkExists) - return 404; + auto network = networks.get(nwids); + if (!network) return 404; if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { + json &members = (*network)["member"]; + if (!members.is_object()) members = json::object(); + uint64_t address = Utils::hexStrToU64(path[3].c_str()); char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + json &member = members[addrs]; + if (!member.is_object()) member = json::object(); + /* int64_t addToNetworkRevision = 0; int64_t memberRowId = 0; @@ -913,7 +920,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( sqlite3_reset(_sDeleteIpAllocations); sqlite3_bind_text(_sDeleteIpAllocations,1,nwids,16,SQLITE_STATIC); sqlite3_bind_text(_sDeleteIpAllocations,2,addrs,10,SQLITE_STATIC); - sqlite3_bind_int(_sDeleteIpAllocations,3,(int)0 /*ZT_IP_ASSIGNMENT_TYPE_ADDRESS*/); + sqlite3_bind_int(_sDeleteIpAllocations,3,(int)0); if (sqlite3_step(_sDeleteIpAllocations) != SQLITE_DONE) return 500; for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { @@ -927,7 +934,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( sqlite3_reset(_sAllocateIp); sqlite3_bind_text(_sAllocateIp,1,nwids,16,SQLITE_STATIC); sqlite3_bind_text(_sAllocateIp,2,addrs,10,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,3,(int)0 /*ZT_IP_ASSIGNMENT_TYPE_ADDRESS*/); + sqlite3_bind_int(_sAllocateIp,3,(int)0); sqlite3_bind_blob(_sAllocateIp,4,(const void *)ipBlob,16,SQLITE_STATIC); sqlite3_bind_int(_sAllocateIp,5,(int)a.netmaskBits()); // NOTE: this field is now ignored but set it anyway sqlite3_bind_int(_sAllocateIp,6,ipVersion); @@ -980,6 +987,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( sqlite3_bind_text(_sSetNetworkRevision,2,nwids,16,SQLITE_STATIC); sqlite3_step(_sSetNetworkRevision); } + */ return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType); } else if ((path.size() == 3)&&(path[2] == "test")) { @@ -990,6 +998,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( test->credentialNetworkId = nwid; test->ptr = (void *)this; + /* json_value *j = json_parse(body.c_str(),body.length()); if (j) { if (j->type == json_object) { @@ -1018,6 +1027,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } json_value_free(j); } + */ if (!test->hopCount) { ::free((void *)test); @@ -1036,7 +1046,6 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); responseBody = json; responseContentType = "application/json"; - return 200; } // else 404 @@ -1420,59 +1429,38 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpDELETE( return 404; Mutex::Lock _l(_lock); - _backupNeeded = true; - if (path[0] == "network") { + auto networks = _db.get("network"); + if (!networks) return 404; if ((path.size() >= 2)&&(path[1].length() == 16)) { - uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - sqlite3_reset(_sGetNetworkById); - sqlite3_bind_text(_sGetNetworkById,1,nwids,16,SQLITE_STATIC); - if (sqlite3_step(_sGetNetworkById) != SQLITE_ROW) - return 404; + auto network = _db.get(nwids); + if (!network) return 404; if (path.size() >= 3) { - if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - uint64_t address = Utils::hexStrToU64(path[3].c_str()); + auto members = network->get("member"); + if (!members) return 404; + + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + auto member = members->get(addrs); + if (!member) return 404; - sqlite3_reset(_sGetMember); - sqlite3_bind_text(_sGetMember,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetMember,2,addrs,10,SQLITE_STATIC); - if (sqlite3_step(_sGetMember) != SQLITE_ROW) - return 404; - - sqlite3_reset(_sDeleteIpAllocations); - sqlite3_bind_text(_sDeleteIpAllocations,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sDeleteIpAllocations,2,addrs,10,SQLITE_STATIC); - sqlite3_bind_int(_sDeleteIpAllocations,3,(int)0 /*ZT_IP_ASSIGNMENT_TYPE_ADDRESS*/); - if (sqlite3_step(_sDeleteIpAllocations) == SQLITE_DONE) { - sqlite3_reset(_sDeleteMember); - sqlite3_bind_text(_sDeleteMember,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sDeleteMember,2,addrs,10,SQLITE_STATIC); - if (sqlite3_step(_sDeleteMember) != SQLITE_DONE) - return 500; - } else return 500; - + members->erase(addrs); + responseBody = member->dump(2); + responseContentType = "application/json"; return 200; } - } else { - - sqlite3_reset(_sDeleteNetwork); - sqlite3_bind_text(_sDeleteNetwork,1,nwids,16,SQLITE_STATIC); - if (sqlite3_step(_sDeleteNetwork) == SQLITE_DONE) { - sqlite3_reset(_sDeleteAllNetworkMembers); - sqlite3_bind_text(_sDeleteAllNetworkMembers,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteAllNetworkMembers); - return 200; - } else return 500; - + networks->erase(nwids); + responseBody = network->dump(2); + responseContentType = "application/json"; + return 200; } } // else 404 @@ -1484,18 +1472,20 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpDELETE( void SqliteNetworkController::threadMain() throw() { - bool run = true; - while(run) { - Thread::sleep(250); - try { - std::vector errors; - Mutex::Lock _l(_lock); - run = _dbCommitThreadRun; - if (!_db.commit(&errors)) { + uint64_t lastCommit = OSUtils::now(); + while(_dbCommitThreadRun) { + Thread::sleep(200); + if ((OSUtils::now() - lastCommit) > 2000) { + lastCommit = OSUtils::now(); + try { + std::vector errors; + Mutex::Lock _l(_lock); + if (!_db.commit(&errors)) { + // TODO: handle anything really bad + } + } catch ( ... ) { // TODO: handle anything really bad } - } catch ( ... ) { - // TODO: handle anything really bad } } } -- cgit v1.2.3 From bd15262e5459c6003e54bcfd1d98966ff6bd1f97 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 15 Aug 2016 18:49:50 -0700 Subject: Bunch of rule JSON stuff. --- controller/SqliteNetworkController.cpp | 440 ++++++++++++++++++++++++++++++--- node/InetAddress.hpp | 13 + 2 files changed, 420 insertions(+), 33 deletions(-) (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 226da657..16c02295 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -81,6 +81,271 @@ using json = nlohmann::json; namespace ZeroTier { +struct json::object _renderRule(ZT_VirtualNetworkRule &rule) +{ + char tmp[128]; + json::object r; + r["not"] = ((rule.t & 0x80) != 0); + switch((rule.t) & 0x7f) { + case ZT_NETWORK_RULE_ACTION_DROP: + r["type"] = "ACTION_DROP"; + break; + case ZT_NETWORK_RULE_ACTION_ACCEPT: + r["type"] = "ACTION_ACCEPT"; + break; + case ZT_NETWORK_RULE_ACTION_TEE: + r["type"] = "ACTION_TEE"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_ACTION_REDIRECT: + r["type"] = "ACTION_REDIRECT"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + r["type"] = "MATCH_VLAN_ID"; + r["vlanId"] = (uint64_t)rule.v.vlanId; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + r["type"] = "MATCH_VLAN_PCP"; + r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + r["type"] = "MATCH_VLAN_DEI"; + r["vlanDei"] = (uint64_t)rule.v.vlanDei; + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + r["type"] = "MATCH_ETHERTYPE"; + r["etherType"] = (uint64_t)rule.v.etherType; + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + r["type"] = "MATCH_MAC_SOURCE"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + r["type"] = "MATCH_MAC_DEST"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + r["type"] = "MATCH_IPV4_SOURCE"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + r["type"] = "MATCH_IPV4_DEST"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + r["type"] = "MATCH_IPV6_SOURCE"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + r["type"] = "MATCH_IPV6_DEST"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + r["type"] = "MATCH_IP_TOS"; + r["ipTos"] = (uint64_t)rule.v.ipTos; + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + r["type"] = "MATCH_IP_PROTOCOL"; + r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + r["type"] = "MATCH_CHARACTERISTICS"; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); + r["mask"] = tmp; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); + r["value"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["start"] = (uint64_t)rule.v.frameSize[0]; + r["end"] = (uint64_t)rule.v.frameSize[1]; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + r["type"] = "MATCH_TAGS_SAMENESS"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + } + return r; +} + +struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) +{ + std::string t = r["type"]; + memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); + if (r.value("not",false)) + rule.t = 0x80; + else rule.t = 0x00; + if (t == "ACTION_DROP") { + rule.t |= ZT_NETWORK_RULE_ACTION_DROP; + return true; + } else if (t == "ACTION_ACCEPT") { + rule.t |= ZT_NETWORK_RULE_ACTION_ACCEPT; + return true; + } else if (t == "ACTION_TEE") { + rule.t |= ZT_NETWORK_RULE_ACTION_TEE; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "ACTION_REDIRECT") { + rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_VLAN_ID") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; + rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_VLAN_PCP") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; + rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_VLAN_DEI") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; + rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_ETHERTYPE") { + rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; + rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_MAC_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_MAC_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_IPV4_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV4_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV6_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IPV6_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IP_TOS") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; + rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_PROTOCOL") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; + rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_IP_DEST_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_CHARACTERISTICS") { + rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; + if (r.count("mask")) { + auto mask = r["mask"]; + rule.v.characteristics[0] = (mask.is_integer() ? (uint64_t)mask : Utils::hexStrToU64(mask)); + } + if (r.count("value")) { + auto value = r["value"]; + rule.v.characteristics[1] = (value.is_integer() ? (uint64_t)value : Utils::hexStrToU64(value)); + } + return true; + } else if (t == "MATCH_FRAME_SIZE_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; + rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0")) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0")) & 0xffffULL); + return true; + } else if (t == "MATCH_TAGS_SAMENESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_AND") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_OR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_XOR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } + return false; +} + SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath) : _node(node), _db(dbPath), @@ -1050,49 +1315,157 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } // else 404 } else { - std::vector path_copy(path); + // POST to network ID - if (!networkExists) { - if (path[1].substr(10) == "______") { - // A special POST /network/##########______ feature lets users create a network - // with an arbitrary unused network number at this controller. - nwid = 0; + json::object b; + try { + b = json::parse(body); + } catch ( ... ) { + return 403; + } - uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; - uint64_t nwidPostfix = 0; + // Magic ID ending with ______ picks a random unused network ID + if (path[1].substr(10) == "______") { + nwid = 0; + uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; + uint64_t nwidPostfix = 0; + for(unsigned long k=0;k<100000;++k) { // sanity limit on trials Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); - uint64_t nwidOriginalPostfix = nwidPostfix; - do { - uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); - if (!nwidPostfix) - tryNwid |= 1; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - - sqlite3_reset(_sGetNetworkRevision); - sqlite3_bind_text(_sGetNetworkRevision,1,nwids,16,SQLITE_STATIC); - if (sqlite3_step(_sGetNetworkRevision) != SQLITE_ROW) { - nwid = tryNwid; - break; + uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); + if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); + if (!networks.get(nwids)) { + nwid = tryNwid; + break; + } + } + if (!nwid) + return 503; + } + + json::object &network = networks[nwids]; + if (!network.is_object()) network = json::object(); + + try { + if (b.count("name")) network["name"] = b.get("name"); + if (b.count("private")) network["private"] = b.get("private"); + if (b.count("enableBroadcast")) network["enableBroadcast"] = b.get("enableBroadcast"); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.get("allowPassiveBridging"); + if (b.count("multicastLimit")) network["multicastLimit"] = b.get("multicastLimit"); + + if (b.count("v4AssignMode")) { + auto nv4m = network["v4AssignMode"]; + if (!nv4m.is_object()) nv6m = json::object(); + if (b["v4AssignMode"].is_string()) { // backward compatibility + nv4m["zt"] = (b.get("v4AssignMode") == "zt"); + } else if (b["v4AssignMode"].is_object()) { + auto v4m = b["v4AssignMode"]; + if (v4m.count("zt")) nv4m["zt"] = v4m.get("zt"); + } + if (!nv4m.count("zt")) nv4m["zt"] = false; + } + + if (b.count("v6AssignMode")) { + auto nv6m = network["v6AssignMode"]; + if (!nv6m.is_object()) nv6m = json::object(); + if (b["v6AssignMode"].is_string()) { // backward compatibility + std::vector v6m(Utils::split(b.get("v6AssignMode").c_str(),",","","")); + std::sort(v6m.begin(),v6m.end()); + v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); + for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { + if (*i == "rfc4193") + nv6m["rfc4193"] = true; + else if (*i == "zt") + nv6m["zt"] = true; + else if (*i == "6plane") + nv6m["6plane"] = true; } + } else if (b["v6AssignMode"].is_object()) { + auto v6m = b["v6AssignMode"]; + if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.get("rfc4193"); + if (v6m.count("zt")) nv6m["rfc4193"] = v6m.get("zt"); + if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.get("6plane"); + } + if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; + if (!nv6m.count("zt")) nv6m["zt"] = false; + if (!nv6m.count("6plane")) nv6m["6plane"] = false; + } - ++nwidPostfix; - } while (nwidPostfix != nwidOriginalPostfix); + if (b.count("routes")) { + auto rts = b["routes"]; + if (rts.is_array()) { + for(unsigned long i=0;i("target")); + InetAddress v(rt.get("via")); + if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.ss_family == v.ss_family) && (t.netmaskBitsValid()) ) { + auto nrts = network["routes"]; + if (!nrts.is_array()) nrts = json::array(); + json::object tmp; + tmp["target"] = target.toString(); + tmp["via"] = target.toIpString(); + nrts.push_back(tmp); + } + } + } + } + } - // 503 means we have no more free IDs for this prefix. You shouldn't host anywhere - // near 16 million networks on the same controller, so shouldn't happen. - if (!nwid) - return 503; + if (b.count("ipAssignmentPools")) { + auto ipp = b["ipAssignmentPools"]; + if (ipp.is_array()) { + for(unsigned long i=0;i("ipRangeStart")); + InetAddress t(ip.get("ipRangeEnd")); + if ( ((f.ss_family == AF_INET)||(f.ss_family == AF_INET6)) && (f.ss_family == t.ss_family) ) { + auto nipp = network["ipAssignmentPools"]; + if (!nipp.is_array()) nipp = json::array(); + json::object tmp; + tmp["ipRangeStart"] = f.toIpString(); + tmp["ipRangeEnd"] = t.toIpString(); + nipp.push_back(tmp); + } + } + } + } } - sqlite3_reset(_sCreateNetwork); - sqlite3_bind_text(_sCreateNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sCreateNetwork,2,"",0,SQLITE_STATIC); - sqlite3_bind_int64(_sCreateNetwork,3,(long long)OSUtils::now()); - if (sqlite3_step(_sCreateNetwork) != SQLITE_DONE) - return 500; - path_copy[1].assign(nwids); + if (b.count("rules")) { + auto rules = b["rules"]; + if (rules.is_array()) { + for(unsigned long i=0;i("revision") + 1ULL; + + return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); + + /* json_value *j = json_parse(body.c_str(),body.length()); if (j) { if (j->type == json_object) { @@ -1409,6 +1782,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); } + */ } // else 404 diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index b97e2ca6..2cd4dbd3 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -300,6 +300,19 @@ struct InetAddress : public sockaddr_storage */ inline unsigned int netmaskBits() const throw() { return port(); } + /** + * @return True if netmask bits is valid for the address type + */ + inline bool netmaskBitsValid() const + { + const unsigned int n = port(); + switch(ss_family) { + case AF_INET: return (n <= 32); + case AF_INET6: return (n <= 128); + } + return false; + } + /** * Alias for port() * -- cgit v1.2.3 From b08ca49580c63abe79a650adbb0d14ca87a1cd24 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 16 Aug 2016 14:05:17 -0700 Subject: More controller work -- it builds! --- controller/SqliteNetworkController.cpp | 729 +-- controller/SqliteNetworkController.hpp | 77 +- ext/json/LICENSE.MIT | 22 + ext/json/README.md | 511 ++ ext/json/json.hpp | 10435 +++++++++++++++++++++++++++++++ ext/offbase/README.md | 59 - ext/offbase/json/LICENSE.MIT | 22 - ext/offbase/json/README.md | 511 -- ext/offbase/json/json.hpp | 10435 ------------------------------- ext/offbase/offbase.hpp | 393 -- make-mac.mk | 4 +- node/InetAddress.cpp | 2 +- osdep/OSUtils.cpp | 80 + osdep/OSUtils.hpp | 18 +- 14 files changed, 11280 insertions(+), 12018 deletions(-) create mode 100644 ext/json/LICENSE.MIT create mode 100644 ext/json/README.md create mode 100644 ext/json/json.hpp delete mode 100644 ext/offbase/README.md delete mode 100644 ext/offbase/json/LICENSE.MIT delete mode 100644 ext/offbase/json/README.md delete mode 100644 ext/offbase/json/json.hpp delete mode 100644 ext/offbase/offbase.hpp (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 16c02295..94d2e2e6 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -53,8 +53,6 @@ #include "../node/MAC.hpp" #include "../node/Address.hpp" -#include "../osdep/OSUtils.hpp" - // offbase includes and builds upon nlohmann::json using json = nlohmann::json; @@ -73,18 +71,15 @@ using json = nlohmann::json; // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 -// Delay between backups in milliseconds -//#define ZT_NETCONF_BACKUP_PERIOD 300000 - // Nodes are considered active if they've queried in less than this long #define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) namespace ZeroTier { -struct json::object _renderRule(ZT_VirtualNetworkRule &rule) +static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; - json::object r; + json r = json::object(); r["not"] = ((rule.t & 0x80) != 0); switch((rule.t) & 0x7f) { case ZT_NETWORK_RULE_ACTION_DROP: @@ -205,8 +200,10 @@ struct json::object _renderRule(ZT_VirtualNetworkRule &rule) return r; } -struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) +static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) { + if (r.is_object()) + return false; std::string t = r["type"]; memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); if (r.value("not",false)) @@ -220,19 +217,19 @@ struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_VLAN_ID") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; @@ -309,48 +306,58 @@ struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; if (r.count("mask")) { - auto mask = r["mask"]; - rule.v.characteristics[0] = (mask.is_integer() ? (uint64_t)mask : Utils::hexStrToU64(mask)); + auto v = r["mask"]; + if (v.is_number()) { + rule.v.characteristics[0] = v; + } else { + std::string tmp = v; + rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); + } } if (r.count("value")) { - auto value = r["value"]; - rule.v.characteristics[1] = (value.is_integer() ? (uint64_t)value : Utils::hexStrToU64(value)); + auto v = r["value"]; + if (v.is_number()) { + rule.v.characteristics[1] = v; + } else { + std::string tmp = v; + rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); + } } return true; } else if (t == "MATCH_FRAME_SIZE_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0")) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0")) & 0xffffULL); + rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0").c_str()) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0").c_str()) & 0xffffULL); return true; } else if (t == "MATCH_TAGS_SAMENESS") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } return false; } -SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath) : +SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath) : _node(node), - _db(dbPath), - _dbCommitThreadRun(true) + _path(dbPath) { + OSUtils::mkdir(dbPath); /* if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) throw std::runtime_error("SqliteNetworkController cannot open database file"); @@ -592,16 +599,10 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c _backupThread = Thread::start(this); */ - - _dbCommitThread = Thread::start(this); } SqliteNetworkController::~SqliteNetworkController() { - _lock.lock(); - _dbCommitThreadRun = false; - _lock.unlock(); - Thread::join(_dbCommitThread); } NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) @@ -1068,7 +1069,6 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpGET( std::string &responseBody, std::string &responseContentType) { - Mutex::Lock _l(_lock); return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType); } @@ -1082,41 +1082,21 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( { if (path.empty()) return 404; - Mutex::Lock _l(_lock); if (path[0] == "network") { - json &networks = _db["network"]; - if (!networks.is_object()) networks = json::object(); if ((path.size() >= 2)&&(path[1].length() == 16)) { uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - /* - int64_t revision = 0; - sqlite3_reset(_sGetNetworkRevision); - sqlite3_bind_text(_sGetNetworkRevision,1,nwids,16,SQLITE_STATIC); - bool networkExists = false; - if (sqlite3_step(_sGetNetworkRevision) == SQLITE_ROW) { - networkExists = true; - revision = sqlite3_column_int64(_sGetNetworkRevision,0); - } - */ - if (path.size() >= 3) { - auto network = networks.get(nwids); - if (!network) return 404; + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - json &members = (*network)["member"]; - if (!members.is_object()) members = json::object(); - uint64_t address = Utils::hexStrToU64(path[3].c_str()); - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - json &member = members[addrs]; - if (!member.is_object()) member = json::object(); /* int64_t addToNetworkRevision = 0; @@ -1256,6 +1236,9 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType); } else if ((path.size() == 3)&&(path[2] == "test")) { + + Mutex::Lock _l(_circuitTests_m); + ZT_CircuitTest *test = (ZT_CircuitTest *)malloc(sizeof(ZT_CircuitTest)); memset(test,0,sizeof(ZT_CircuitTest)); @@ -1263,6 +1246,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( test->credentialNetworkId = nwid; test->ptr = (void *)this; + // TODO TODO /* json_value *j = json_parse(body.c_str(),body.length()); if (j) { @@ -1312,12 +1296,13 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( responseBody = json; responseContentType = "application/json"; return 200; + } // else 404 } else { // POST to network ID - json::object b; + json b; try { b = json::parse(body); } catch ( ... ) { @@ -1334,7 +1319,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (!networks.get(nwids)) { + if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) { nwid = tryNwid; break; } @@ -1343,24 +1328,23 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( return 503; } - json::object &network = networks[nwids]; - if (!network.is_object()) network = json::object(); + json network(_readJson(_networkJP(nwid,true))); try { - if (b.count("name")) network["name"] = b.get("name"); - if (b.count("private")) network["private"] = b.get("private"); - if (b.count("enableBroadcast")) network["enableBroadcast"] = b.get("enableBroadcast"); - if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.get("allowPassiveBridging"); - if (b.count("multicastLimit")) network["multicastLimit"] = b.get("multicastLimit"); + if (b.count("name")) network["name"] = b.value("name",""); + if (b.count("private")) network["private"] = b.value("private",true); + if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); + if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); if (b.count("v4AssignMode")) { auto nv4m = network["v4AssignMode"]; - if (!nv4m.is_object()) nv6m = json::object(); + if (!nv4m.is_object()) nv4m = json::object(); if (b["v4AssignMode"].is_string()) { // backward compatibility - nv4m["zt"] = (b.get("v4AssignMode") == "zt"); + nv4m["zt"] = (b.value("v4AssignMode","") == "zt"); } else if (b["v4AssignMode"].is_object()) { auto v4m = b["v4AssignMode"]; - if (v4m.count("zt")) nv4m["zt"] = v4m.get("zt"); + if (v4m.count("zt")) nv4m["zt"] = v4m.value("zt",false); } if (!nv4m.count("zt")) nv4m["zt"] = false; } @@ -1369,7 +1353,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( auto nv6m = network["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (b["v6AssignMode"].is_string()) { // backward compatibility - std::vector v6m(Utils::split(b.get("v6AssignMode").c_str(),",","","")); + std::vector v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); std::sort(v6m.begin(),v6m.end()); v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { @@ -1382,9 +1366,9 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } } else if (b["v6AssignMode"].is_object()) { auto v6m = b["v6AssignMode"]; - if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.get("rfc4193"); - if (v6m.count("zt")) nv6m["rfc4193"] = v6m.get("zt"); - if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.get("6plane"); + if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); + if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); + if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); } if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; if (!nv6m.count("zt")) nv6m["zt"] = false; @@ -1397,14 +1381,14 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i("target")); - InetAddress v(rt.get("via")); + InetAddress t(rt.value("target","")); + InetAddress v(rt.value("via","")); if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.ss_family == v.ss_family) && (t.netmaskBitsValid()) ) { auto nrts = network["routes"]; if (!nrts.is_array()) nrts = json::array(); - json::object tmp; - tmp["target"] = target.toString(); - tmp["via"] = target.toIpString(); + json tmp; + tmp["target"] = t.toString(); + tmp["via"] = v.toIpString(); nrts.push_back(tmp); } } @@ -1418,12 +1402,12 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i("ipRangeStart")); - InetAddress t(ip.get("ipRangeEnd")); + InetAddress f(ip.value("ipRangeStart","")); + InetAddress t(ip.value("ipRangeEnd","")); if ( ((f.ss_family == AF_INET)||(f.ss_family == AF_INET6)) && (f.ss_family == t.ss_family) ) { auto nipp = network["ipAssignmentPools"]; if (!nipp.is_array()) nipp = json::array(); - json::object tmp; + json tmp; tmp["ipRangeStart"] = f.toIpString(); tmp["ipRangeEnd"] = t.toIpString(); nipp.push_back(tmp); @@ -1461,328 +1445,14 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } network["lastModified"] = OSUtils::now(); - network["revision"] = network.get("revision") + 1ULL; - - return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); - - /* - json_value *j = json_parse(body.c_str(),body.length()); - if (j) { - if (j->type == json_object) { - for(unsigned int k=0;ku.object.length;++k) { - sqlite3_stmt *stmt = (sqlite3_stmt *)0; - - if (!strcmp(j->u.object.values[k].name,"name")) { - if ((j->u.object.values[k].value->type == json_string)&&(j->u.object.values[k].value->u.string.ptr[0])) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"name\" = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_text(stmt,1,j->u.object.values[k].value->u.string.ptr,-1,SQLITE_STATIC); - } - } else if (!strcmp(j->u.object.values[k].name,"private")) { - if (j->u.object.values[k].value->type == json_boolean) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"private\" = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - } - } else if (!strcmp(j->u.object.values[k].name,"enableBroadcast")) { - if (j->u.object.values[k].value->type == json_boolean) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET enableBroadcast = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - } - } else if (!strcmp(j->u.object.values[k].name,"allowPassiveBridging")) { - if (j->u.object.values[k].value->type == json_boolean) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET allowPassiveBridging = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - } - } else if (!strcmp(j->u.object.values[k].name,"v4AssignMode")) { - if ((j->u.object.values[k].value->type == json_string)&&(!strcmp(j->u.object.values[k].value->u.string.ptr,"zt"))) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"flags\" = (\"flags\" | ?) WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(int)ZT_DB_NETWORK_FLAG_ZT_MANAGED_V4_AUTO_ASSIGN); - } else { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"flags\" = (\"flags\" & ?) WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(int)(ZT_DB_NETWORK_FLAG_ZT_MANAGED_V4_AUTO_ASSIGN ^ 0xfffffff)); - } - } else if (!strcmp(j->u.object.values[k].name,"v6AssignMode")) { - int fl = 0; - if (j->u.object.values[k].value->type == json_string) { - char *saveptr = (char *)0; - for(char *f=Utils::stok(j->u.object.values[k].value->u.string.ptr,",",&saveptr);(f);f=Utils::stok((char *)0,",",&saveptr)) { - if (!strcmp(f,"rfc4193")) - fl |= ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_RFC4193; - else if (!strcmp(f,"6plane")) - fl |= ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_6PLANE; - else if (!strcmp(f,"zt")) - fl |= ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_AUTO_ASSIGN; - } - } - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"flags\" = ((\"flags\" & " ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_MASK_S ") | ?) WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,fl); - } else if (!strcmp(j->u.object.values[k].name,"multicastLimit")) { - if (j->u.object.values[k].value->type == json_integer) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET multicastLimit = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(int)j->u.object.values[k].value->u.integer); - } - } else if (!strcmp(j->u.object.values[k].name,"relays")) { - if (j->u.object.values[k].value->type == json_array) { - std::map nodeIdToPhyAddress; - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *relay = j->u.object.values[k].value->u.array.values[kk]; - const char *address = (const char *)0; - const char *phyAddress = (const char *)0; - if ((relay)&&(relay->type == json_object)) { - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(relay->u.object.values[rk].name,"address"))&&(relay->u.object.values[rk].value->type == json_string)) - address = relay->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(relay->u.object.values[rk].name,"phyAddress"))&&(relay->u.object.values[rk].value->type == json_string)) - phyAddress = relay->u.object.values[rk].value->u.string.ptr; - } - } - if ((address)&&(phyAddress)) - nodeIdToPhyAddress[Address(address)] = InetAddress(phyAddress); - } - - sqlite3_reset(_sDeleteRelaysForNetwork); - sqlite3_bind_text(_sDeleteRelaysForNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteRelaysForNetwork); - - for(std::map::iterator rl(nodeIdToPhyAddress.begin());rl!=nodeIdToPhyAddress.end();++rl) { - sqlite3_reset(_sCreateRelay); - sqlite3_bind_text(_sCreateRelay,1,nwids,16,SQLITE_STATIC); - std::string a(rl->first.toString()),b(rl->second.toString()); // don't destroy strings until sqlite3_step() - sqlite3_bind_text(_sCreateRelay,2,a.c_str(),-1,SQLITE_STATIC); - sqlite3_bind_text(_sCreateRelay,3,b.c_str(),-1,SQLITE_STATIC); - sqlite3_step(_sCreateRelay); - } - } - } else if (!strcmp(j->u.object.values[k].name,"routes")) { - sqlite3_reset(_sDeleteRoutes); - sqlite3_bind_text(_sDeleteRoutes,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteRoutes); - if (j->u.object.values[k].value->type == json_array) { - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *r = j->u.object.values[k].value->u.array.values[kk]; - if ((r)&&(r->type == json_object)) { - InetAddress r_target,r_via; - int r_flags = 0; - int r_metric = 0; - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(r->u.object.values[rk].name,"target"))&&(r->u.object.values[rk].value->type == json_string)) - r_target = InetAddress(std::string(r->u.object.values[rk].value->u.string.ptr)); - else if ((!strcmp(r->u.object.values[rk].name,"via"))&&(r->u.object.values[rk].value->type == json_string)) - r_via = InetAddress(std::string(r->u.object.values[rk].value->u.string.ptr),0); - else if ((!strcmp(r->u.object.values[rk].name,"flags"))&&(r->u.object.values[rk].value->type == json_integer)) - r_flags = (int)(r->u.object.values[rk].value->u.integer & 0xffff); - else if ((!strcmp(r->u.object.values[rk].name,"metric"))&&(r->u.object.values[rk].value->type == json_integer)) - r_metric = (int)(r->u.object.values[rk].value->u.integer & 0xffff); - } - if ((r_target)&&((!r_via)||(r_via.ss_family == r_target.ss_family))) { - int r_ipVersion = 0; - char r_targetBlob[16]; - char r_viaBlob[16]; - _ipToBlob(r_target,r_targetBlob,r_ipVersion); - if (r_ipVersion) { - int r_targetNetmaskBits = r_target.netmaskBits(); - if ((r_ipVersion == 4)&&(r_targetNetmaskBits > 32)) r_targetNetmaskBits = 32; - else if ((r_ipVersion == 6)&&(r_targetNetmaskBits > 128)) r_targetNetmaskBits = 128; - sqlite3_reset(_sCreateRoute); - sqlite3_bind_text(_sCreateRoute,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_blob(_sCreateRoute,2,(const void *)r_targetBlob,16,SQLITE_STATIC); - if (r_via) { - _ipToBlob(r_via,r_viaBlob,r_ipVersion); - sqlite3_bind_blob(_sCreateRoute,3,(const void *)r_viaBlob,16,SQLITE_STATIC); - } else { - sqlite3_bind_null(_sCreateRoute,3); - } - sqlite3_bind_int(_sCreateRoute,4,r_targetNetmaskBits); - sqlite3_bind_int(_sCreateRoute,5,r_ipVersion); - sqlite3_bind_int(_sCreateRoute,6,r_flags); - sqlite3_bind_int(_sCreateRoute,7,r_metric); - sqlite3_step(_sCreateRoute); - } - } - } - } - } - } else if (!strcmp(j->u.object.values[k].name,"ipAssignmentPools")) { - if (j->u.object.values[k].value->type == json_array) { - std::vector< std::pair > pools; - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *pool = j->u.object.values[k].value->u.array.values[kk]; - const char *iprs = (const char *)0; - const char *ipre = (const char *)0; - if ((pool)&&(pool->type == json_object)) { - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(pool->u.object.values[rk].name,"ipRangeStart"))&&(pool->u.object.values[rk].value->type == json_string)) - iprs = pool->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(pool->u.object.values[rk].name,"ipRangeEnd"))&&(pool->u.object.values[rk].value->type == json_string)) - ipre = pool->u.object.values[rk].value->u.string.ptr; - } - } - if ((iprs)&&(ipre)) { - InetAddress iprs2(iprs); - InetAddress ipre2(ipre); - if (iprs2.ss_family == ipre2.ss_family) { - iprs2.setPort(0); - ipre2.setPort(0); - pools.push_back(std::pair(iprs2,ipre2)); - } - } - } - std::sort(pools.begin(),pools.end()); - pools.erase(std::unique(pools.begin(),pools.end()),pools.end()); - - sqlite3_reset(_sDeleteIpAssignmentPoolsForNetwork); - sqlite3_bind_text(_sDeleteIpAssignmentPoolsForNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteIpAssignmentPoolsForNetwork); - - for(std::vector< std::pair >::const_iterator p(pools.begin());p!=pools.end();++p) { - char ipBlob1[16],ipBlob2[16]; - sqlite3_reset(_sCreateIpAssignmentPool); - sqlite3_bind_text(_sCreateIpAssignmentPool,1,nwids,16,SQLITE_STATIC); - if (p->first.ss_family == AF_INET) { - memset(ipBlob1,0,12); - memcpy(ipBlob1 + 12,p->first.rawIpData(),4); - memset(ipBlob2,0,12); - memcpy(ipBlob2 + 12,p->second.rawIpData(),4); - sqlite3_bind_blob(_sCreateIpAssignmentPool,2,(const void *)ipBlob1,16,SQLITE_STATIC); - sqlite3_bind_blob(_sCreateIpAssignmentPool,3,(const void *)ipBlob2,16,SQLITE_STATIC); - sqlite3_bind_int(_sCreateIpAssignmentPool,4,4); - } else if (p->first.ss_family == AF_INET6) { - sqlite3_bind_blob(_sCreateIpAssignmentPool,2,p->first.rawIpData(),16,SQLITE_STATIC); - sqlite3_bind_blob(_sCreateIpAssignmentPool,3,p->second.rawIpData(),16,SQLITE_STATIC); - sqlite3_bind_int(_sCreateIpAssignmentPool,4,6); - } else continue; - sqlite3_step(_sCreateIpAssignmentPool); - } - } - } else if (!strcmp(j->u.object.values[k].name,"rules")) { - if (j->u.object.values[k].value->type == json_array) { - sqlite3_reset(_sDeleteRulesForNetwork); - sqlite3_bind_text(_sDeleteRulesForNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteRulesForNetwork); - - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *rj = j->u.object.values[k].value->u.array.values[kk]; - if ((rj)&&(rj->type == json_object)) { - struct { // NULL pointers indicate missing or NULL -- wildcards - const json_int_t *ruleNo; - const char *nodeId; - const char *sourcePort; - const char *destPort; - const json_int_t *vlanId; - const json_int_t *vlanPcp; - const json_int_t *etherType; - const char *macSource; - const char *macDest; - const char *ipSource; - const char *ipDest; - const json_int_t *ipTos; - const json_int_t *ipProtocol; - const json_int_t *ipSourcePort; - const json_int_t *ipDestPort; - const json_int_t *flags; - const json_int_t *invFlags; - const char *action; - } rule; - memset(&rule,0,sizeof(rule)); - - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(rj->u.object.values[rk].name,"ruleNo"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ruleNo = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"nodeId"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.nodeId = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"sourcePort"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.sourcePort = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"destPort"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.destPort = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"vlanId"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.vlanId = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"vlanPcp"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.vlanPcp = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"etherType"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.etherType = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"macSource"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.macSource = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"macDest"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.macDest = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"ipSource"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.ipSource = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"ipDest"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.ipDest = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"ipTos"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipTos = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"ipProtocol"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipProtocol = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"ipSourcePort"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipSourcePort = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"ipDestPort"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipDestPort = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"flags"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.flags = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"invFlags"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.invFlags = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"action"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.action = rj->u.object.values[rk].value->u.string.ptr; - } - - if ((rule.ruleNo)&&(rule.action)&&(rule.action[0])) { - char mactmp1[16],mactmp2[16]; - sqlite3_reset(_sCreateRule); - sqlite3_bind_text(_sCreateRule,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_int64(_sCreateRule,2,*rule.ruleNo); - - // Optional values: null by default - for(int i=3;i<=18;++i) - sqlite3_bind_null(_sCreateRule,i); - if ((rule.nodeId)&&(strlen(rule.nodeId) == 10)) sqlite3_bind_text(_sCreateRule,3,rule.nodeId,10,SQLITE_STATIC); - if ((rule.sourcePort)&&(strlen(rule.sourcePort) == 10)) sqlite3_bind_text(_sCreateRule,4,rule.sourcePort,10,SQLITE_STATIC); - if ((rule.destPort)&&(strlen(rule.destPort) == 10)) sqlite3_bind_text(_sCreateRule,5,rule.destPort,10,SQLITE_STATIC); - if (rule.vlanId) sqlite3_bind_int(_sCreateRule,6,(int)*rule.vlanId); - if (rule.vlanPcp) sqlite3_bind_int(_sCreateRule,7,(int)*rule.vlanPcp); - if (rule.etherType) sqlite3_bind_int(_sCreateRule,8,(int)*rule.etherType & (int)0xffff); - if (rule.macSource) { - MAC m(rule.macSource); - Utils::snprintf(mactmp1,sizeof(mactmp1),"%.12llx",(unsigned long long)m.toInt()); - sqlite3_bind_text(_sCreateRule,9,mactmp1,-1,SQLITE_STATIC); - } - if (rule.macDest) { - MAC m(rule.macDest); - Utils::snprintf(mactmp2,sizeof(mactmp2),"%.12llx",(unsigned long long)m.toInt()); - sqlite3_bind_text(_sCreateRule,10,mactmp2,-1,SQLITE_STATIC); - } - if (rule.ipSource) sqlite3_bind_text(_sCreateRule,11,rule.ipSource,-1,SQLITE_STATIC); - if (rule.ipDest) sqlite3_bind_text(_sCreateRule,12,rule.ipDest,-1,SQLITE_STATIC); - if (rule.ipTos) sqlite3_bind_int(_sCreateRule,13,(int)*rule.ipTos); - if (rule.ipProtocol) sqlite3_bind_int(_sCreateRule,14,(int)*rule.ipProtocol); - if (rule.ipSourcePort) sqlite3_bind_int(_sCreateRule,15,(int)*rule.ipSourcePort & (int)0xffff); - if (rule.ipDestPort) sqlite3_bind_int(_sCreateRule,16,(int)*rule.ipDestPort & (int)0xffff); - if (rule.flags) sqlite3_bind_int64(_sCreateRule,17,(int64_t)*rule.flags); - if (rule.invFlags) sqlite3_bind_int64(_sCreateRule,18,(int64_t)*rule.invFlags); - - sqlite3_bind_text(_sCreateRule,19,rule.action,-1,SQLITE_STATIC); - sqlite3_step(_sCreateRule); - } - } - } - } - } + network["revision"] = network.value("revision",0ULL) + 1ULL; - if (stmt) { - sqlite3_bind_text(stmt,2,nwids,16,SQLITE_STATIC); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - } - } - } - json_value_free(j); - } - - sqlite3_reset(_sSetNetworkRevision); - sqlite3_bind_int64(_sSetNetworkRevision,1,revision += 1); - sqlite3_bind_text(_sSetNetworkRevision,2,nwids,16,SQLITE_STATIC); - sqlite3_step(_sSetNetworkRevision); + _writeJson(_networkJP(nwid,true),network); - return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); - } - */ + responseBody = network.dump(2); + responseContentType = "application/json"; + return 200; + } // else 404 } // else 404 @@ -1801,38 +1471,32 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpDELETE( { if (path.empty()) return 404; - Mutex::Lock _l(_lock); if (path[0] == "network") { - auto networks = _db.get("network"); - if (!networks) return 404; - if ((path.size() >= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - auto network = _db.get(nwids); - if (!network) return 404; + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - auto members = network->get("member"); - if (!members) return 404; - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - auto member = members->get(addrs); - if (!member) return 404; - members->erase(addrs); - responseBody = member->dump(2); + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + + OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str()); + + responseBody = member.dump(2); responseContentType = "application/json"; return 200; } } else { - networks->erase(nwids); - responseBody = network->dump(2); + OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); + responseBody = network.dump(2); responseContentType = "application/json"; return 200; } @@ -1843,27 +1507,6 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpDELETE( return 404; } -void SqliteNetworkController::threadMain() - throw() -{ - uint64_t lastCommit = OSUtils::now(); - while(_dbCommitThreadRun) { - Thread::sleep(200); - if ((OSUtils::now() - lastCommit) > 2000) { - lastCommit = OSUtils::now(); - try { - std::vector errors; - Mutex::Lock _l(_lock); - if (!_db.commit(&errors)) { - // TODO: handle anything really bad - } - } catch ( ... ) { - // TODO: handle anything really bad - } - } - } -} - unsigned int SqliteNetworkController::_doCPGet( const std::vector &path, const std::map &urlArgs, @@ -1874,137 +1517,79 @@ unsigned int SqliteNetworkController::_doCPGet( { // Assumes _lock is locked if ((path.size() > 0)&&(path[0] == "network")) { - auto networks = _db.get("network"); - if (!networks) return 404; if ((path.size() >= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - auto network = _db.get(nwids); - if (!network) return 404; + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; if (path.size() >= 3) { if (path[2] == "member") { - auto members = network->get("member"); - if (!members) return 404; if (path.size() >= 4) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - auto member = members->get(addrs); - if (!member) return 404; - nlohmann::json o(member); + json o(member); o["nwid"] = nwids; o["address"] = addrs; - o["controllerInstanceId"] = _instanceId; o["clock"] = OSUtils::now(); responseBody = o.dump(2); responseContentType = "application/json"; - return 200; - /* - sqlite3_reset(_sGetMember2); - sqlite3_bind_text(_sGetMember2,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetMember2,2,addrs,10,SQLITE_STATIC); - if (sqlite3_step(_sGetMember2) == SQLITE_ROW) { - const char *memberIdStr = (const char *)sqlite3_column_text(_sGetMember2,3); - - Utils::snprintf(json,sizeof(json), - "{\n" - "\t\"nwid\": \"%s\",\n" - "\t\"address\": \"%s\",\n" - "\t\"controllerInstanceId\": \"%s\",\n" - "\t\"authorized\": %s,\n" - "\t\"activeBridge\": %s,\n" - "\t\"memberRevision\": %llu,\n" - "\t\"clock\": %llu,\n" - "\t\"identity\": \"%s\",\n" - "\t\"ipAssignments\": [", - nwids, - addrs, - _instanceId.c_str(), - (sqlite3_column_int(_sGetMember2,0) > 0) ? "true" : "false", - (sqlite3_column_int(_sGetMember2,1) > 0) ? "true" : "false", - (unsigned long long)sqlite3_column_int64(_sGetMember2,2), - (unsigned long long)OSUtils::now(), - _jsonEscape(memberIdStr).c_str()); - responseBody = json; - - sqlite3_reset(_sGetIpAssignmentsForNode); - sqlite3_bind_text(_sGetIpAssignmentsForNode,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetIpAssignmentsForNode,2,addrs,10,SQLITE_STATIC); - bool firstIp = true; - while (sqlite3_step(_sGetIpAssignmentsForNode) == SQLITE_ROW) { - int ipversion = sqlite3_column_int(_sGetIpAssignmentsForNode,2); - char ipBlob[16]; - memcpy(ipBlob,(const void *)sqlite3_column_blob(_sGetIpAssignmentsForNode,0),16); - InetAddress ip( - (const void *)(ipversion == 6 ? ipBlob : &ipBlob[12]), - (ipversion == 6 ? 16 : 4), - (unsigned int)sqlite3_column_int(_sGetIpAssignmentsForNode,1) - ); - responseBody.append(firstIp ? "\"" : ",\""); - responseBody.append(_jsonEscape(ip.toIpString())); - responseBody.push_back('"'); - firstIp = false; - } - - responseBody.append("],\n\t\"recentLog\": ["); + return 200; + } else { - const void *histb = sqlite3_column_blob(_sGetMember2,6); - if (histb) { - MemberRecentHistory rh; - rh.fromBlob((const char *)histb,sqlite3_column_bytes(_sGetMember2,6)); - for(MemberRecentHistory::const_iterator i(rh.begin());i!=rh.end();++i) { - if (i != rh.begin()) - responseBody.push_back(','); + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); + responseBody.append("\":"); + const std::string rc = member.value("memberRevision","0"); + responseBody.append(rc); } } - - responseBody.append("]\n}\n"); - - responseContentType = "application/json"; - return 200; - } // else 404 - */ - - } else { - - responseBody.push_back('{'); - for(auto i(members->begin());i!=members->end();++i) { - responseBody.append((i == members->begin()) ? "\"" : ",\""); - responseBody.append(i->key()); - responseBody.append("\":\""); - const std::string rc = i->value().value("memberRevision","0"); - responseBody.append(rc); - responseBody.append('"'); } responseBody.push_back('}'); responseContentType = "application/json"; - return 200; + return 200; } } else if ((path[2] == "active")&&(path.size() == 3)) { - responseBody.push_back('{'); - bool firstMember = true; + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; - for(auto i(members->begin());i!=members->end();++i) { - auto recentLog = i->value()->get("recentLog"); - if ((recentLog)&&(recentLog.size() > 0)) { - auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { - responseBody.append((firstMember) ? "\"" : ",\""); - firstMember = false; - responseBody.append(i->key()); - responseBody.append("\":"); - responseBody.append(mostRecentLog.dump()); + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + auto recentLog = member["recentLog"]; + if ((recentLog.is_array())&&(recentLog.size() > 0)) { + auto mostRecentLog = recentLog[0]; + if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + responseBody.append(mostRecentLog.dump()); + } + } } } } @@ -2014,6 +1599,7 @@ unsigned int SqliteNetworkController::_doCPGet( } else if ((path[2] == "test")&&(path.size() >= 4)) { + Mutex::Lock _l(_circuitTests_m); std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); if ((cte != _circuitTests.end())&&(cte->second.test)) { @@ -2032,7 +1618,6 @@ unsigned int SqliteNetworkController::_doCPGet( nlohmann::json o(network); o["nwid"] = nwids; - o["controllerInstanceId"] = _instanceId; o["clock"] = OSUtils::now(); responseBody = o.dump(2); responseContentType = "application/json"; @@ -2042,10 +1627,13 @@ unsigned int SqliteNetworkController::_doCPGet( } else if (path.size() == 1) { responseBody = "["; - for(auto i(networks->begin());i!=networks.end();++i) { - responseBody.append((i == networks->begin()) ? "\"" : ",\""); - responseBody.append(i->key()); - responseBody.append("\""); + std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); + for(auto i(networks.begin());i!=networks.end();++i) { + if (i->length() == 16) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\""); + } } responseBody.push_back(']'); responseContentType = "application/json"; @@ -2053,16 +1641,11 @@ unsigned int SqliteNetworkController::_doCPGet( } // else 404 - } else if ((path.size() > 0)&&(path[0] == "_dump")) { - - responseBody = _db.dump(2); - responseContentType = "application/json"; - return 200; - } else { - Utils::snprintf(json,sizeof(json),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu,\n\t\"instanceId\": \"%s\"\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now(),_instanceId.c_str()); - responseBody = json; + char tmp[4096]; + Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + responseBody = tmp; responseContentType = "application/json"; return 200; @@ -2081,7 +1664,7 @@ void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest if (!report) return; - Mutex::Lock _l(self->_lock); + Mutex::Lock _l(self->_circuitTests_m); std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? @@ -2092,25 +1675,25 @@ void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest Utils::snprintf(tmp,sizeof(tmp), "%s{\n" - "\t\"timestamp\": %llu,"ZT_EOL_S - "\t\"testId\": \"%.16llx\","ZT_EOL_S - "\t\"upstream\": \"%.10llx\","ZT_EOL_S - "\t\"current\": \"%.10llx\","ZT_EOL_S - "\t\"receivedTimestamp\": %llu,"ZT_EOL_S - "\t\"remoteTimestamp\": %llu,"ZT_EOL_S - "\t\"sourcePacketId\": \"%.16llx\","ZT_EOL_S - "\t\"flags\": %llu,"ZT_EOL_S - "\t\"sourcePacketHopCount\": %u,"ZT_EOL_S - "\t\"errorCode\": %u,"ZT_EOL_S - "\t\"vendor\": %d,"ZT_EOL_S - "\t\"protocolVersion\": %u,"ZT_EOL_S - "\t\"majorVersion\": %u,"ZT_EOL_S - "\t\"minorVersion\": %u,"ZT_EOL_S - "\t\"revision\": %u,"ZT_EOL_S - "\t\"platform\": %d,"ZT_EOL_S - "\t\"architecture\": %d,"ZT_EOL_S - "\t\"receivedOnLocalAddress\": \"%s\","ZT_EOL_S - "\t\"receivedFromRemoteAddress\": \"%s\""ZT_EOL_S + "\t\"timestamp\": %llu," ZT_EOL_S + "\t\"testId\": \"%.16llx\"," ZT_EOL_S + "\t\"upstream\": \"%.10llx\"," ZT_EOL_S + "\t\"current\": \"%.10llx\"," ZT_EOL_S + "\t\"receivedTimestamp\": %llu," ZT_EOL_S + "\t\"remoteTimestamp\": %llu," ZT_EOL_S + "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S + "\t\"flags\": %llu," ZT_EOL_S + "\t\"sourcePacketHopCount\": %u," ZT_EOL_S + "\t\"errorCode\": %u," ZT_EOL_S + "\t\"vendor\": %d," ZT_EOL_S + "\t\"protocolVersion\": %u," ZT_EOL_S + "\t\"majorVersion\": %u," ZT_EOL_S + "\t\"minorVersion\": %u," ZT_EOL_S + "\t\"revision\": %u," ZT_EOL_S + "\t\"platform\": %d," ZT_EOL_S + "\t\"architecture\": %d," ZT_EOL_S + "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S + "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S "}", ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), (unsigned long long)report->timestamp, diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp index 0d549abc..15e0968f 100644 --- a/controller/SqliteNetworkController.hpp +++ b/controller/SqliteNetworkController.hpp @@ -14,15 +14,6 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ */ #ifndef ZT_SQLITENETWORKCONTROLLER_HPP @@ -38,9 +29,11 @@ #include "../node/NetworkController.hpp" #include "../node/Mutex.hpp" -#include "../osdep/Thread.hpp" +#include "../node/Utils.hpp" + +#include "../osdep/OSUtils.hpp" -#include "../ext/offbase/offbase.hpp" +#include "../ext/json/json.hpp" namespace ZeroTier { @@ -82,10 +75,6 @@ public: std::string &responseBody, std::string &responseContentType); - // threadMain() for backup thread -- do not call directly - void threadMain() - throw(); - private: unsigned int _doCPGet( const std::vector &path, @@ -97,11 +86,57 @@ private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); + inline nlohmann::json _readJson(const std::string &path) + { + std::string buf; + if (OSUtils::readFile(path.c_str(),buf)) { + try { + return nlohmann::json::parse(buf); + } catch ( ... ) {} + } + return nlohmann::json::object(); + } + + inline bool _writeJson(const std::string &path,const nlohmann::json &obj) + { + std::string buf(obj.dump(2)); + return OSUtils::writeFile(path.c_str(),buf); + } + + inline std::string _networkBP(const uint64_t nwid,bool create) + { + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nwid); + std::string p(_path + ZT_PATH_SEPARATOR_S + "network"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(tmp); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _networkJP(const uint64_t nwid,bool create) + { + return (_networkBP(nwid,create) + ZT_PATH_SEPARATOR + "config.json"); + } + inline std::string _memberBP(const uint64_t nwid,const Address &member,bool create) + { + std::string p(_networkBP(nwid,create)); + p.push_back(ZT_PATH_SEPARATOR); + p.append("member"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(member.toString()); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _memberJP(const uint64_t nwid,const Address &member,bool create) + { + return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); + } + + // These are const after construction Node *const _node; - std::string _instanceId; - offbase _db; - Thread _dbCommitThread; - volatile bool _dbCommitThreadRun; + std::string _path; // Circuit tests outstanding struct _CircuitTestEntry @@ -110,11 +145,11 @@ private: std::string jsonResults; }; std::map< uint64_t,_CircuitTestEntry > _circuitTests; + Mutex _circuitTests_m; // Last request time by address, for rate limitation std::map< std::pair,uint64_t > _lastRequestTime; - - Mutex _lock; + Mutex _lastRequestTime_m; }; } // namespace ZeroTier diff --git a/ext/json/LICENSE.MIT b/ext/json/LICENSE.MIT new file mode 100644 index 00000000..e2ac4891 --- /dev/null +++ b/ext/json/LICENSE.MIT @@ -0,0 +1,22 @@ +The library is licensed under the MIT License +: + +Copyright (c) 2013-2016 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ext/json/README.md b/ext/json/README.md new file mode 100644 index 00000000..c0bb61b1 --- /dev/null +++ b/ext/json/README.md @@ -0,0 +1,511 @@ +[![JSON for Modern C++](https://raw.githubusercontent.com/nlohmann/json/master/doc/json.gif)](https://github.com/nlohmann/json/releases) + +[![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) +[![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk?svg=true)](https://ci.appveyor.com/project/nlohmann/json) +[![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) +[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) +[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/p5o4znPnGHJpDVqN) +[![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) +[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) +[![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) +[![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) + +## Design goals + +There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: + +- **Intuitive syntax**. In languages such as Python, JSON feels like a first class data type. We used all the operator magic of modern C++ to achieve the same feeling in your code. Check out the [examples below](#examples) and you know, what I mean. + +- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/src/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. + +- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/blob/master/test/src/unit.cpp) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) that there are no memory leaks. + +Other aspects were not so important to us: + +- **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. + +- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). + +See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. + + +## Integration + +The single required source, file `json.hpp` is in the `src` directory or [released here](https://github.com/nlohmann/json/releases). All you need to do is add + +```cpp +#include "json.hpp" + +// for convenience +using json = nlohmann::json; +``` + +to the files you want to use JSON objects. That's it. Do not forget to set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). + +:beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. + + +## Examples + +Here are some examples to give you an idea how to use the class. + +Assume you want to create the JSON object + +```json +{ + "pi": 3.141, + "happy": true, + "name": "Niels", + "nothing": null, + "answer": { + "everything": 42 + }, + "list": [1, 0, 2], + "object": { + "currency": "USD", + "value": 42.99 + } +} +``` + +With the JSON class, you could write: + +```cpp +// create an empty structure (null) +json j; + +// add a number that is stored as double (note the implicit conversion of j to an object) +j["pi"] = 3.141; + +// add a Boolean that is stored as bool +j["happy"] = true; + +// add a string that is stored as std::string +j["name"] = "Niels"; + +// add another null object by passing nullptr +j["nothing"] = nullptr; + +// add an object inside the object +j["answer"]["everything"] = 42; + +// add an array that is stored as std::vector (using an initializer list) +j["list"] = { 1, 0, 2 }; + +// add another object (using an initializer list of pairs) +j["object"] = { {"currency", "USD"}, {"value", 42.99} }; + +// instead, you could also write (which looks very similar to the JSON above) +json j2 = { + {"pi", 3.141}, + {"happy", true}, + {"name", "Niels"}, + {"nothing", nullptr}, + {"answer", { + {"everything", 42} + }}, + {"list", {1, 0, 2}}, + {"object", { + {"currency", "USD"}, + {"value", 42.99} + }} +}; +``` + +Note that in all these cases, you never need to "tell" the compiler which JSON value you want to use. If you want to be explicit or express some edge cases, the functions `json::array` and `json::object` will help: + +```cpp +// a way to express the empty array [] +json empty_array_explicit = json::array(); + +// ways to express the empty object {} +json empty_object_implicit = json({}); +json empty_object_explicit = json::object(); + +// a way to express an _array_ of key/value pairs [["currency", "USD"], ["value", 42.99]] +json array_not_object = { json::array({"currency", "USD"}), json::array({"value", 42.99}) }; +``` + + +### Serialization / Deserialization + +You can create an object (deserialization) by appending `_json` to a string literal: + +```cpp +// create object from string literal +json j = "{ \"happy\": true, \"pi\": 3.141 }"_json; + +// or even nicer with a raw string literal +auto j2 = R"( + { + "happy": true, + "pi": 3.141 + } +)"_json; + +// or explicitly +auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); +``` + +You can also get a string representation (serialize): + +```cpp +// explicit conversion to string +std::string s = j.dump(); // {\"happy\":true,\"pi\":3.141} + +// serialization with pretty printing +// pass in the amount of spaces to indent +std::cout << j.dump(4) << std::endl; +// { +// "happy": true, +// "pi": 3.141 +// } +``` + +You can also use streams to serialize and deserialize: + +```cpp +// deserialize from standard input +json j; +std::cin >> j; + +// serialize to standard output +std::cout << j; + +// the setw manipulator was overloaded to set the indentation for pretty printing +std::cout << std::setw(4) << j << std::endl; +``` + +These operators work for any subclasses of `std::istream` or `std::ostream`. + +Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. + + +### STL-like access + +We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) requirement. + +```cpp +// create an array using push_back +json j; +j.push_back("foo"); +j.push_back(1); +j.push_back(true); + +// iterate the array +for (json::iterator it = j.begin(); it != j.end(); ++it) { + std::cout << *it << '\n'; +} + +// range-based for +for (auto& element : j) { + std::cout << element << '\n'; +} + +// getter/setter +const std::string tmp = j[0]; +j[1] = 42; +bool foo = j.at(2); + +// other stuff +j.size(); // 3 entries +j.empty(); // false +j.type(); // json::value_t::array +j.clear(); // the array is empty again + +// convenience type checkers +j.is_null(); +j.is_boolean(); +j.is_number(); +j.is_object(); +j.is_array(); +j.is_string(); + +// comparison +j == "[\"foo\", 1, true]"_json; // true + +// create an object +json o; +o["foo"] = 23; +o["bar"] = false; +o["baz"] = 3.141; + +// special iterator member functions for objects +for (json::iterator it = o.begin(); it != o.end(); ++it) { + std::cout << it.key() << " : " << it.value() << "\n"; +} + +// find an entry +if (o.find("foo") != o.end()) { + // there is an entry with key "foo" +} + +// or simpler using count() +int foo_present = o.count("foo"); // 1 +int fob_present = o.count("fob"); // 0 + +// delete an entry +o.erase("foo"); +``` + + +### Conversion from STL containers + +Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON types (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends how the elements are ordered in the respective STL container. + +```cpp +std::vector c_vector {1, 2, 3, 4}; +json j_vec(c_vector); +// [1, 2, 3, 4] + +std::deque c_deque {1.2, 2.3, 3.4, 5.6}; +json j_deque(c_deque); +// [1.2, 2.3, 3.4, 5.6] + +std::list c_list {true, true, false, true}; +json j_list(c_list); +// [true, true, false, true] + +std::forward_list c_flist {12345678909876, 23456789098765, 34567890987654, 45678909876543}; +json j_flist(c_flist); +// [12345678909876, 23456789098765, 34567890987654, 45678909876543] + +std::array c_array {{1, 2, 3, 4}}; +json j_array(c_array); +// [1, 2, 3, 4] + +std::set c_set {"one", "two", "three", "four", "one"}; +json j_set(c_set); // only one entry for "one" is used +// ["four", "one", "three", "two"] + +std::unordered_set c_uset {"one", "two", "three", "four", "one"}; +json j_uset(c_uset); // only one entry for "one" is used +// maybe ["two", "three", "four", "one"] + +std::multiset c_mset {"one", "two", "one", "four"}; +json j_mset(c_mset); // only one entry for "one" is used +// maybe ["one", "two", "four"] + +std::unordered_multiset c_umset {"one", "two", "one", "four"}; +json j_umset(c_umset); // both entries for "one" are used +// maybe ["one", "two", "one", "four"] +``` + +Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON types (see examples above) can be used to to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. + +```cpp +std::map c_map { {"one", 1}, {"two", 2}, {"three", 3} }; +json j_map(c_map); +// {"one": 1, "three": 3, "two": 2 } + +std::unordered_map c_umap { {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; +json j_umap(c_umap); +// {"one": 1.2, "two": 2.3, "three": 3.4} + +std::multimap c_mmap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; +json j_mmap(c_mmap); // only one entry for key "three" is used +// maybe {"one": true, "two": true, "three": true} + +std::unordered_multimap c_ummap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; +json j_ummap(c_ummap); // only one entry for key "three" is used +// maybe {"one": true, "two": true, "three": true} +``` + +### JSON Pointer and JSON Patch + +The library supports **JSON Pointer** ([RFC 6901](https://tools.ietf.org/html/rfc6901)) as alternative means to address structured values. On top of this, **JSON Patch** ([RFC 6902](https://tools.ietf.org/html/rfc6902)) allows to describe differences between two JSON values - effectively allowing patch and diff operations known from Unix. + +```cpp +// a JSON value +json j_original = R"({ + "baz": ["one", "two", "three"], + "foo": "bar" +})"_json; + +// access members with a JSON pointer (RFC 6901) +j_original["/baz/1"_json_pointer]; +// "two" + +// a JSON patch (RFC 6902) +json j_patch = R"([ + { "op": "replace", "path": "/baz", "value": "boo" }, + { "op": "add", "path": "/hello", "value": ["world"] }, + { "op": "remove", "path": "/foo"} +])"_json; + +// apply the patch +json j_result = j_original.patch(j_patch); +// { +// "baz": "boo", +// "hello": ["world"] +// } + +// calculate a JSON patch from two JSON values +json::diff(j_result, j_original); +// [ +// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, +// { "op": "remove","path": "/hello" }, +// { "op": "add", "path": "/foo", "value": "bar" } +// ] +``` + + +### Implicit conversions + +The type of the JSON object is determined automatically by the expression to store. Likewise, the stored value is implicitly converted. + +```cpp +// strings +std::string s1 = "Hello, world!"; +json js = s1; +std::string s2 = js; + +// Booleans +bool b1 = true; +json jb = b1; +bool b2 = jb; + +// numbers +int i = 42; +json jn = i; +double f = jn; + +// etc. +``` + +You can also explicitly ask for the value: + +```cpp +std::string vs = js.get(); +bool vb = jb.get(); +int vi = jn.get(); + +// etc. +``` + + +## Supported compilers + +Though it's 2016 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: + +- GCC 4.9 - 6.0 (and possibly later) +- Clang 3.4 - 3.9 (and possibly later) +- Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) + +I would be happy to learn about other compilers/versions. + +Please note: + +- GCC 4.8 does not work because of two bugs ([55817](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55817) and [57824](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57824)) in the C++11 support. Note there is a [pull request](https://github.com/nlohmann/json/pull/212) to fix some of the issues. +- Android defaults to using very old compilers and C++ libraries. To fix this, add the following to your `Application.mk`. This will switch to the LLVM C++ library, the Clang compiler, and enable C++11 and other features disabled by default. + + ``` + APP_STL := c++_shared + NDK_TOOLCHAIN_VERSION := clang3.6 + APP_CPPFLAGS += -frtti -fexceptions + ``` + + The code compiles successfully with [Android NDK](https://developer.android.com/ndk/index.html?hl=ml), Revision 9 - 11 (and possibly later) and [CrystaX's Android NDK](https://www.crystax.net/en/android/ndk) version 10. + +- For GCC running on MinGW or Android SDK, the error `'to_string' is not a member of 'std'` (or similarly, for `strtod`) may occur. Note this is not an issue with the code, but rather with the compiler itself. On Android, see above to build with a newer environment. For MinGW, please refer to [this site](http://tehsausage.com/mingw-to-string) and [this discussion](https://github.com/nlohmann/json/issues/136) for information on how to fix this bug. For Android NDK using `APP_STL := gnustl_static`, please refer to [this discussion](https://github.com/nlohmann/json/issues/219). + +The following compilers are currently used in continuous integration at [Travis](https://travis-ci.org/nlohmann/json) and [AppVeyor](https://ci.appveyor.com/project/nlohmann/json): + +| Compiler | Operating System | Version String | +|-----------------|------------------------------|----------------| +| GCC 4.9.3 | Ubuntu 14.04.4 LTS | g++-4.9 (Ubuntu 4.9.3-8ubuntu2~14.04) 4.9.3 | +| GCC 5.3.0 | Ubuntu 14.04.4 LTS | g++-5 (Ubuntu 5.3.0-3ubuntu1~14.04) 5.3.0 20151204 | +| GCC 6.1.1 | Ubuntu 14.04.4 LTS | g++-6 (Ubuntu 6.1.1-3ubuntu11~14.04.1) 6.1.1 20160511 | +| Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | +| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | +| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | +| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | +| Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | +| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | +| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | +| Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | +| Clang Xcode 8.0 | Darwin Kernel Version 15.5.0 (OSX 10.11.5) | Apple LLVM version 8.0.0 (clang-800.0.24.1) | +| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | + + +## License + + + +The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): + +Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +## Thanks + +I deeply appreciate the help of the following people. + +- [Teemperor](https://github.com/Teemperor) implemented CMake support and lcov integration, realized escape and Unicode handling in the string parser, and fixed the JSON serialization. +- [elliotgoodrich](https://github.com/elliotgoodrich) fixed an issue with double deletion in the iterator classes. +- [kirkshoop](https://github.com/kirkshoop) made the iterators of the class composable to other libraries. +- [wancw](https://github.com/wanwc) fixed a bug that hindered the class to compile with Clang. +- Tomas Åblad found a bug in the iterator implementation. +- [Joshua C. Randall](https://github.com/jrandall) fixed a bug in the floating-point serialization. +- [Aaron Burghardt](https://github.com/aburgh) implemented code to parse streams incrementally. Furthermore, he greatly improved the parser class by allowing the definition of a filter function to discard undesired elements while parsing. +- [Daniel Kopeček](https://github.com/dkopecek) fixed a bug in the compilation with GCC 5.0. +- [Florian Weber](https://github.com/Florianjw) fixed a bug in and improved the performance of the comparison operators. +- [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. +- [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. +- [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. +- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. +- [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. +- [dariomt](https://github.com/dariomt) fixed some typos in the examples. +- [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. +- [Colin Hirsch](https://github.com/ColinH) took care of a small namespace issue. +- [Huu Nguyen](https://github.com/whoshuu) correct a variable name in the documentation. +- [Silverweed](https://github.com/silverweed) overloaded `parse()` to accept an rvalue reference. +- [dariomt](https://github.com/dariomt) fixed a subtlety in MSVC type support and implemented the `get_ref()` function to get a reference to stored values. +- [ZahlGraf](https://github.com/ZahlGraf) added a workaround that allows compilation using Android NDK. +- [whackashoe](https://github.com/whackashoe) replaced a function that was marked as unsafe by Visual Studio. +- [406345](https://github.com/406345) fixed two small warnings. +- [Glen Fernandes](https://github.com/glenfe) noted a potential portability problem in the `has_mapped_type` function. +- [Corbin Hughes](https://github.com/nibroc) fixed some typos in the contribution guidelines. +- [twelsby](https://github.com/twelsby) fixed the array subscript operator, an issue that failed the MSVC build, and floating-point parsing/dumping. He further added support for unsigned integer numbers and implemented better roundtrip support for parsed numbers. +- [Volker Diels-Grabsch](https://github.com/vog) fixed a link in the README file. +- [msm-](https://github.com/msm-) added support for american fuzzy lop. +- [Annihil](https://github.com/Annihil) fixed an example in the README file. +- [Themercee](https://github.com/Themercee) noted a wrong URL in the README file. +- [Lv Zheng](https://github.com/lv-zheng) fixed a namespace issue with `int64_t` and `uint64_t`. +- [abc100m](https://github.com/abc100m) analyzed the issues with GCC 4.8 and proposed a [partial solution](https://github.com/nlohmann/json/pull/212). +- [zewt](https://github.com/zewt) added useful notes to the README file about Android. +- [Róbert Márki](https://github.com/robertmrk) added a fix to use move iterators and improved the integration via CMake. +- [Chris Kitching](https://github.com/ChrisKitching) cleaned up the CMake files. +- [Tom Needham](https://github.com/06needhamt) fixed a subtle bug with MSVC 2015 which was also proposed by [Michael K.](https://github.com/Epidal). +- [Mário Feroldi](https://github.com/thelostt) fixed a small typo. +- [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. +- [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. + +Thanks a lot for helping out! + + +## Notes + +- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). +- As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions. + + +## Execute unit tests + +To compile and run the tests, you need to execute + +```sh +$ make +$ ./json_unit "*" + +=============================================================================== +All tests passed (8905012 assertions in 32 test cases) +``` + +For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). diff --git a/ext/json/json.hpp b/ext/json/json.hpp new file mode 100644 index 00000000..878fb899 --- /dev/null +++ b/ext/json/json.hpp @@ -0,0 +1,10435 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 2.0.2 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +Copyright (c) 2013-2016 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// exclude unsupported compilers +#if defined(__clang__) + #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) + #if CLANG_VERSION < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#elif defined(__GNUC__) + #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) + #if GCC_VERSION < 40900 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + + +/*! +@brief unnamed namespace with internal helper functions +@since version 1.0.0 +*/ +namespace +{ +/*! +@brief Helper to determine whether there's a key_type for T. + +Thus helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. + +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0 +*/ +template +struct has_mapped_type +{ + private: + template static char test(typename C::mapped_type*); + template static char (&test(...))[2]; + public: + static constexpr bool value = sizeof(test(0)) == 1; +}; + +/*! +@brief helper class to create locales with decimal point + +This struct is used a default locale during the JSON serialization. JSON +requires the decimal point to be `.`, so this function overloads the +`do_decimal_point()` function to return `.`. This function is called by +float-to-string conversions to retrieve the decimal separator between integer +and fractional parts. + +@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 +@since version 2.0.0 +*/ +struct DecimalSeparator : std::numpunct +{ + char do_decimal_point() const + { + return '.'; + } +}; + +} + +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null value. + - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + JSON values have + [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the class + has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](http://en.cppreference.com/w/cpp/concept/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. + +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 + +@nosubgrouping +*/ +template < + template class ObjectType = std::map, + template class ArrayType = std::vector, + class StringType = std::string, + class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator + > +class basic_json +{ + private: + /// workaround type for MSVC + using basic_json_t = basic_json; + + public: + // forward declarations + template class json_reverse_iterator; + class json_pointer; + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + class iterator; + /// a const iterator for a basic_json container + class const_iterator; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() + { + return allocator_type(); + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /*! + @brief a type for an object + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. + + To store objects in C++, a type is defined by the template parameters + described below. + + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) + + #### Default type + + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: + + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less, // key_compare + std::allocator> // allocator_type + > + @endcode + + #### Behavior + + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: + + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, later stored name/value + pairs overwrite previously stored name/value pairs, leaving the used + names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will + be treated as equal and both stored as `{"key": 1}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType, + AllocatorType>>; + + /*! + @brief a type for an array + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. + + To store objects in C++, a type is defined by the template parameters + explained below. + + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + + #### Default type + + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: + + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator // allocator_type + > + @endcode + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the array's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. + + #### Storage + + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value + + @since version 1.0.0 + */ + using array_t = ArrayType>; + + /*! + @brief a type for a string + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. + + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. + + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. + + #### Default type + + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: + + @code {.cpp} + std::string + @endcode + + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage + + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. + + @since version 1.0.0 + */ + using string_t = StringType; + + /*! + @brief a type for a boolean + + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. + + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. + + #### Default type + + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. + + @since version 1.0.0 + */ + using boolean_t = BooleanType; + + /*! + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. + + #### Default type + + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: + + @code {.cpp} + int64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_integer_t = NumberIntegerType; + + /*! + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. + + #### Default type + + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: + + @code {.cpp} + uint64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) + + @since version 2.0.0 + */ + using number_unsigned_t = NumberUnsignedType; + + /*! + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. + + #### Default type + + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: + + @code {.cpp} + double + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. + + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. + + #### Storage + + Floating-point number values are stored directly inside a @ref basic_json + type. + + @sa @ref number_integer_t -- type for number values (integer) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_float_t = NumberFloatType; + + /// @} + + + /////////////////////////// + // JSON type enumeration // + /////////////////////////// + + /*! + @brief the JSON type enumeration + + This enumeration collects the different JSON types. It is internally used + to distinguish the stored values, and the functions @ref is_null(), @ref + is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref + is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and + @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and + @ref is_structured() rely on it. + + @note There are three enumeration entries (number_integer, + number_unsigned, and number_float), because the library distinguishes + these three types for numbers: @ref number_unsigned_t is used for unsigned + integers, @ref number_integer_t is used for signed integers, and @ref + number_float_t is used for floating-point numbers or to approximate + integers which do not fit in the limits of their respective type. + + @sa @ref basic_json(const value_t value_type) -- create a JSON value with + the default value for a given type + + @since version 1.0.0 + */ + enum class value_t : uint8_t + { + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function + }; + + + private: + + /// helper for exception-safe object creation + template + static T* create(Args&& ... args) + { + AllocatorType alloc; + auto deleter = [&](T * object) + { + alloc.deallocate(object, 1); + }; + std::unique_ptr object(alloc.allocate(1), deleter); + alloc.construct(object.get(), std::forward(args)...); + assert(object.get() != nullptr); + return object.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } + + case value_t::array: + { + array = create(); + break; + } + + case value_t::string: + { + string = create(""); + break; + } + + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } + + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } + + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } + + default: + { + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) + { + string = create(value); + } + + /// constructor for objects + json_value(const object_t& value) + { + object = create(value); + } + + /// constructor for arrays + json_value(const array_t& value) + { + array = create(value); + } + }; + + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + */ + void assert_invariant() const + { + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /*! + @brief JSON callback events + + This enumeration lists the parser events that can trigger calling a + callback function of type @ref parser_callback_t during parsing. + + @image html callback_events.png "Example when certain parse events are triggered" + + @since version 1.0.0 + */ + enum class parse_event_t : uint8_t + { + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; + + /*! + @brief per-element parser callback type + + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse(std::istream&, const + parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), + it is called on certain events (passed as @ref parse_event_t via parameter + @a event) with a set recursion depth @a depth and context JSON value + @a parsed. The return value of the callback function is a boolean + indicating whether the element that emitted the callback shall be kept or + not. + + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. + + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: + + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. + + @param[in] depth the depth of the recursion during parsing + + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called + + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events + + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. + + @sa @ref parse(std::istream&, parser_callback_t) or + @ref parse(const string_t&, parser_callback_t) for examples + + @since version 1.0.0 + */ + using parser_callback_t = std::function; + + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /*! + @brief create an empty value with a given type + + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @param[in] value_type the type of the value to create + + @complexity Constant. + + @throw std::bad_alloc if allocation for object, array, or string value + fails + + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} + + @sa @ref basic_json(std::nullptr_t) -- create a `null` value + @sa @ref basic_json(boolean_t value) -- create a boolean value + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const object_t&) -- create a object value + @sa @ref basic_json(const array_t&) -- create a array value + @sa @ref basic_json(const number_float_t) -- create a number + (floating-point) value + @sa @ref basic_json(const number_integer_t) -- create a number (integer) + value + @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) + value + + @since version 1.0.0 + */ + basic_json(const value_t value_type) + : m_type(value_type), m_value(value_type) + { + assert_invariant(); + } + + /*! + @brief create a null object (implicitly) + + Create a `null` JSON value. This is the implicit version of the `null` + value constructor as it takes no parameters. + + @note The class invariant is satisfied, because it poses no requirements + for null values. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - As postcondition, it holds: `basic_json().empty() == true`. + + @liveexample{The following code shows the constructor for a `null` JSON + value.,basic_json} + + @sa @ref basic_json(std::nullptr_t) -- create a `null` value + + @since version 1.0.0 + */ + basic_json() = default; + + /*! + @brief create a null object (explicitly) + + Create a `null` JSON value. This is the explicitly version of the `null` + value constructor as it takes a null pointer as parameter. It allows to + create `null` values by explicitly assigning a `nullptr` to a JSON value. + The passed null pointer itself is not read -- it is only used to choose + the right constructor. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @liveexample{The following code shows the constructor with null pointer + parameter.,basic_json__nullptr_t} + + @sa @ref basic_json() -- default constructor (implicitly creating a `null` + value) + + @since version 1.0.0 + */ + basic_json(std::nullptr_t) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /*! + @brief create an object (explicit) + + Create an object JSON value with a given content. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with an @ref + object_t parameter.,basic_json__object_t} + + @sa @ref basic_json(const CompatibleObjectType&) -- create an object value + from a compatible STL container + + @since version 1.0.0 + */ + basic_json(const object_t& val) + : m_type(value_t::object), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an object (implicit) + + Create an object JSON value with a given content. This constructor allows + any type @a CompatibleObjectType that can be used to construct values of + type @ref object_t. + + @tparam CompatibleObjectType An object type whose `key_type` and + `value_type` is compatible to @ref object_t. Examples include `std::map`, + `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with + a `key_type` of `std::string`, and a `value_type` from which a @ref + basic_json value can be constructed. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with several + compatible object type parameters.,basic_json__CompatibleObjectType} + + @sa @ref basic_json(const object_t&) -- create an object value + + @since version 1.0.0 + */ + template ::value and + std::is_constructible::value, int>::type + = 0> + basic_json(const CompatibleObjectType& val) + : m_type(value_t::object) + { + using std::begin; + using std::end; + m_value.object = create(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create an array (explicit) + + Create an array JSON value with a given content. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with an @ref array_t + parameter.,basic_json__array_t} + + @sa @ref basic_json(const CompatibleArrayType&) -- create an array value + from a compatible STL containers + + @since version 1.0.0 + */ + basic_json(const array_t& val) + : m_type(value_t::array), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an array (implicit) + + Create an array JSON value with a given content. This constructor allows + any type @a CompatibleArrayType that can be used to construct values of + type @ref array_t. + + @tparam CompatibleArrayType An object type whose `value_type` is + compatible to @ref array_t. Examples include `std::vector`, `std::deque`, + `std::list`, `std::forward_list`, `std::array`, `std::set`, + `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a + `value_type` from which a @ref basic_json value can be constructed. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with several + compatible array type parameters.,basic_json__CompatibleArrayType} + + @sa @ref basic_json(const array_t&) -- create an array value + + @since version 1.0.0 + */ + template ::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + std::is_constructible::value, int>::type + = 0> + basic_json(const CompatibleArrayType& val) + : m_type(value_t::array) + { + using std::begin; + using std::end; + m_value.array = create(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create an string JSON value with a given content. + + @param[in] val a value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with an @ref + string_t parameter.,basic_json__string_t} + + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const string_t& val) + : m_type(value_t::string), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create a string JSON value with a given content. + + @param[in] val a literal value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with string literal + parameter.,basic_json__string_t_value_type} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const typename string_t::value_type* val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a string (implicit) + + Create a string JSON value with a given content. + + @param[in] val a value for the string + + @tparam CompatibleStringType an string type which is compatible to @ref + string_t, for instance `std::string`. + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the construction of a string value + from a compatible type.,basic_json__CompatibleStringType} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + + @since version 1.0.0 + */ + template ::value, int>::type + = 0> + basic_json(const CompatibleStringType& val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a boolean (explicit) + + Creates a JSON boolean type from a given value. + + @param[in] val a boolean value to store + + @complexity Constant. + + @liveexample{The example below demonstrates boolean + values.,basic_json__boolean_t} + + @since version 1.0.0 + */ + basic_json(boolean_t val) noexcept + : m_type(value_t::boolean), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number (explicit) + + Create an integer number JSON value with a given content. + + @tparam T A helper type to remove this function via SFINAE in case @ref + number_integer_t is the same as `int`. In this case, this constructor + would have the same signature as @ref basic_json(const int value). Note + the helper type @a T is not visible in this constructor's interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value.,basic_json__number_integer_t} + + @sa @ref basic_json(const int) -- create a number value (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + template::value) + and std::is_same::value + , int>::type + = 0> + basic_json(const number_integer_t val) noexcept + : m_type(value_t::number_integer), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number from an enum type (explicit) + + Create an integer number JSON value with a given content. + + @param[in] val an integer to create a JSON number from + + @note This constructor allows to pass enums directly to a constructor. As + C++ has no way of specifying the type of an anonymous enum explicitly, we + can only rely on the fact that such values implicitly convert to int. As + int may already be the same type of number_integer_t, we may need to + switch off the constructor @ref basic_json(const number_integer_t). + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value from an anonymous enum.,basic_json__const_int} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const int val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create an integer number (implicit) + + Create an integer number JSON value with a given content. This constructor + allows any type @a CompatibleNumberIntegerType that can be used to + construct values of type @ref number_integer_t. + + @tparam CompatibleNumberIntegerType An integer type which is compatible to + @ref number_integer_t. Examples include the types `int`, `int32_t`, + `long`, and `short`. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of several integer + number values from compatible + types.,basic_json__CompatibleIntegerNumberType} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const int) -- create a number value (integer) + + @since version 1.0.0 + */ + template::value and + std::numeric_limits::is_integer and + std::numeric_limits::is_signed, + CompatibleNumberIntegerType>::type + = 0> + basic_json(const CompatibleNumberIntegerType val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create an unsigned integer number (explicit) + + Create an unsigned integer number JSON value with a given content. + + @tparam T helper type to compare number_unsigned_t and unsigned int (not + visible in) the interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number + value (unsigned integer) from a compatible number type + + @since version 2.0.0 + */ + template::value) + and std::is_same::value + , int>::type + = 0> + basic_json(const number_unsigned_t val) noexcept + : m_type(value_t::number_unsigned), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an unsigned number (implicit) + + Create an unsigned number JSON value with a given content. This + constructor allows any type @a CompatibleNumberUnsignedType that can be + used to construct values of type @ref number_unsigned_t. + + @tparam CompatibleNumberUnsignedType An integer type which is compatible + to @ref number_unsigned_t. Examples may include the types `unsigned int`, + `uint32_t`, or `unsigned short`. + + @param[in] val an unsigned integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const number_unsigned_t) -- create a number value + (unsigned) + + @since version 2.0.0 + */ + template ::value and + std::numeric_limits::is_integer and + not std::numeric_limits::is_signed, + CompatibleNumberUnsignedType>::type + = 0> + basic_json(const CompatibleNumberUnsignedType val) noexcept + : m_type(value_t::number_unsigned), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create a floating-point number (explicit) + + Create a floating-point number JSON value with a given content. + + @param[in] val a floating-point value to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is created + instead. + + @complexity Constant. + + @liveexample{The following example creates several floating-point + values.,basic_json__number_float_t} + + @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number + value (floating-point) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const number_float_t val) noexcept + : m_type(value_t::number_float), m_value(val) + { + // replace infinity and NAN by null + if (not std::isfinite(val)) + { + m_type = value_t::null; + m_value = json_value(); + } + + assert_invariant(); + } + + /*! + @brief create an floating-point number (implicit) + + Create an floating-point number JSON value with a given content. This + constructor allows any type @a CompatibleNumberFloatType that can be used + to construct values of type @ref number_float_t. + + @tparam CompatibleNumberFloatType A floating-point type which is + compatible to @ref number_float_t. Examples may include the types `float` + or `double`. + + @param[in] val a floating-point to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is + created instead. + + @complexity Constant. + + @liveexample{The example below shows the construction of several + floating-point number values from compatible + types.,basic_json__CompatibleNumberFloatType} + + @sa @ref basic_json(const number_float_t) -- create a number value + (floating-point) + + @since version 1.0.0 + */ + template::value and + std::is_floating_point::value>::type + > + basic_json(const CompatibleNumberFloatType val) noexcept + : basic_json(number_float_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a container (array or object) from an initializer list + + Creates a JSON value of type array or object from the passed initializer + list @a init. In case @a type_deduction is `true` (default), the type of + the JSON value to be created is deducted from the initializer list @a init + according to the following rules: + + 1. If the list is empty, an empty JSON object value `{}` is created. + 2. If the list consists of pairs whose first element is a string, a JSON + object value is created where the first elements of the pairs are + treated as keys and the second elements are as values. + 3. In all other cases, an array is created. + + The rules aim to create the best fit between a C++ initializer list and + JSON values. The rationale is as follows: + + 1. The empty initializer list is written as `{}` which is exactly an empty + JSON object. + 2. C++ has now way of describing mapped types other than to list a list of + pairs. As JSON requires that keys must be of type string, rule 2 is the + weakest constraint one can pose on initializer lists to interpret them + as an object. + 3. In all other cases, the initializer list could not be interpreted as + JSON object type, so interpreting it as JSON array type is safe. + + With the rules described above, the following JSON values cannot be + expressed by an initializer list: + + - the empty array (`[]`): use @ref array(std::initializer_list) + with an empty initializer list in this case + - arrays whose elements satisfy rule 2: use @ref + array(std::initializer_list) with the same initializer list + in this case + + @note When used without parentheses around an empty initializer list, @ref + basic_json() is called instead of this function, yielding the JSON null + value. + + @param[in] init initializer list with JSON values + + @param[in] type_deduction internal parameter; when set to `true`, the type + of the JSON value is deducted from the initializer list @a init; when set + to `false`, the type provided via @a manual_type is forced. This mode is + used by the functions @ref array(std::initializer_list) and + @ref object(std::initializer_list). + + @param[in] manual_type internal parameter; when @a type_deduction is set + to `false`, the created JSON value will use the provided type (only @ref + value_t::array and @ref value_t::object are valid); when @a type_deduction + is set to `true`, this parameter has no effect + + @throw std::domain_error if @a type_deduction is `false`, @a manual_type + is `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string; example: `"cannot create object from + initializer list"` + + @complexity Linear in the size of the initializer list @a init. + + @liveexample{The example below shows how JSON values are created from + initializer lists.,basic_json__list_init_t} + + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + basic_json(std::initializer_list init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const basic_json & element) + { + return element.is_array() and element.size() == 2 and element[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (not type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (manual_type == value_t::object and not is_an_object) + { + throw std::domain_error("cannot create object from initializer list"); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + std::for_each(init.begin(), init.end(), [this](const basic_json & element) + { + m_value.object->emplace(*(element[0].m_value.string), element[1]); + }); + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create(init); + } + + assert_invariant(); + } + + /*! + @brief explicitly create an array from an initializer list + + Creates a JSON array value from a given initializer list. That is, given a + list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the + initializer list is empty, the empty array `[]` is created. + + @note This function is only needed to express two edge cases that cannot + be realized with the initializer list constructor (@ref + basic_json(std::initializer_list, bool, value_t)). These cases + are: + 1. creating an array whose elements are all pairs whose first element is a + string -- in this case, the initializer list constructor would create an + object, taking the first elements as keys + 2. creating an empty array -- passing the empty initializer list to the + initializer list constructor yields an empty object + + @param[in] init initializer list with JSON values to create an array from + (optional) + + @return JSON array value + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `array` + function.,array} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + static basic_json array(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::array); + } + + /*! + @brief explicitly create an object from an initializer list + + Creates a JSON object value from a given initializer list. The initializer + lists elements must be pairs, and their first elements must be strings. If + the initializer list is empty, the empty object `{}` is created. + + @note This function is only added for symmetry reasons. In contrast to the + related function @ref array(std::initializer_list), there are + no cases which can only be expressed by this function. That is, any + initializer list @a init can also be passed to the initializer list + constructor @ref basic_json(std::initializer_list, bool, + value_t). + + @param[in] init initializer list to create an object from (optional) + + @return JSON object value + + @throw std::domain_error if @a init is not a pair whose first elements are + strings; thrown by + @ref basic_json(std::initializer_list, bool, value_t) + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `object` + function.,object} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + + @since version 1.0.0 + */ + static basic_json object(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::object); + } + + /*! + @brief construct an array with count copies of given value + + Constructs a JSON array value by creating @a cnt copies of a passed value. + In case @a cnt is `0`, an empty array is created. As postcondition, + `std::distance(begin(),end()) == cnt` holds. + + @param[in] cnt the number of JSON copies of @a val to create + @param[in] val the JSON value to copy + + @complexity Linear in @a cnt. + + @liveexample{The following code shows examples for the @ref + basic_json(size_type\, const basic_json&) + constructor.,basic_json__size_type_basic_json} + + @since version 1.0.0 + */ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + assert_invariant(); + } + + /*! + @brief construct a JSON container given an iterator range + + Constructs the JSON value with the contents of the range `[first, last)`. + The semantics depends on the different types a JSON value can have: + - In case of primitive types (number, boolean, or string), @a first must + be `begin()` and @a last must be `end()`. In this case, the value is + copied. Otherwise, std::out_of_range is thrown. + - In case of structured types (array, object), the constructor behaves as + similar versions for `std::vector`. + - In case of a null type, std::domain_error is thrown. + + @tparam InputIT an input iterator type (@ref iterator or @ref + const_iterator) + + @param[in] first begin of the range to copy from (included) + @param[in] last end of the range to copy from (excluded) + + @pre Iterators @a first and @a last must be initialized. + + @throw std::domain_error if iterators are not compatible; that is, do not + belong to the same JSON value; example: `"iterators are not compatible"` + @throw std::out_of_range if iterators are for a primitive type (number, + boolean, or string) where an out of range error can be detected easily; + example: `"iterators out of range"` + @throw std::bad_alloc if allocation for object, array, or string fails + @throw std::domain_error if called with a null value; example: `"cannot + use construct with iterators from null"` + + @complexity Linear in distance between @a first and @a last. + + @liveexample{The example below shows several ways to create JSON values by + specifying a subrange with iterators.,basic_json__InputIt_InputIt} + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + basic_json(InputIT first, InputIT last) + { + assert(first.m_object != nullptr); + assert(last.m_object != nullptr); + + // make sure iterator fits the current value + if (first.m_object != last.m_object) + { + throw std::domain_error("iterators are not compatible"); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + throw std::out_of_range("iterators out of range"); + } + break; + } + + default: + { + break; + } + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); + } + } + + assert_invariant(); + } + + /*! + @brief construct a JSON value given an input stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates constructing a JSON value from + a `std::stringstream` with and without callback + function.,basic_json__istream} + + @since version 2.0.0 + */ + explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) + { + *this = parser(i, cb).parse(); + assert_invariant(); + } + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + /*! + @brief copy constructor + + Creates a copy of a given JSON value. + + @param[in] other the JSON value to copy + + @complexity Linear in the size of @a other. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - As postcondition, it holds: `other == basic_json(other)`. + + @throw std::bad_alloc if allocation for object, array, or string fails. + + @liveexample{The following code shows an example for the copy + constructor.,basic_json__basic_json} + + @since version 1.0.0 + */ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + default: + { + break; + } + } + + assert_invariant(); + } + + /*! + @brief move constructor + + Move constructor. Constructs a JSON value with the contents of the given + value @a other using move semantics. It "steals" the resources from @a + other and leaves it as JSON null value. + + @param[in,out] other value to move to this object + + @post @a other is a JSON null value + + @complexity Constant. + + @liveexample{The code below shows the move constructor explicitly called + via std::move.,basic_json__moveconstructor} + + @since version 1.0.0 + */ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + assert_invariant(); + } + + /*! + @brief copy assignment + + Copy assignment operator. Copies a JSON value via the "copy and swap" + strategy: It is expressed in terms of the copy constructor, destructor, + and the swap() member function. + + @param[in] other value to copy from + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + + @liveexample{The code below shows and example for the copy assignment. It + creates a copy of value `a` which is then swapped with `b`. Finally\, the + copy of `a` (which is the null value after the swap) is + destroyed.,basic_json__copyassignment} + + @since version 1.0.0 + */ + reference& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + assert_invariant(); + return *this; + } + + /*! + @brief destructor + + Destroys the JSON value and frees all allocated memory. + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - All stored elements are destroyed and all memory is freed. + + @since version 1.0.0 + */ + ~basic_json() + { + assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + AllocatorType alloc; + alloc.destroy(m_value.object); + alloc.deallocate(m_value.object, 1); + break; + } + + case value_t::array: + { + AllocatorType alloc; + alloc.destroy(m_value.array); + alloc.deallocate(m_value.array, 1); + break; + } + + case value_t::string: + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + break; + } + + default: + { + // all other types need no specific destructor + break; + } + } + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /*! + @brief serialization + + Serialization function for JSON values. The function tries to mimic + Python's `json.dumps()` function, and currently supports its @a indent + parameter. + + @param[in] indent If indent is nonnegative, then array elements and object + members will be pretty-printed with that indent level. An indent level of + `0` will only insert newlines. `-1` (the default) selects the most compact + representation. + + @return string containing the serialization of the JSON value + + @complexity Linear. + + @liveexample{The following example shows the effect of different @a indent + parameters to the result of the serialization.,dump} + + @see https://docs.python.org/2/library/json.html#json.dump + + @since version 1.0.0 + */ + string_t dump(const int indent = -1) const + { + std::stringstream ss; + // fix locale problems + ss.imbue(std::locale(std::locale(), new DecimalSeparator)); + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits::digits10 + ss.precision(std::numeric_limits::digits10); + + if (indent >= 0) + { + dump(ss, true, static_cast(indent)); + } + else + { + dump(ss, false, 0); + } + + return ss.str(); + } + + /*! + @brief return the type of the JSON value (explicit) + + Return the type of the JSON value as a value from the @ref value_t + enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `type()` for all JSON + types.,type} + + @since version 1.0.0 + */ + constexpr value_t type() const noexcept + { + return m_type; + } + + /*! + @brief return whether type is primitive + + This function returns true iff the JSON type is primitive (string, number, + boolean, or null). + + @return `true` if type is primitive (string, number, boolean, or null), + `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_primitive()` for all JSON + types.,is_primitive} + + @sa @ref is_structured() -- returns whether JSON value is structured + @sa @ref is_null() -- returns whether JSON value is `null` + @sa @ref is_string() -- returns whether JSON value is a string + @sa @ref is_boolean() -- returns whether JSON value is a boolean + @sa @ref is_number() -- returns whether JSON value is a number + + @since version 1.0.0 + */ + constexpr bool is_primitive() const noexcept + { + return is_null() or is_string() or is_boolean() or is_number(); + } + + /*! + @brief return whether type is structured + + This function returns true iff the JSON type is structured (array or + object). + + @return `true` if type is structured (array or object), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_structured()` for all JSON + types.,is_structured} + + @sa @ref is_primitive() -- returns whether value is primitive + @sa @ref is_array() -- returns whether value is an array + @sa @ref is_object() -- returns whether value is an object + + @since version 1.0.0 + */ + constexpr bool is_structured() const noexcept + { + return is_array() or is_object(); + } + + /*! + @brief return whether value is null + + This function returns true iff the JSON value is null. + + @return `true` if type is null, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_null()` for all JSON + types.,is_null} + + @since version 1.0.0 + */ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /*! + @brief return whether value is a boolean + + This function returns true iff the JSON value is a boolean. + + @return `true` if type is boolean, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_boolean()` for all JSON + types.,is_boolean} + + @since version 1.0.0 + */ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /*! + @brief return whether value is a number + + This function returns true iff the JSON value is a number. This includes + both integer and floating-point values. + + @return `true` if type is number (regardless whether integer, unsigned + integer or floating-type), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number()` for all JSON + types.,is_number} + + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number() const noexcept + { + return is_number_integer() or is_number_float(); + } + + /*! + @brief return whether value is an integer number + + This function returns true iff the JSON value is an integer or unsigned + integer number. This excludes floating-point values. + + @return `true` if type is an integer or unsigned integer number, `false` + otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_integer()` for all + JSON types.,is_number_integer} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer or m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is an unsigned integer number + + This function returns true iff the JSON value is an unsigned integer + number. This excludes floating-point and (signed) integer values. + + @return `true` if type is an unsigned integer number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_unsigned()` for all + JSON types.,is_number_unsigned} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 2.0.0 + */ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is a floating-point number + + This function returns true iff the JSON value is a floating-point number. + This excludes integer and unsigned integer values. + + @return `true` if type is a floating-point number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_float()` for all + JSON types.,is_number_float} + + @sa @ref is_number() -- check if value is number + @sa @ref is_number_integer() -- check if value is an integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + + @since version 1.0.0 + */ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /*! + @brief return whether value is an object + + This function returns true iff the JSON value is an object. + + @return `true` if type is object, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_object()` for all JSON + types.,is_object} + + @since version 1.0.0 + */ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /*! + @brief return whether value is an array + + This function returns true iff the JSON value is an array. + + @return `true` if type is array, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_array()` for all JSON + types.,is_array} + + @since version 1.0.0 + */ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /*! + @brief return whether value is a string + + This function returns true iff the JSON value is a string. + + @return `true` if type is string, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_string()` for all JSON + types.,is_string} + + @since version 1.0.0 + */ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /*! + @brief return whether value is discarded + + This function returns true iff the JSON value was discarded during parsing + with a callback function (see @ref parser_callback_t). + + @note This function will always be `false` for JSON values after parsing. + That is, discarded values can only occur during parsing, but will be + removed when inside a structured value or replaced by null in other cases. + + @return `true` if type is discarded, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_discarded()` for all JSON + types.,is_discarded} + + @since version 1.0.0 + */ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /*! + @brief return the type of the JSON value (implicit) + + Implicitly return the type of the JSON value as a value from the @ref + value_t enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies the @ref value_t operator for + all JSON types.,operator__value_t} + + @since version 1.0.0 + */ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get an object (explicit) + template ::value and + std::is_convertible::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_object()) + { + return T(m_value.object->begin(), m_value.object->end()); + } + else + { + throw std::domain_error("type must be object, but is " + type_name()); + } + } + + /// get an object (explicit) + object_t get_impl(object_t*) const + { + if (is_object()) + { + return *(m_value.object); + } + else + { + throw std::domain_error("type must be object, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not std::is_same::value and + not std::is_arithmetic::value and + not std::is_convertible::value and + not has_mapped_type::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + T to_vector; + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get(); + }); + return to_vector; + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not std::is_same::value + , int>::type = 0> + std::vector get_impl(std::vector*) const + { + if (is_array()) + { + std::vector to_vector; + to_vector.reserve(m_value.array->size()); + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get(); + }); + return to_vector; + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not has_mapped_type::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + return T(m_value.array->begin(), m_value.array->end()); + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + array_t get_impl(array_t*) const + { + if (is_array()) + { + return *(m_value.array); + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get a string (explicit) + template ::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_string()) + { + return *m_value.string; + } + else + { + throw std::domain_error("type must be string, but is " + type_name()); + } + } + + /// get a number (explicit) + template::value + , int>::type = 0> + T get_impl(T*) const + { + switch (m_type) + { + case value_t::number_integer: + { + return static_cast(m_value.number_integer); + } + + case value_t::number_unsigned: + { + return static_cast(m_value.number_unsigned); + } + + case value_t::number_float: + { + return static_cast(m_value.number_float); + } + + default: + { + throw std::domain_error("type must be number, but is " + type_name()); + } + } + } + + /// get a boolean (explicit) + constexpr boolean_t get_impl(boolean_t*) const + { + return is_boolean() + ? m_value.boolean + : throw std::domain_error("type must be boolean, but is " + type_name()); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t*) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t*) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t*) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t*) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t*) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t*) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t*) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t*) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t*) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This funcion helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw std::domain_error if ReferenceType does not match underlying value + type of the current JSON + */ + template + static ReferenceType get_ref_impl(ThisType& obj) + { + // helper type + using PointerType = typename std::add_pointer::type; + + // delegate the call to get_ptr<>() + auto ptr = obj.template get_ptr(); + + if (ptr != nullptr) + { + return *ptr; + } + else + { + throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + + obj.type_name()); + } + } + + public: + + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON; example: `"type must be object, but is null"` + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,get__ValueType_const} + + @internal + The idea of using a casted null pointer to choose the correct + implementation is from . + @endinternal + + @sa @ref operator ValueType() const for implicit conversion + @sa @ref get() for pointer-member access + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + ValueType get() const + { + return get_impl(static_cast(nullptr)); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + PointerType get() noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template::value + , int>::type = 0> + constexpr const PointerType get() const noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are + made. + + @warning Writing data to the pointee of the result yields an undefined + state. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get_ptr} + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + PointerType get_ptr() noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a pointer value (implicit) + @copydoc get_ptr() + */ + template::value + and std::is_const::type>::value + , int>::type = 0> + constexpr const PointerType get_ptr() const noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a reference value (implicit) + + Implict reference access to the internally stored JSON value. No copies + are made. + + @warning Writing data to the referee of the result yields an undefined + state. + + @tparam ReferenceType reference type; must be a reference to @ref array_t, + @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or + @ref number_float_t. Enforced by static assertion. + + @return reference to the internally stored JSON value if the requested + reference type @a ReferenceType fits to the JSON value; throws + std::domain_error otherwise + + @throw std::domain_error in case passed type @a ReferenceType is + incompatible with the stored JSON value + + @complexity Constant. + + @liveexample{The example shows several calls to `get_ref()`.,get_ref} + + @since version 1.1.0 + */ + template::value + , int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a reference value (implicit) + @copydoc get_ref() + */ + template::value + and std::is_const::type>::value + , int>::type = 0> + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON, thrown by @ref get() const + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename + std::enable_if < + not std::is_pointer::value + and not std::is_same::value +#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 + and not std::is_same>::value +#endif + , int >::type = 0 > + operator ValueType() const + { + // delegate the call to get<>() const + return get(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /*! + @brief access specified array element with bounds checking + + Returns a reference to the element at specified location @a idx, with + bounds checking. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read and + written using `at()`.,at__size_type} + + @since version 1.0.0 + */ + reference at(size_type idx) + { + // at only works for arrays + if (is_array()) + { + try + { + return m_value.array->at(idx); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element with bounds checking + + Returns a const reference to the element at specified location @a idx, + with bounds checking. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + `at()`.,at__size_type_const} + + @since version 1.0.0 + */ + const_reference at(size_type idx) const + { + // at only works for arrays + if (is_array()) + { + try + { + return m_value.array->at(idx); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a reference to the element at with specified key @a key, with + bounds checking. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using `at()`.,at__object_t_key_type} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (is_object()) + { + try + { + return m_value.object->at(key); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("key '" + key + "' not found"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a const reference to the element at with specified key @a key, + with bounds checking. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + `at()`.,at__object_t_key_type_const} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (is_object()) + { + try + { + return m_value.object->at(key); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("key '" + key + "' not found"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a reference to the element at specified location @a idx. + + @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), + then the array is silently filled up with `null` values to make `idx` a + valid reference to the last stored element. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array or null; example: + `"cannot use operator[] with string"` + + @complexity Constant if @a idx is in the range of the array. Otherwise + linear in `idx - size()`. + + @liveexample{The example below shows how array elements can be read and + written using `[]` operator. Note the addition of `null` + values.,operatorarray__size_type} + + @since version 1.0.0 + */ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create(); + assert_invariant(); + } + + // operator[] only works for arrays + if (is_array()) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { + m_value.array->insert(m_value.array->end(), + idx - m_value.array->size() + 1, + basic_json()); + } + + return m_value.array->operator[](idx); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a const reference to the element at specified location @a idx. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array; example: `"cannot use + operator[] with null"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + the `[]` operator.,operatorarray__size_type_const} + + @since version 1.0.0 + */ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (is_array()) + { + return m_value.array->operator[](idx); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + // operator[] only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + reference operator[](T * (&key)[n]) + { + return operator[](static_cast(key)); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @note This function is required for compatibility reasons with Clang. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + const_reference operator[](T * (&key)[n]) const + { + return operator[](static_cast(key)); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + const_reference operator[](T* key) const + { + // at only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(key); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const typename object_t::key_type&), this function + does not throw if the given key @a key was not found. + + @note Unlike @ref operator[](const typename object_t::key_type& key), this + function does not implicitly add an element to the position defined by @a + key. This function is furthermore also applicable to const objects. + + @param[in] key key of the element to access + @param[in] default_value the value to return if @a key is not found + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + + @since version 1.0.0 + */ + template ::value + , int>::type = 0> + ValueType value(const typename object_t::key_type& key, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return *it; + } + else + { + return default_value; + } + } + else + { + throw std::domain_error("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const + */ + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /*! + @brief access specified object element via JSON Pointer with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(ptr); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const json_pointer&), this function does not throw + if the given key @a key was not found. + + @param[in] ptr a JSON pointer to the element to access + @param[in] default_value the value to return if @a ptr found no value + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value_ptr} + + @sa @ref operator[](const json_pointer&) for unchecked access by reference + + @since version 2.0.2 + */ + template ::value + , int>::type = 0> + ValueType value(const json_pointer& ptr, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if pointer resolves a value, return it or use default value + try + { + return ptr.get_checked(this); + } + catch (std::out_of_range&) + { + return default_value; + } + } + else + { + throw std::domain_error("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const json_pointer&, ValueType) const + */ + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /*! + @brief access the first element + + Returns a reference to the first element in the container. For a JSON + container `c`, the expression `c.front()` is equivalent to `*c.begin()`. + + @return In case of a structured type (array or object), a reference to the + first element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, guarded by assertions). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value + + @liveexample{The following code shows an example for `front()`.,front} + + @sa @ref back() -- access the last element + + @since version 1.0.0 + */ + reference front() + { + return *begin(); + } + + /*! + @copydoc basic_json::front() + */ + const_reference front() const + { + return *cbegin(); + } + + /*! + @brief access the last element + + Returns a reference to the last element in the container. For a JSON + container `c`, the expression `c.back()` is equivalent to + @code {.cpp} + auto tmp = c.end(); + --tmp; + return *tmp; + @endcode + + @return In case of a structured type (array or object), a reference to the + last element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, guarded by assertions). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value. + + @liveexample{The following code shows an example for `back()`.,back} + + @sa @ref front() -- access the first element + + @since version 1.0.0 + */ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /*! + @copydoc basic_json::back() + */ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /*! + @brief remove element given an iterator + + Removes the element specified by iterator @a pos. The iterator @a pos must + be valid and dereferenceable. Thus the `end()` iterator (which is valid, + but is not dereferenceable) cannot be used as a value for @a pos. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] pos iterator to the element to remove + @return Iterator following the last removed element. If the iterator @a + pos refers to the last element, the `end()` iterator is returned. + + @tparam InteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on an iterator which does not belong to + the current JSON value; example: `"iterator does not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterator (i.e., any iterator which is not `begin()`); example: `"iterator + out of range"` + + @complexity The complexity depends on the type: + - objects: amortized constant + - arrays: linear in distance between pos and the end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType} + + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + InteratorType erase(InteratorType pos) + { + // make sure iterator fits the current value + if (this != pos.m_object) + { + throw std::domain_error("iterator does not fit current value"); + } + + InteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not pos.m_it.primitive_iterator.is_begin()) + { + throw std::out_of_range("iterator out of range"); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove elements given an iterator range + + Removes the element specified by the range `[first; last)`. The iterator + @a first does not need to be dereferenceable if `first == last`: erasing + an empty range is a no-op. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] first iterator to the beginning of the range to remove + @param[in] last iterator past the end of the range to remove + @return Iterator following the last removed element. If the iterator @a + second refers to the last element, the `end()` iterator is returned. + + @tparam InteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on iterators which does not belong to + the current JSON value; example: `"iterators do not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterators (i.e., if `first != begin()` and `last != end()`); example: + `"iterators out of range"` + + @complexity The complexity depends on the type: + - objects: `log(size()) + std::distance(first, last)` + - arrays: linear in the distance between @a first and @a last, plus linear + in the distance between @a last and end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType_IteratorType} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + InteratorType erase(InteratorType first, InteratorType last) + { + // make sure iterator fits the current value + if (this != first.m_object or this != last.m_object) + { + throw std::domain_error("iterators do not fit current value"); + } + + InteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + throw std::out_of_range("iterators out of range"); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove element from a JSON object given a key + + Removes elements from a JSON object with the key value @a key. + + @param[in] key value of the elements to remove + + @return Number of elements removed. If @a ObjectType is the default + `std::map` type, the return value will always be `0` (@a key was not + found) or `1` (@a key was found). + + @post References and iterators to the erased elements are invalidated. + Other references and iterators are not affected. + + @throw std::domain_error when called on a type other than JSON object; + example: `"cannot use erase() with null"` + + @complexity `log(size()) + count(key)` + + @liveexample{The example shows the effect of `erase()`.,erase__key_type} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (is_object()) + { + return m_value.object->erase(key); + } + else + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + /*! + @brief remove element from a JSON array given an index + + Removes element from a JSON array at the index @a idx. + + @param[in] idx index of the element to remove + + @throw std::domain_error when called on a type other than JSON array; + example: `"cannot use erase() with null"` + @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + is out of range"` + + @complexity Linear in distance between @a idx and the end of the container. + + @liveexample{The example shows the effect of `erase()`.,erase__size_type} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + + @since version 1.0.0 + */ + void erase(const size_type idx) + { + // this erase only works for arrays + if (is_array()) + { + if (idx >= size()) + { + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + + m_value.array->erase(m_value.array->begin() + static_cast(idx)); + } + else + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /*! + @brief find an element in a JSON object + + Finds an element in a JSON object with key equivalent to @a key. If the + element is not found or the JSON value is not an object, end() is + returned. + + @param[in] key key value of the element to search for + + @return Iterator to an element with key equivalent to @a key. If no such + element is found, past-the-end (see end()) iterator is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `find()` is used.,find__key_type} + + @since version 1.0.0 + */ + iterator find(typename object_t::key_type key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief find an element in a JSON object + @copydoc find(typename object_t::key_type) + */ + const_iterator find(typename object_t::key_type key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief returns the number of occurrences of a key in a JSON object + + Returns the number of elements with key @a key. If ObjectType is the + default `std::map` type, the return value will always be `0` (@a key was + not found) or `1` (@a key was found). + + @param[in] key key value of the element to count + + @return Number of elements with key @a key. If the JSON value is not an + object, the return value will be `0`. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `count()` is used.,count} + + @since version 1.0.0 + */ + size_type count(typename object_t::key_type key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(key) : 0; + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /*! + @brief returns an iterator to the first element + + Returns an iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `begin()`.,begin} + + @sa @ref cbegin() -- returns a const iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /*! + @copydoc basic_json::cbegin() + */ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /*! + @brief returns a const iterator to the first element + + Returns a const iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).begin()`. + + @liveexample{The following code shows an example for `cbegin()`.,cbegin} + + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /*! + @brief returns an iterator to one past the last element + + Returns an iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `end()`.,end} + + @sa @ref cend() -- returns a const iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /*! + @copydoc basic_json::cend() + */ + const_iterator end() const noexcept + { + return cend(); + } + + /*! + @brief returns a const iterator to one past the last element + + Returns a const iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).end()`. + + @liveexample{The following code shows an example for `cend()`.,cend} + + @sa @ref end() -- returns an iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /*! + @brief returns an iterator to the reverse-beginning + + Returns an iterator to the reverse-beginning; that is, the last element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(end())`. + + @liveexample{The following code shows an example for `rbegin()`.,rbegin} + + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /*! + @copydoc basic_json::crbegin() + */ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /*! + @brief returns an iterator to the reverse-end + + Returns an iterator to the reverse-end; that is, one before the first + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(begin())`. + + @liveexample{The following code shows an example for `rend()`.,rend} + + @sa @ref crend() -- returns a const reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /*! + @copydoc basic_json::crend() + */ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /*! + @brief returns a const reverse iterator to the last element + + Returns a const iterator to the reverse-beginning; that is, the last + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rbegin()`. + + @liveexample{The following code shows an example for `crbegin()`.,crbegin} + + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /*! + @brief returns a const reverse iterator to one before the first + + Returns a const reverse iterator to the reverse-end; that is, one before + the first element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rend()`. + + @liveexample{The following code shows an example for `crend()`.,crend} + + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + private: + // forward declaration + template class iteration_proxy; + + public: + /*! + @brief wrapper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + @note The name of this function is not yet final and may change in the + future. + */ + static iteration_proxy iterator_wrapper(reference cont) + { + return iteration_proxy(cont); + } + + /*! + @copydoc iterator_wrapper(reference) + */ + static iteration_proxy iterator_wrapper(const_reference cont) + { + return iteration_proxy(cont); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /*! + @brief checks whether the container is empty + + Checks if a JSON value has no elements. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `true` + boolean | `false` + string | `false` + number | `false` + object | result of function `object_t::empty()` + array | result of function `array_t::empty()` + + @note This function does not return whether a string stored as JSON value + is empty - it returns whether the JSON container itself is empty which is + false in the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `empty()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `begin() == end()`. + + @liveexample{The following code uses `empty()` to check if a JSON + object contains any elements.,empty} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + default: + { + // all other types are nonempty + return false; + } + } + } + + /*! + @brief returns the number of elements + + Returns the number of elements in a JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` + boolean | `1` + string | `1` + number | `1` + object | result of function object_t::size() + array | result of function array_t::size() + + @note This function does not return the length of a string stored as JSON + value - it returns the number of elements in the JSON value which is 1 in + the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their size() functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `std::distance(begin(), end())`. + + @liveexample{The following code calls `size()` on the different value + types.,size} + + @sa @ref empty() -- checks whether the container is empty + @sa @ref max_size() -- returns the maximal number of elements + + @since version 1.0.0 + */ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + default: + { + // all other types have size 1 + return 1; + } + } + } + + /*! + @brief returns the maximum possible number of elements + + Returns the maximum number of elements a JSON value is able to hold due to + system or library implementation limitations, i.e. `std::distance(begin(), + end())` for the JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` (same as `size()`) + boolean | `1` (same as `size()`) + string | `1` (same as `size()`) + number | `1` (same as `size()`) + object | result of function `object_t::max_size()` + array | result of function `array_t::max_size()` + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `max_size()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of returning `b.size()` where `b` is the largest + possible JSON value. + + @liveexample{The following code calls `max_size()` on the different value + types. Note the output is implementation specific.,max_size} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /*! + @brief clears the contents + + Clears the content of a JSON value and resets it to the default value as + if @ref basic_json(value_t) would have been called: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @note Floating-point numbers are set to `0.0` which will be serialized to + `0`. The vale type remains @ref number_float_t. + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows the effect of `clear()` to different + JSON types.,clear} + + @since version 1.0.0 + */ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + default: + { + break; + } + } + } + + /*! + @brief add an object to an array + + Appends the given element @a val to the end of the JSON value. If the + function is called on a JSON null value, an empty array is created before + appending @a val. + + @param[in] val the value to add to the JSON array + + @throw std::domain_error when called on a type other than JSON array or + null; example: `"cannot use push_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON array. Note how the `null` value was silently + converted to a JSON array.,push_back} + + @since version 1.0.0 + */ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + m_value.array->push_back(std::move(val)); + // invalidate object + val.m_type = value_t::null; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + m_value.array->push_back(val); + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + Inserts the given element @a val to the JSON object. If the function is + called on a JSON null value, an empty object is created before inserting + @a val. + + @param[in] val the value to add to the JSON object + + @throw std::domain_error when called on a type other than JSON object or + null; example: `"cannot use push_back() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON object. Note how the `null` value was silently + converted to a JSON object.,push_back__object_t__value} + + @since version 1.0.0 + */ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (not(is_null() or is_object())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array + m_value.object->insert(val); + } + + /*! + @brief add an object to an object + @copydoc push_back(const typename object_t::value_type&) + */ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + This function allows to use `push_back` with an initializer list. In case + + 1. the current value is an object, + 2. the initializer list @a init contains only two elements, and + 3. the first element of @a init is a string, + + @a init is converted into an object element and added using + @ref push_back(const typename object_t::value_type&). Otherwise, @a init + is converted to a JSON value and added using @ref push_back(basic_json&&). + + @param init an initializer list + + @complexity Linear in the size of the initializer list @a init. + + @note This function is required to resolve an ambiguous overload error, + because pairs like `{"key", "value"}` can be both interpreted as + `object_t::value_type` or `std::initializer_list`, see + https://github.com/nlohmann/json/issues/235 for more information. + + @liveexample{The example shows how initializer lists are treated as + objects when possible.,push_back__initializer_list} + */ + void push_back(std::initializer_list init) + { + if (is_object() and init.size() == 2 and init.begin()->is_string()) + { + const string_t key = *init.begin(); + push_back(typename object_t::value_type(key, *(init.begin() + 1))); + } + else + { + push_back(basic_json(init)); + } + } + + /*! + @brief add an object to an object + @copydoc push_back(std::initializer_list) + */ + reference operator+=(std::initializer_list init) + { + push_back(init); + return *this; + } + + /*! + @brief inserts element + + Inserts element @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] val element to insert + @return iterator pointing to the inserted @a val. + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Constant plus linear in the distance between pos and end of the + container. + + @liveexample{The example shows how `insert()` is used.,insert} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); + return result; + } + else + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts element + @copydoc insert(const_iterator, const basic_json&) + */ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /*! + @brief inserts elements + + Inserts @a cnt copies of @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] cnt number of copies of @a val to insert + @param[in] val element to insert + @return iterator pointing to the first element inserted, or @a pos if + `cnt==0` + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Linear in @a cnt plus linear in the distance between @a pos + and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__count} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + return result; + } + else + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts elements + + Inserts elements from range `[first, last)` before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + @throw std::domain_error if @a first and @a last do not belong to the same + JSON value; example: `"iterators do not fit"` + @throw std::domain_error if @a first or @a last are iterators into + container for which insert is called; example: `"passed iterators may not + belong to container"` + + @return iterator pointing to the first element inserted, or @a pos if + `first==last` + + @complexity Linear in `std::distance(first, last)` plus linear in the + distance between @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__range} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (not is_array()) + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + throw std::domain_error("iterators do not fit"); + } + + if (first.m_object == this or last.m_object == this) + { + throw std::domain_error("passed iterators may not belong to container"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert( + pos.m_it.array_iterator, + first.m_it.array_iterator, + last.m_it.array_iterator); + return result; + } + + /*! + @brief inserts elements + + Inserts elements from initializer list @a ilist before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] ilist initializer list to insert the values from + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @return iterator pointing to the first element inserted, or @a pos if + `ilist` is empty + + @complexity Linear in `ilist.size()` plus linear in the distance between + @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__ilist} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, std::initializer_list ilist) + { + // insert only works for arrays + if (not is_array()) + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); + return result; + } + + /*! + @brief exchanges the values + + Exchanges the contents of the JSON value with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other JSON value to exchange the contents with + + @complexity Constant. + + @liveexample{The example below shows how JSON values can be swapped with + `swap()`.,swap__reference} + + @since version 1.0.0 + */ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + assert_invariant(); + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON array with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other array to exchange the contents with + + @throw std::domain_error when JSON value is not an array; example: `"cannot + use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how arrays can be swapped with + `swap()`.,swap__array_t} + + @since version 1.0.0 + */ + void swap(array_t& other) + { + // swap only works for arrays + if (is_array()) + { + std::swap(*(m_value.array), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON object with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other object to exchange the contents with + + @throw std::domain_error when JSON value is not an object; example: + `"cannot use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how objects can be swapped with + `swap()`.,swap__object_t} + + @since version 1.0.0 + */ + void swap(object_t& other) + { + // swap only works for objects + if (is_object()) + { + std::swap(*(m_value.object), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON string with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other string to exchange the contents with + + @throw std::domain_error when JSON value is not a string; example: `"cannot + use swap() with boolean"` + + @complexity Constant. + + @liveexample{The example below shows how strings can be swapped with + `swap()`.,swap__string_t} + + @since version 1.0.0 + */ + void swap(string_t& other) + { + // swap only works for strings + if (is_string()) + { + std::swap(*(m_value.string), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /// @} + + + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + private: + /*! + @brief comparison operator for JSON types + + Returns an ordering that is similar to Python: + - order: null < boolean < number < object < array < string + - furthermore, each type is not smaller than itself + + @since version 1.0.0 + */ + friend bool operator<(const value_t lhs, const value_t rhs) noexcept + { + static constexpr std::array order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) + { + return false; + } + + return order[static_cast(lhs)] < order[static_cast(rhs)]; + } + + public: + /*! + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same. + - Integer and floating-point numbers are automatically converted before + comparison. Floating-point numbers are compared indirectly: two + floating-point numbers `f1` and `f2` are considered equal if neither + `f1 > f2` nor `f2 > f1` holds. + - Two JSON null values are equal. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array == *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object == *rhs.m_value.object; + } + case value_t::null: + { + return true; + } + case value_t::string: + { + return *lhs.m_value.string == *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean == rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer == rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float == rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; + } + + /*! + @brief comparison: equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__equal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator==(const_reference v, std::nullptr_t) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, std::nullptr_t) + */ + friend bool operator==(std::nullptr_t, const_reference v) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs == rhs); + } + + /*! + @brief comparison: not equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `not v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is not null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__notequal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference v, std::nullptr_t) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, std::nullptr_t) + */ + friend bool operator!=(std::nullptr_t, const_reference v) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: less than + + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. + - Integer and floating-point numbers are automatically converted before + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__less} + + @since version 1.0.0 + */ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array < *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object < *rhs.m_value.object; + } + case value_t::null: + { + return false; + } + case value_t::string: + { + return *lhs.m_value.string < *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean < rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer < rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float < rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /*! + @brief comparison: less than or equal + + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} + + @since version 1.0.0 + */ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return not (rhs < lhs); + } + + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__lessequal} + + @since version 1.0.0 + */ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs <= rhs); + } + + /*! + @brief comparison: greater than or equal + + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} + + @since version 1.0.0 + */ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs < rhs); + } + + /// @} + + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ + + /*! + @brief serialize to stream + + Serialize the given JSON value @a j to the output stream @a o. The JSON + value will be serialized using the @ref dump member function. The + indentation of the output can be controlled with the member variable + `width` of the output stream @a o. For instance, using the manipulator + `std::setw(4)` on @a o sets the indentation level to `4` and the + serialization result is the same as calling `dump(4)`. + + @note During serializaion, the locale and the precision of the output + stream @a o are changed. The original values are restored when the + function returns. + + @param[in,out] o stream to serialize to + @param[in] j JSON value to serialize + + @return the stream @a o + + @complexity Linear. + + @liveexample{The example below shows the serialization with different + parameters to `width` to adjust the indentation level.,operator_serialize} + + @since version 1.0.0 + */ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = (o.width() > 0); + const auto indentation = (pretty_print ? o.width() : 0); + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // fix locale problems + const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator)); + // set precision + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits::digits10 + const auto old_precision = o.precision(std::numeric_limits::digits10); + + // do the actual serialization + j.dump(o, pretty_print, static_cast(indentation)); + + // reset locale and precision + o.imbue(old_locale); + o.precision(old_precision); + return o; + } + + /*! + @brief serialize to stream + @copydoc operator<<(std::ostream&, const basic_json&) + */ + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } + + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /*! + @brief deserialize from string + + @param[in] s string to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @sa @ref parse(std::istream&, const parser_callback_t) for a version that + reads from an input stream + + @since version 1.0.0 + */ + static basic_json parse(const string_t& s, + const parser_callback_t cb = nullptr) + { + return parser(s, cb).parse(); + } + + /*! + @brief deserialize from stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__istream__parser_callback_t} + + @sa @ref parse(const string_t&, const parser_callback_t) for a version + that reads from a string + + @since version 1.0.0 + */ + static basic_json parse(std::istream& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @copydoc parse(std::istream&, const parser_callback_t) + */ + static basic_json parse(std::istream&& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @brief deserialize from stream + + Deserializes an input stream to a JSON value. + + @param[in,out] i input stream to read a serialized JSON value from + @param[in,out] j JSON value to write the deserialized input to + + @throw std::invalid_argument in case of parse errors + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below shows how a JSON value is constructed by + reading a serialization from a stream.,operator_deserialize} + + @sa parse(std::istream&, const parser_callback_t) for a variant with a + parser callback function to filter values while parsing + + @since version 1.0.0 + */ + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; + } + + /*! + @brief deserialize from stream + @copydoc operator<<(basic_json&, std::istream&) + */ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + j = parser(i).parse(); + return i; + } + + /// @} + + + private: + /////////////////////////// + // convenience functions // + /////////////////////////// + + /*! + @brief return the type as string + + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. + + @return basically a string representation of a the @ref m_type member + + @complexity Constant. + + @since version 1.0.0 + */ + std::string type_name() const + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; + } + } + + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + else + { + return res; + } + } + } + }); + } + + /*! + @brief escape a string + + Escape a string by replacing certain special characters by a sequence of + an escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. + + @param[in] s the string to escape + @return the escaped string + + @complexity Linear in the length of string @a s. + */ + static string_t escape_string(const string_t& s) + { + const auto space = extra_space(s); + if (space == 0) + { + return s; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + } + else + { + // all other characters are added as-is + result[pos++] = c; + } + break; + } + } + } + + return result; + } + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. Note that + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[out] o stream to write to + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(std::ostream& o, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) const + { + // variable to hold indentation for recursive calls + unsigned int new_indent = current_indent; + + switch (m_type) + { + case value_t::object: + { + if (m_value.object->empty()) + { + o << "{}"; + return; + } + + o << "{"; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) + { + if (i != m_value.object->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' ') << "\"" + << escape_string(i->first) << "\":" + << (pretty_print ? " " : ""); + i->second.dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') + "}"; + return; + } + + case value_t::array: + { + if (m_value.array->empty()) + { + o << "[]"; + return; + } + + o << "["; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) + { + if (i != m_value.array->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' '); + i->dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') << "]"; + return; + } + + case value_t::string: + { + o << string_t("\"") << escape_string(*m_value.string) << "\""; + return; + } + + case value_t::boolean: + { + o << (m_value.boolean ? "true" : "false"); + return; + } + + case value_t::number_integer: + { + o << m_value.number_integer; + return; + } + + case value_t::number_unsigned: + { + o << m_value.number_unsigned; + return; + } + + case value_t::number_float: + { + if (m_value.number_float == 0) + { + // special case for zero to get "0.0"/"-0.0" + o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); + } + else + { + o << m_value.number_float; + } + return; + } + + case value_t::discarded: + { + o << ""; + return; + } + + case value_t::null: + { + o << "null"; + return; + } + } + } + + private: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + + + private: + /////////////// + // iterators // + /////////////// + + /*! + @brief an iterator for primitive JSON types + + This class models an iterator for primitive JSON types (boolean, number, + string). It's only purpose is to allow the iterator/const_iterator classes + to "iterate" over primitive values. Internally, the iterator is modeled by + a `difference_type` variable. Value begin_value (`0`) models the begin, + end_value (`1`) models past the end. + */ + class primitive_iterator_t + { + public: + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return (m_it == begin_value); + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return (m_it == end_value); + } + + /// return reference to the value to change and compare + operator difference_type& () noexcept + { + return m_it; + } + + /// return value to compare + constexpr operator difference_type () const noexcept + { + return m_it; + } + + private: + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + /// iterator as signed integer type + difference_type m_it = std::numeric_limits::denorm_min(); + }; + + /*! + @brief an iterator value + + @note This structure could easily be a union, but MSVC currently does not + allow unions members with complex constructors, see + https://github.com/nlohmann/json/pull/105. + */ + struct internal_iterator + { + /// iterator for JSON objects + typename object_t::iterator object_iterator; + /// iterator for JSON arrays + typename array_t::iterator array_iterator; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator; + + /// create an uninitialized internal_iterator + internal_iterator() noexcept + : object_iterator(), array_iterator(), primitive_iterator() + {} + }; + + /// proxy class for the iterator_wrapper functions + template + class iteration_proxy + { + private: + /// helper class for iteration + class iteration_proxy_internal + { + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + size_t array_index = 0; + + public: + explicit iteration_proxy_internal(IteratorType it) noexcept + : anchor(it) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_internal& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_internal& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// inequality operator (needed for range-based for) + bool operator!= (const iteration_proxy_internal& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + typename basic_json::string_t key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + return std::to_string(array_index); + } + + // use key from the object + case value_t::object: + { + return anchor.key(); + } + + // use an empty key for all primitive types + default: + { + return ""; + } + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } + }; + + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) + : container(cont) + {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_internal begin() noexcept + { + return iteration_proxy_internal(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_internal end() noexcept + { + return iteration_proxy_internal(container.end()); + } + }; + + public: + /*! + @brief a const random access iterator for the @ref basic_json class + + This class implements a const iterator for the @ref basic_json class. From + this class, the @ref iterator class is derived. + + @note An iterator is called *initialized* when a pointer to a JSON value + has been set (e.g., by a constructor or a copy assignment). If the + iterator is default-constructed, it is *uninitialized* and most + methods are undefined. The library uses assertions to detect calls + on uninitialized iterators. + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + + @since version 1.0.0 + */ + class const_iterator : public std::iterator + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /// the type of the values when the iterator is dereferenced + using value_type = typename basic_json::value_type; + /// a type to represent differences between iterators + using difference_type = typename basic_json::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename basic_json::const_pointer; + /// defines a reference to the type iterated over (value_type) + using reference = typename basic_json::const_reference; + /// the category of the iterator + using iterator_category = std::bidirectional_iterator_tag; + + /// default constructor + const_iterator() = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit const_iterator(pointer object) noexcept + : m_object(object) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @brief copy constructor given a non-const iterator + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + explicit const_iterator(const iterator& other) noexcept + : m_object(other.m_object) + { + if (m_object != nullptr) + { + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = other.m_it.object_iterator; + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = other.m_it.array_iterator; + break; + } + + default: + { + m_it.primitive_iterator = other.m_it.primitive_iterator; + break; + } + } + } + } + + /*! + @brief copy constructor + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator(const const_iterator& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief copy assignment + @param[in,out] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator& operator=(const_iterator other) noexcept( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_object, other.m_object); + std::swap(m_it, other.m_it); + return *this; + } + + private: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case basic_json::value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case basic_json::value_t::null: + { + throw std::out_of_range("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return *m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator++(int) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator++() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator--(int) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator--() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator==(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + throw std::domain_error("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator == other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator!=(const const_iterator& other) const + { + return not operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + throw std::domain_error("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot compare order of object iterators"); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator < other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const const_iterator& other) const + { + return not other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const const_iterator& other) const + { + return not operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const const_iterator& other) const + { + return not operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator+=(difference_type i) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const const_iterator& other) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + return m_it.array_iterator - other.m_it.array_iterator; + } + + default: + { + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use operator[] for object iterators"); + } + + case basic_json::value_t::array: + { + return *std::next(m_it.array_iterator, n); + } + + case basic_json::value_t::null: + { + throw std::out_of_range("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator == -n) + { + return *m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + typename object_t::key_type key() const + { + assert(m_object != nullptr); + + if (m_object->is_object()) + { + return m_it.object_iterator->first; + } + else + { + throw std::domain_error("cannot use key() for non-object iterators"); + } + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + private: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator m_it = internal_iterator(); + }; + + /*! + @brief a mutable random access iterator for the @ref basic_json class + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element. + + @since version 1.0.0 + */ + class iterator : public const_iterator + { + public: + using base_iterator = const_iterator; + using pointer = typename basic_json::pointer; + using reference = typename basic_json::reference; + + /// default constructor + iterator() = default; + + /// constructor for a given JSON instance + explicit iterator(pointer object) noexcept + : base_iterator(object) + {} + + /// copy constructor + iterator(const iterator& other) noexcept + : base_iterator(other) + {} + + /// copy assignment + iterator& operator=(iterator other) noexcept( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + base_iterator::operator=(other); + return *this; + } + + /// return a reference to the value pointed to by the iterator + reference operator*() const + { + return const_cast(base_iterator::operator*()); + } + + /// dereference the iterator + pointer operator->() const + { + return const_cast(base_iterator::operator->()); + } + + /// post-increment (it++) + iterator operator++(int) + { + iterator result = *this; + base_iterator::operator++(); + return result; + } + + /// pre-increment (++it) + iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + iterator operator--(int) + { + iterator result = *this; + base_iterator::operator--(); + return result; + } + + /// pre-decrement (--it) + iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// subtract from iterator + iterator& operator-=(difference_type i) + { + base_iterator::operator-=(i); + return *this; + } + + /// add to iterator + iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const iterator& other) const + { + return base_iterator::operator-(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return const_cast(base_iterator::operator[](n)); + } + + /// return the value of an iterator + reference value() const + { + return const_cast(base_iterator::value()); + } + }; + + /*! + @brief a template for a reverse iterator class + + @tparam Base the base iterator type to reverse. Valid types are @ref + iterator (to create @ref reverse_iterator) and @ref const_iterator (to + create @ref const_reverse_iterator). + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + + @since version 1.0.0 + */ + template + class json_reverse_iterator : public std::reverse_iterator + { + public: + /// shortcut to the reverse iterator adaptor + using base_iterator = std::reverse_iterator; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) + {} + + /// create reverse iterator from base class + json_reverse_iterator(const base_iterator& it) noexcept + : base_iterator(it) + {} + + /// post-increment (it++) + json_reverse_iterator operator++(int) + { + return base_iterator::operator++(1); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + json_reverse_iterator operator--(int) + { + return base_iterator::operator--(1); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return this->base() - other.base(); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + typename object_t::key_type key() const + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } + }; + + + private: + ////////////////////// + // lexer and parser // + ////////////////////// + + /*! + @brief lexical analysis + + This class organizes the lexical analysis during JSON deserialization. The + core of it is a scanner generated by [re2c](http://re2c.org) that + processes a buffer and recognizes tokens according to RFC 7159. + */ + class lexer + { + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_number, ///< a number -- use get_number() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input ///< indicating the end of the input buffer + }; + + /// the char type to use in the lexer + using lexer_char_t = unsigned char; + + /// constructor with a given buffer + explicit lexer(const string_t& s) noexcept + : m_stream(nullptr), m_buffer(s) + { + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + s.size(); + } + + /// constructor with a given stream + explicit lexer(std::istream* s) noexcept + : m_stream(s), m_buffer() + { + assert(m_stream != nullptr); + std::getline(*m_stream, m_buffer); + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + m_buffer.size(); + } + + /// default constructor + lexer() = default; + + // switch off unwanted functions + lexer(const lexer&) = delete; + lexer operator=(const lexer&) = delete; + + /*! + @brief create a string from one or two Unicode code points + + There are two cases: (1) @a codepoint1 is in the Basic Multilingual + Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) + @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to + represent a code point above U+FFFF. + + @param[in] codepoint1 the code point (can be high surrogate) + @param[in] codepoint2 the code point (can be low surrogate or 0) + + @return string representation of the code point; the length of the + result string is between 1 and 4 characters. + + @throw std::out_of_range if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` + @throw std::invalid_argument if the low surrogate is invalid; example: + `""missing or wrong low surrogate""` + + @complexity Constant. + + @see + */ + static string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) + { + // calculate the code point from the given code points + std::size_t codepoint = codepoint1; + + // check if codepoint1 is a high surrogate + if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) + { + // check if codepoint2 is a low surrogate + if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) + { + codepoint = + // high surrogate occupies the most significant 22 bits + (codepoint1 << 10) + // low surrogate occupies the least significant 15 bits + + codepoint2 + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00; + } + else + { + throw std::invalid_argument("missing or wrong low surrogate"); + } + } + + string_t result; + + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + result.append(1, static_cast(codepoint)); + } + else if (codepoint <= 0x7ff) + { + // 2-byte characters: 110xxxxx 10xxxxxx + result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xffff) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0x10ffff) + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else + { + throw std::out_of_range("code points above 0x10FFFF are invalid"); + } + + return result; + } + + /// return name of values of type token_type (only used for errors) + static std::string token_type_name(const token_type t) + { + switch (t) + { + case token_type::uninitialized: + return ""; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case token_type::value_number: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return ""; + case token_type::end_of_input: + return "end of input"; + default: + { + // catch non-enum values + return "unknown token"; // LCOV_EXCL_LINE + } + } + } + + /*! + This function implements a scanner for JSON. It is specified using + regular expressions that try to follow RFC 7159 as close as possible. + These regular expressions are then translated into a minimized + deterministic finite automaton (DFA) by the tool + [re2c](http://re2c.org). As a result, the translated code for this + function consists of a large block of code with `goto` jumps. + + @return the class of the next token read from the buffer + + @complexity Linear in the length of the input.\n + + Proposition: The loop below will always terminate for finite input.\n + + Proof (by contradiction): Assume a finite input. To loop forever, the + loop must never hit code with a `break` statement. The only code + snippets without a `break` statement are the continue statements for + whitespace and byte-order-marks. To loop forever, the input must be an + infinite sequence of whitespace or byte-order-marks. This contradicts + the assumption of finite input, q.e.d. + */ + token_type scan() noexcept + { + while (true) + { + // pointer for backtracking information + m_marker = nullptr; + + // remember the begin of the token + m_start = m_cursor; + assert(m_start != nullptr); + + + { + lexer_char_t yych; + unsigned int yyaccept = 0; + static const unsigned char yybm[] = + { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 32, 0, 0, 32, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + }; + if ((m_limit - m_cursor) < 5) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + if (yych <= '\\') + { + if (yych <= '-') + { + if (yych <= '"') + { + if (yych <= 0x00) + { + goto basic_json_parser_2; + } + if (yych <= '!') + { + goto basic_json_parser_4; + } + goto basic_json_parser_9; + } + else + { + if (yych <= '+') + { + goto basic_json_parser_4; + } + if (yych <= ',') + { + goto basic_json_parser_10; + } + goto basic_json_parser_12; + } + } + else + { + if (yych <= '9') + { + if (yych <= '/') + { + goto basic_json_parser_4; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + goto basic_json_parser_15; + } + else + { + if (yych <= ':') + { + goto basic_json_parser_17; + } + if (yych == '[') + { + goto basic_json_parser_19; + } + goto basic_json_parser_4; + } + } + } + else + { + if (yych <= 't') + { + if (yych <= 'f') + { + if (yych <= ']') + { + goto basic_json_parser_21; + } + if (yych <= 'e') + { + goto basic_json_parser_4; + } + goto basic_json_parser_23; + } + else + { + if (yych == 'n') + { + goto basic_json_parser_24; + } + if (yych <= 's') + { + goto basic_json_parser_4; + } + goto basic_json_parser_25; + } + } + else + { + if (yych <= '|') + { + if (yych == '{') + { + goto basic_json_parser_26; + } + goto basic_json_parser_4; + } + else + { + if (yych <= '}') + { + goto basic_json_parser_28; + } + if (yych == 0xEF) + { + goto basic_json_parser_30; + } + goto basic_json_parser_4; + } + } + } +basic_json_parser_2: + ++m_cursor; + { + last_token_type = token_type::end_of_input; + break; + } +basic_json_parser_4: + ++m_cursor; +basic_json_parser_5: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_6: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + { + continue; + } +basic_json_parser_9: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych <= 0x1F) + { + goto basic_json_parser_5; + } + goto basic_json_parser_32; +basic_json_parser_10: + ++m_cursor; + { + last_token_type = token_type::value_separator; + break; + } +basic_json_parser_12: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_5; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + if (yych <= '9') + { + goto basic_json_parser_15; + } + goto basic_json_parser_5; +basic_json_parser_13: + yyaccept = 1; + yych = *(m_marker = ++m_cursor); + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_37; + } + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + } +basic_json_parser_14: + { + last_token_type = token_type::value_number; + break; + } +basic_json_parser_15: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 64) + { + goto basic_json_parser_15; + } + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_37; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + goto basic_json_parser_14; + } +basic_json_parser_17: + ++m_cursor; + { + last_token_type = token_type::name_separator; + break; + } +basic_json_parser_19: + ++m_cursor; + { + last_token_type = token_type::begin_array; + break; + } +basic_json_parser_21: + ++m_cursor; + { + last_token_type = token_type::end_array; + break; + } +basic_json_parser_23: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'a') + { + goto basic_json_parser_39; + } + goto basic_json_parser_5; +basic_json_parser_24: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'u') + { + goto basic_json_parser_40; + } + goto basic_json_parser_5; +basic_json_parser_25: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'r') + { + goto basic_json_parser_41; + } + goto basic_json_parser_5; +basic_json_parser_26: + ++m_cursor; + { + last_token_type = token_type::begin_object; + break; + } +basic_json_parser_28: + ++m_cursor; + { + last_token_type = token_type::end_object; + break; + } +basic_json_parser_30: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 0xBB) + { + goto basic_json_parser_42; + } + goto basic_json_parser_5; +basic_json_parser_31: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; +basic_json_parser_32: + if (yybm[0 + yych] & 128) + { + goto basic_json_parser_31; + } + if (yych <= 0x1F) + { + goto basic_json_parser_33; + } + if (yych <= '"') + { + goto basic_json_parser_34; + } + goto basic_json_parser_36; +basic_json_parser_33: + m_cursor = m_marker; + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } +basic_json_parser_34: + ++m_cursor; + { + last_token_type = token_type::value_string; + break; + } +basic_json_parser_36: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= 'e') + { + if (yych <= '/') + { + if (yych == '"') + { + goto basic_json_parser_31; + } + if (yych <= '.') + { + goto basic_json_parser_33; + } + goto basic_json_parser_31; + } + else + { + if (yych <= '\\') + { + if (yych <= '[') + { + goto basic_json_parser_33; + } + goto basic_json_parser_31; + } + else + { + if (yych == 'b') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + } + } + else + { + if (yych <= 'q') + { + if (yych <= 'f') + { + goto basic_json_parser_31; + } + if (yych == 'n') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 's') + { + if (yych <= 'r') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 't') + { + goto basic_json_parser_31; + } + if (yych <= 'u') + { + goto basic_json_parser_43; + } + goto basic_json_parser_33; + } + } + } +basic_json_parser_37: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_44; + } + goto basic_json_parser_33; +basic_json_parser_38: + yych = *++m_cursor; + if (yych <= ',') + { + if (yych == '+') + { + goto basic_json_parser_46; + } + goto basic_json_parser_33; + } + else + { + if (yych <= '-') + { + goto basic_json_parser_46; + } + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_47; + } + goto basic_json_parser_33; + } +basic_json_parser_39: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_49; + } + goto basic_json_parser_33; +basic_json_parser_40: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_50; + } + goto basic_json_parser_33; +basic_json_parser_41: + yych = *++m_cursor; + if (yych == 'u') + { + goto basic_json_parser_51; + } + goto basic_json_parser_33; +basic_json_parser_42: + yych = *++m_cursor; + if (yych == 0xBF) + { + goto basic_json_parser_52; + } + goto basic_json_parser_33; +basic_json_parser_43: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_54; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_54; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_54; + } + goto basic_json_parser_33; + } +basic_json_parser_44: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= 'D') + { + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_44; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + goto basic_json_parser_14; + } +basic_json_parser_46: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych >= ':') + { + goto basic_json_parser_33; + } +basic_json_parser_47: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_47; + } + goto basic_json_parser_14; +basic_json_parser_49: + yych = *++m_cursor; + if (yych == 's') + { + goto basic_json_parser_55; + } + goto basic_json_parser_33; +basic_json_parser_50: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_56; + } + goto basic_json_parser_33; +basic_json_parser_51: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_58; + } + goto basic_json_parser_33; +basic_json_parser_52: + ++m_cursor; + { + continue; + } +basic_json_parser_54: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_60; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_60; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_60; + } + goto basic_json_parser_33; + } +basic_json_parser_55: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_61; + } + goto basic_json_parser_33; +basic_json_parser_56: + ++m_cursor; + { + last_token_type = token_type::literal_null; + break; + } +basic_json_parser_58: + ++m_cursor; + { + last_token_type = token_type::literal_true; + break; + } +basic_json_parser_60: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_63; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_63; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_63; + } + goto basic_json_parser_33; + } +basic_json_parser_61: + ++m_cursor; + { + last_token_type = token_type::literal_false; + break; + } +basic_json_parser_63: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_31; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + } + + } + + return last_token_type; + } + + /// append data from the stream to the internal buffer + void yyfill() noexcept + { + if (m_stream == nullptr or not * m_stream) + { + return; + } + + const auto offset_start = m_start - m_content; + const auto offset_marker = m_marker - m_start; + const auto offset_cursor = m_cursor - m_start; + + m_buffer.erase(0, static_cast(offset_start)); + std::string line; + assert(m_stream != nullptr); + std::getline(*m_stream, line); + m_buffer += "\n" + line; // add line with newline symbol + + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_content; + m_marker = m_start + offset_marker; + m_cursor = m_start + offset_cursor; + m_limit = m_start + m_buffer.size() - 1; + } + + /// return string representation of last read token + string_t get_token_string() const + { + assert(m_start != nullptr); + return string_t(reinterpret_cast(m_start), + static_cast(m_cursor - m_start)); + } + + /*! + @brief return string value for string tokens + + The function iterates the characters between the opening and closing + quotes of the string value. The complete string is the range + [m_start,m_cursor). Consequently, we iterate from m_start+1 to + m_cursor-1. + + We differentiate two cases: + + 1. Escaped characters. In this case, a new character is constructed + according to the nature of the escape. Some escapes create new + characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied + as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape + `"\\uxxxx"` need special care. In this case, to_unicode takes care + of the construction of the values. + 2. Unescaped characters are copied as is. + + @pre `m_cursor - m_start >= 2`, meaning the length of the last token + is at least 2 bytes which is trivially true for any string (which + consists of at least two quotes). + + " c1 c2 c3 ... " + ^ ^ + m_start m_cursor + + @complexity Linear in the length of the string.\n + + Lemma: The loop body will always terminate.\n + + Proof (by contradiction): Assume the loop body does not terminate. As + the loop body does not contain another loop, one of the called + functions must never return. The called functions are `std::strtoul` + and to_unicode. Neither function can loop forever, so the loop body + will never loop forever which contradicts the assumption that the loop + body does not terminate, q.e.d.\n + + Lemma: The loop condition for the for loop is eventually false.\n + + Proof (by contradiction): Assume the loop does not terminate. Due to + the above lemma, this can only be due to a tautological loop + condition; that is, the loop condition i < m_cursor - 1 must always be + true. Let x be the change of i for any loop iteration. Then + m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This + can be rephrased to m_cursor - m_start - 2 > x. With the + precondition, we x <= 0, meaning that the loop condition holds + indefinitly if i is always decreased. However, observe that the value + of i is strictly increasing with each iteration, as it is incremented + by 1 in the iteration expression and never decremented inside the loop + body. Hence, the loop condition will eventually be false which + contradicts the assumption that the loop condition is a tautology, + q.e.d. + + @return string value of current token without opening and closing + quotes + @throw std::out_of_range if to_unicode fails + */ + string_t get_string() const + { + assert(m_cursor - m_start >= 2); + + string_t result; + result.reserve(static_cast(m_cursor - m_start - 2)); + + // iterate the result between the quotes + for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) + { + // process escaped characters + if (*i == '\\') + { + // read next character + ++i; + + switch (*i) + { + // the default escapes + case 't': + { + result += "\t"; + break; + } + case 'b': + { + result += "\b"; + break; + } + case 'f': + { + result += "\f"; + break; + } + case 'n': + { + result += "\n"; + break; + } + case 'r': + { + result += "\r"; + break; + } + case '\\': + { + result += "\\"; + break; + } + case '/': + { + result += "/"; + break; + } + case '"': + { + result += "\""; + break; + } + + // unicode + case 'u': + { + // get code xxxx from uxxxx + auto codepoint = std::strtoul(std::string(reinterpret_cast(i + 1), + 4).c_str(), nullptr, 16); + + // check if codepoint is a high surrogate + if (codepoint >= 0xD800 and codepoint <= 0xDBFF) + { + // make sure there is a subsequent unicode + if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') + { + throw std::invalid_argument("missing low surrogate"); + } + + // get code yyyy from uxxxx\uyyyy + auto codepoint2 = std::strtoul(std::string(reinterpret_cast + (i + 7), 4).c_str(), nullptr, 16); + result += to_unicode(codepoint, codepoint2); + // skip the next 10 characters (xxxx\uyyyy) + i += 10; + } + else + { + // add unicode character(s) + result += to_unicode(codepoint); + // skip the next four characters (xxxx) + i += 4; + } + break; + } + } + } + else + { + // all other characters are just copied to the end of the + // string + result.append(1, static_cast(*i)); + } + } + + return result; + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + long double str_to_float_t(long double* /* type */, char** endptr) const + { + return std::strtold(reinterpret_cast(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + double str_to_float_t(double* /* type */, char** endptr) const + { + return std::strtod(reinterpret_cast(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + float str_to_float_t(float* /* type */, char** endptr) const + { + return std::strtof(reinterpret_cast(m_start), endptr); + } + + /*! + @brief return number value for number tokens + + This function translates the last token into the most appropriate + number type (either integer, unsigned integer or floating point), + which is passed back to the caller via the result parameter. + + This function parses the integer component up to the radix point or + exponent while collecting information about the 'floating point + representation', which it stores in the result parameter. If there is + no radix point or exponent, and the number can fit into a @ref + number_integer_t or @ref number_unsigned_t then it sets the result + parameter accordingly. + + If the number is a floating point number the number is then parsed + using @a std:strtod (or @a std:strtof or @a std::strtold). + + @param[out] result @ref basic_json object to receive the number, or + NAN if the conversion read past the current token. The latter case + needs to be treated by the caller function. + */ + void get_number(basic_json& result) const + { + assert(m_start != nullptr); + + const lexer::lexer_char_t* curptr = m_start; + + // accumulate the integer conversion result (unsigned for now) + number_unsigned_t value = 0; + + // maximum absolute value of the relevant integer type + number_unsigned_t max; + + // temporarily store the type to avoid unecessary bitfield access + value_t type; + + // look for sign + if (*curptr == '-') + { + type = value_t::number_integer; + max = static_cast((std::numeric_limits::max)()) + 1; + curptr++; + } + else + { + type = value_t::number_unsigned; + max = static_cast((std::numeric_limits::max)()); + } + + // count the significant figures + for (; curptr < m_cursor; curptr++) + { + // quickly skip tests if a digit + if (*curptr < '0' || *curptr > '9') + { + if (*curptr == '.') + { + // don't count '.' but change to float + type = value_t::number_float; + continue; + } + // assume exponent (if not then will fail parse): change to + // float, stop counting and record exponent details + type = value_t::number_float; + break; + } + + // skip if definitely not an integer + if (type != value_t::number_float) + { + // multiply last value by ten and add the new digit + auto temp = value * 10 + *curptr - '0'; + + // test for overflow + if (temp < value || temp > max) + { + // overflow + type = value_t::number_float; + } + else + { + // no overflow - save it + value = temp; + } + } + } + + // save the value (if not a float) + if (type == value_t::number_unsigned) + { + result.m_value.number_unsigned = value; + } + else if (type == value_t::number_integer) + { + result.m_value.number_integer = -static_cast(value); + } + else + { + // parse with strtod + result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); + } + + // save the type + result.m_type = type; + } + + private: + /// optional input stream + std::istream* m_stream = nullptr; + /// the buffer + string_t m_buffer; + /// the buffer pointer + const lexer_char_t* m_content = nullptr; + /// pointer to the beginning of the current symbol + const lexer_char_t* m_start = nullptr; + /// pointer for backtracking information + const lexer_char_t* m_marker = nullptr; + /// pointer to the current symbol + const lexer_char_t* m_cursor = nullptr; + /// pointer to the end of the buffer + const lexer_char_t* m_limit = nullptr; + /// the last token type + token_type last_token_type = token_type::end_of_input; + }; + + /*! + @brief syntax analysis + + This class implements a recursive decent parser. + */ + class parser + { + public: + /// constructor for strings + parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(s) + { + // read first token + get_token(); + } + + /// a parser reading from an input stream + parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(&_is) + { + // read first token + get_token(); + } + + /// public parser interface + basic_json parse() + { + basic_json result = parse_internal(true); + result.assert_invariant(); + + expect(lexer::token_type::end_of_input); + + // return parser result and replace it with null in case the + // top-level value was discarded by the callback function + return result.is_discarded() ? basic_json() : std::move(result); + } + + private: + /// the actual parser + basic_json parse_internal(bool keep) + { + auto result = basic_json(value_t::discarded); + + switch (last_token) + { + case lexer::token_type::begin_object: + { + if (keep and (not callback or (keep = callback(depth++, parse_event_t::object_start, result)))) + { + // explicitly set result to object to cope with {} + result.m_type = value_t::object; + result.m_value = value_t::object; + } + + // read next token + get_token(); + + // closing } -> we are done + if (last_token == lexer::token_type::end_object) + { + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse key-value pairs + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // store key + expect(lexer::token_type::value_string); + const auto key = m_lexer.get_string(); + + bool keep_tag = false; + if (keep) + { + if (callback) + { + basic_json k(key); + keep_tag = callback(depth, parse_event_t::key, k); + } + else + { + keep_tag = true; + } + } + + // parse separator (:) + get_token(); + expect(lexer::token_type::name_separator); + + // parse and add value + get_token(); + auto value = parse_internal(keep); + if (keep and keep_tag and not value.is_discarded()) + { + result[key] = std::move(value); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing } + expect(lexer::token_type::end_object); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::begin_array: + { + if (keep and (not callback or (keep = callback(depth++, parse_event_t::array_start, result)))) + { + // explicitly set result to object to cope with [] + result.m_type = value_t::array; + result.m_value = value_t::array; + } + + // read next token + get_token(); + + // closing ] -> we are done + if (last_token == lexer::token_type::end_array) + { + get_token(); + if (callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse values + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // parse value + auto value = parse_internal(keep); + if (keep and not value.is_discarded()) + { + result.push_back(std::move(value)); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing ] + expect(lexer::token_type::end_array); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::literal_null: + { + get_token(); + result.m_type = value_t::null; + break; + } + + case lexer::token_type::value_string: + { + const auto s = m_lexer.get_string(); + get_token(); + result = basic_json(s); + break; + } + + case lexer::token_type::literal_true: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = true; + break; + } + + case lexer::token_type::literal_false: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = false; + break; + } + + case lexer::token_type::value_number: + { + m_lexer.get_number(result); + get_token(); + break; + } + + default: + { + // the last token was unexpected + unexpect(last_token); + } + } + + if (keep and callback and not callback(depth, parse_event_t::value, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + /// get next token from lexer + typename lexer::token_type get_token() noexcept + { + last_token = m_lexer.scan(); + return last_token; + } + + void expect(typename lexer::token_type t) const + { + if (t != last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + error_msg += "; expected " + lexer::token_type_name(t); + throw std::invalid_argument(error_msg); + } + } + + void unexpect(typename lexer::token_type t) const + { + if (t == last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + throw std::invalid_argument(error_msg); + } + } + + private: + /// current level of recursion + int depth = 0; + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + typename lexer::token_type last_token = lexer::token_type::uninitialized; + /// the lexer + lexer m_lexer; + }; + + public: + /*! + @brief JSON Pointer + + A JSON pointer defines a string syntax for identifying a specific value + within a JSON document. It can be used with functions `at` and + `operator[]`. Furthermore, JSON pointers are the base for JSON patches. + + @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + class json_pointer + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /*! + @brief create JSON pointer + + Create a JSON pointer according to the syntax described in + [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). + + @param[in] s string representing the JSON pointer; if omitted, the + empty string is assumed which references the whole JSON + value + + @throw std::domain_error if reference token is nonempty and does not + begin with a slash (`/`); example: `"JSON pointer must be empty or + begin with /"` + @throw std::domain_error if a tilde (`~`) is not followed by `0` + (representing `~`) or `1` (representing `/`); example: `"escape error: + ~ must be followed with 0 or 1"` + + @liveexample{The example shows the construction several valid JSON + pointers as well as the exceptional behavior.,json_pointer} + + @since version 2.0.0 + */ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /*! + @brief return a string representation of the JSON pointer + + @invariant For each JSON pointer `ptr`, it holds: + @code {.cpp} + ptr == json_pointer(ptr.to_string()); + @endcode + + @return a string representation of the JSON pointer + + @liveexample{The example shows the result of `to_string`., + json_pointer__to_string} + + @since version 2.0.0 + */ + std::string to_string() const noexcept + { + return std::accumulate(reference_tokens.begin(), + reference_tokens.end(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + escape(b); + }); + } + + /// @copydoc to_string() + operator std::string() const + { + return to_string(); + } + + private: + /// remove and return last reference pointer + std::string pop_back() + { + if (is_root()) + { + throw std::domain_error("JSON pointer has no parent"); + } + + auto last = reference_tokens.back(); + reference_tokens.pop_back(); + return last; + } + + /// return whether pointer points to the root document + bool is_root() const + { + return reference_tokens.empty(); + } + + json_pointer top() const + { + if (is_root()) + { + throw std::domain_error("JSON pointer has no parent"); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + */ + reference get_and_create(reference j) const + { + pointer result = &j; + + // in case no reference tokens exist, return a reference to the + // JSON value j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case value_t::array: + { + // create an entry in the array + result = &result->operator[](static_cast(std::stoi(reference_token))); + break; + } + + /* + The following code is only reached if there exists a + reference token _and_ the current value is primitive. In + this case, we have an error situation, because primitive + values may only occur as single value; that is, with an + empty list of reference tokens. + */ + default: + { + throw std::domain_error("invalid value to unflatten"); + } + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + */ + reference get_unchecked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + if (reference_token == "-") + { + // explicityly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + reference get_checked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + */ + const_reference get_unchecked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" cannot be used for const access + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // use unchecked array access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + const_reference get_checked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /// split the string input to reference tokens + static std::vector split(std::string reference_string) + { + std::vector result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (reference_string[0] != '/') + { + throw std::domain_error("JSON pointer must be empty or begin with '/'"); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + size_t slash = reference_string.find_first_of("/", 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == string::npos+1 = 0 + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = slash + 1, + // find next slash + slash = reference_string.find_first_of("/", start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (size_t pos = reference_token.find_first_of("~"); + pos != std::string::npos; + pos = reference_token.find_first_of("~", pos + 1)) + { + assert(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (pos == reference_token.size() - 1 or + (reference_token[pos + 1] != '0' and + reference_token[pos + 1] != '1')) + { + throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); + } + } + + // finally, store the reference token + unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @brief replace all occurrences of a substring by another string + + @param[in,out] s the string to manipulate + @param[in] f the substring to replace with @a t + @param[in] t the string to replace @a f + + @return The string @a s where all occurrences of @a f are replaced + with @a t. + + @pre The search string @a f must not be empty. + + @since version 2.0.0 + */ + static void replace_substring(std::string& s, + const std::string& f, + const std::string& t) + { + assert(not f.empty()); + + for ( + size_t pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t + pos = s.find(f, pos + t.size()) // find next occurrence of f + ); + } + + /// escape tilde and slash + static std::string escape(std::string s) + { + // escape "~"" to "~0" and "/" to "~1" + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; + } + + /// unescape tilde and slash + static void unescape(std::string& s) + { + // first transform any occurrence of the sequence '~1' to '/' + replace_substring(s, "~1", "/"); + // then transform any occurrence of the sequence '~0' to '~' + replace_substring(s, "~0", "~"); + } + + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const basic_json& value, + basic_json& result) + { + switch (value.m_type) + { + case value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), + element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + */ + static basic_json unflatten(const basic_json& value) + { + if (not value.is_object()) + { + throw std::domain_error("only objects can be unflattened"); + } + + basic_json result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (not element.second.is_primitive()) + { + throw std::domain_error("values in object must be primitive"); + } + + // assign value to reference pointed to by JSON pointer; Note + // that if the JSON pointer is "" (i.e., points to the whole + // value), function get_and_create returns a reference to + // result itself. An assignment will then create a primitive + // value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + private: + /// the reference tokens + std::vector reference_tokens {}; + }; + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. Similar to @ref operator[](const typename + object_t::key_type&), `null` values are created in arrays and objects if + necessary. + + In particular: + - If the JSON pointer points to an object key that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. + - If the JSON pointer points to an array index that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. All indices between the current maximum and the given + index are also filled with `null`. + - The special value `-` is treated as a synonym for the index past the + end. + + @param[in] ptr a JSON pointer + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer} + + @since version 2.0.0 + */ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. The function does not change the JSON + value; no `null` values are created. In particular, the the special value + `-` yields an exception. + + @param[in] ptr JSON pointer to the desired element + + @return const reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} + + @since version 2.0.0 + */ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a reference to the element at with specified JSON pointer @a ptr, + with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer} + + @since version 2.0.0 + */ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a const reference to the element at with specified JSON pointer @a + ptr, with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} + + @since version 2.0.0 + */ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /*! + @brief return flattened JSON value + + The function creates a JSON object whose keys are JSON pointers (see [RFC + 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all + primitive. The original JSON value can be restored using the @ref + unflatten() function. + + @return an object that maps JSON pointers to primitve values + + @note Empty objects and arrays are flattened to `null` and will not be + reconstructed correctly by the @ref unflatten() function. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a JSON object is flattened to an + object whose keys consist of JSON pointers.,flatten} + + @sa @ref unflatten() for the reverse function + + @since version 2.0.0 + */ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /*! + @brief unflatten a previously flattened JSON value + + The function restores the arbitrary nesting of a JSON value that has been + flattened before using the @ref flatten() function. The JSON value must + meet certain constraints: + 1. The value must be an object. + 2. The keys must be JSON pointers (see + [RFC 6901](https://tools.ietf.org/html/rfc6901)) + 3. The mapped values must be primitive JSON types. + + @return the original JSON from a flattened version + + @note Empty objects and arrays are flattened by @ref flatten() to `null` + values and can not unflattened to their original type. Apart from + this example, for a JSON value `j`, the following is always true: + `j == j.flatten().unflatten()`. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a flattened JSON object is + unflattened into the original nested JSON object.,unflatten} + + @sa @ref flatten() for the reverse function + + @since version 2.0.0 + */ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /*! + @brief applies a JSON patch + + [JSON Patch](http://jsonpatch.com) defines a JSON document structure for + expressing a sequence of operations to apply to a JSON) document. With + this funcion, a JSON Patch is applied to the current JSON value by + executing all operations from the patch. + + @param[in] json_patch JSON patch document + @return patched document + + @note The application of a patch is atomic: Either all operations succeed + and the patched document is returned or an exception is thrown. In + any case, the original value is not changed: the patch is applied + to a copy of the value. + + @throw std::out_of_range if a JSON pointer inside the patch could not + be resolved successfully in the current JSON value; example: `"key baz + not found"` + @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + attributes are missing); example: `"operation add must have member path"` + + @complexity Linear in the size of the JSON value and the length of the + JSON patch. As usually only a fraction of the JSON value is affected by + the patch, the complexity can usually be neglected. + + @liveexample{The following code shows how a JSON patch is applied to a + value.,patch} + + @sa @ref diff -- create a JSON patch by comparing two JSON values + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.is_root()) + { + result = val; + } + else + { + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + basic_json& x = result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = std::stoi(last_path); + if (static_cast(idx) > parent.size()) + { + // avoid undefined behavior + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + else + { + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); + } + } + break; + } + + default: + { + // if there exists a parent it cannot be primitive + assert(false); // LCOV_EXCL_LINE + } + } + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [&result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (it != parent.end()) + { + parent.erase(it); + } + else + { + throw std::out_of_range("key '" + last_path + "' not found"); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(static_cast(std::stoi(last_path))); + } + }; + + // type check + if (not json_patch.is_array()) + { + // a JSON patch must be an array of objects + throw std::invalid_argument("JSON patch must be an array of objects"); + } + + // iterate and apply th eoperations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json& + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (it == val.m_value.object->end()) + { + throw std::invalid_argument(error_msg + " must have member '" + member + "'"); + } + + // check if result is of type string + if (string_type and not it->second.is_string()) + { + throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); + } + + // no error: return value + return it->second; + }; + + // type check + if (not val.is_object()) + { + throw std::invalid_argument("JSON patch must be an array of objects"); + } + + // collect mandatory members + const std::string op = get_value("op", "op", true); + const std::string path = get_value(op, "path", true); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const std::string from_path = get_value("move", "from", true); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const std::string from_path = get_value("copy", "from", true);; + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + result[ptr] = result.at(from_ptr); + break; + } + + case patch_operations::test: + { + bool success = false; + try + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + catch (std::out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (not success) + { + throw std::domain_error("unsuccessful: " + val.dump()); + } + + break; + } + + case patch_operations::invalid: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + throw std::invalid_argument("operation value '" + op + "' is invalid"); + } + } + } + + return result; + } + + /*! + @brief creates a diff as a JSON patch + + Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can + be changed into the value @a target by calling @ref patch function. + + @invariant For two JSON values @a source and @a target, the following code + yields always `true`: + @code {.cpp} + source.patch(diff(source, target)) == target; + @endcode + + @note Currently, only `remove`, `add`, and `replace` operations are + generated. + + @param[in] source JSON value to copare from + @param[in] target JSON value to copare against + @param[in] path helper value to create JSON pointers + + @return a JSON patch to convert the @a source to @a target + + @complexity Linear in the lengths of @a source and @a target. + + @liveexample{The following code shows how a JSON patch is created as a + diff for two JSON values.,diff} + + @sa @ref patch -- apply a JSON patch + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + + @since version 2.0.0 + */ + static basic_json diff(const basic_json& source, + const basic_json& target, + std::string path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + } + else + { + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + size_t i = 0; + while (i < source.size() and i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.begin(); it != source.end(); ++it) + { + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, + {"path", path + "/" + key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.begin(); it != target.end(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + key}, + {"value", it.value()} + }); + } + } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + break; + } + } + } + + return result; + } + + /// @} +}; + + +///////////// +// presets // +///////////// + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} + + +/////////////////////// +// nonmember support // +/////////////////////// + +// specialization of std::swap, and std::hash +namespace std +{ +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template <> +inline void swap(nlohmann::json& j1, + nlohmann::json& j2) noexcept( + is_nothrow_move_constructible::value and + is_nothrow_move_assignable::value + ) +{ + j1.swap(j2); +} + +/// hash value for JSON objects +template <> +struct hash +{ + /*! + @brief return a hash value for a JSON object + + @since version 1.0.0 + */ + std::size_t operator()(const nlohmann::json& j) const + { + // a naive hashing via the string representation + const auto& h = hash(); + return h(j.dump()); + } +}; +} + +/*! +@brief user-defined string literal for JSON values + +This operator implements a user-defined string literal for JSON objects. It +can be used by adding `"_json"` to a string literal and returns a JSON object +if no parse error occurred. + +@param[in] s a string representation of a JSON object +@return a JSON object + +@since version 1.0.0 +*/ +inline nlohmann::json operator "" _json(const char* s, std::size_t) +{ + return nlohmann::json::parse(reinterpret_cast(s)); +} + +/*! +@brief user-defined string literal for JSON pointer + +This operator implements a user-defined string literal for JSON Pointers. It +can be used by adding `"_json"` to a string literal and returns a JSON pointer +object if no parse error occurred. + +@param[in] s a string representation of a JSON Pointer +@return a JSON pointer object + +@since version 2.0.0 +*/ +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t) +{ + return nlohmann::json::json_pointer(s); +} + +// restore GCC/clang diagnostic settings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic pop +#endif + +#endif diff --git a/ext/offbase/README.md b/ext/offbase/README.md deleted file mode 100644 index 6668b878..00000000 --- a/ext/offbase/README.md +++ /dev/null @@ -1,59 +0,0 @@ -A super-minimal in-filesystem persistent JSON object -====== - -[We](https://www.zerotier.com/) like minimalism. - -Offbase is an extension of the excellent [nlohmann/json](https://github.com/nlohmann/json) C++11 JSON class that adds simple object persistence to/from the filesystem. Objects are stored into a directory hierarchy in fully "expanded" form with each field/value being represented by a separate file. - -Features: - - - Very easy to use - - Minimal! - - Easy to understand and maintain - - Trivial to implement in other languages - - No dependencies beyond standard libraries - - Small code footprint in both source and binary form - - Easy to port to other platforms - - Exactly reproduces JSON object hierarchies including all JSON type information - - Database can be explored from the shell, browsed in a web browser or file explorer, scanned with `find` and `grep`, etc. - - Database can be backed up, restored, versioned, etc. with tools like `git`, `rsync`, `duplicity`, etc. - - Alien files like `.git` or `.DS_Store` are harmlessly ignored if present - - Saving only changes what's changed to reduce I/O overhead and SSD wear - -Limitations and shortcomings: - - - This creates a lot of tiny files, which is inefficient on some filesystems and might run into inode limits in extreme cases. For data sets with more than, say, a million items we recommend a filesystem like `btrfs` or `reiserfs`. Things like [redisfs](https://steve.fi/Software/redisfs/) are also worth exploring. On Linux another alternative is to put the database into `/dev/shm` (RAM disk) and then regularly back it up with `duplicity` or similar. - - The whole JSON object is held in memory *twice* for diffing purposes. - - Diffing traverses the whole tree and then updates the shadow copy, which makes `commit()` slow for huge data sets. This is not suitable for "big" data where "big" here is probably more than a few hundred megabytes. - - Recursion is used, so if you have object hierarchies that are incredibly deep (hundreds or more) it might be possible to overflow your stack and crash your app. - - This is not thread safe and must be guarded by a mutex if used in a threaded app. - -Caveats: - - - Key names are escaped for safety in the filesystem, but we still don't recommend allowing external users to set just anything into your JSON store. See the point about recursion under limitations. - -Future: - - - It would not be too hard to tie this into a filesystem change monitoring API and automatically read changes from disk if they are detected. This would allow the database to be edited "live" in the filesystem. - - In theory this could provide replication or clustering through distributed filesystems, file syncing, or things like [Amazon Elastic Filesystem](https://aws.amazon.com/efs/). - - Recursion could be factored out to get rid of any object hierarchy depth constraints. - - Mutexes could be integrated somehow to allow for finer grained locking in multithreaded apps. - - Diffing and selective updates could be made more memory and CPU efficient using hashes, etc. - -## How to Use - -The `offbase` class just extends [nlohmann::json](https://github.com/nlohmann/json) and gives you a JSON object. Take care to make sure you don't change the type of the 'root' object represented by the 'offbase' instance from JSON 'object'. Anything under it can of course be any JSON type, including any object. - -Just put data into the object and then periodically call `commit()` to persist changes to disk. The `commit()` method diffs the current contents of the object with what it knows to have been previously persisted to disk and modifies the representation on disk to match. This can be done after writes or periodically in a background thread. - -See comments in `offbase.hpp` for full documentation including details about error handling, etc. - -## Persistence format - -The base object represented by the `offbase` instance is persisted into a directory hierarchy under its base path. Files and directories are named according to a simple convention of `keyname.typecode` where `keyname` is an escaped key name (or hex array index in the case of arrays) and `typecode` is a single character indicating whether the item is a JSON value, array, or object. - - - `*.V`: JSON values (actual value type is inferred during JSON parse) - - `*.O`: JSON objects (these are subdirectories) - - `*.A`: JSON arrays (also subdirectories containing items by hex array index) - -There are in theory simpler ways to represent JSON in a filesystem, such as the "flattened" JSON "pointer" format, but this has the disadvantage of not disambiguating objects vs. arrays. Offbase's persistence format is designed to perfectly reproduce the exact same JSON tree on load as was most recently committed. diff --git a/ext/offbase/json/LICENSE.MIT b/ext/offbase/json/LICENSE.MIT deleted file mode 100644 index e2ac4891..00000000 --- a/ext/offbase/json/LICENSE.MIT +++ /dev/null @@ -1,22 +0,0 @@ -The library is licensed under the MIT License -: - -Copyright (c) 2013-2016 Niels Lohmann - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/ext/offbase/json/README.md b/ext/offbase/json/README.md deleted file mode 100644 index c0bb61b1..00000000 --- a/ext/offbase/json/README.md +++ /dev/null @@ -1,511 +0,0 @@ -[![JSON for Modern C++](https://raw.githubusercontent.com/nlohmann/json/master/doc/json.gif)](https://github.com/nlohmann/json/releases) - -[![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) -[![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk?svg=true)](https://ci.appveyor.com/project/nlohmann/json) -[![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) -[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) -[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/p5o4znPnGHJpDVqN) -[![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) -[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) -[![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) -[![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) - -## Design goals - -There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: - -- **Intuitive syntax**. In languages such as Python, JSON feels like a first class data type. We used all the operator magic of modern C++ to achieve the same feeling in your code. Check out the [examples below](#examples) and you know, what I mean. - -- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/src/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. - -- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/blob/master/test/src/unit.cpp) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) that there are no memory leaks. - -Other aspects were not so important to us: - -- **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. - -- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). - -See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. - - -## Integration - -The single required source, file `json.hpp` is in the `src` directory or [released here](https://github.com/nlohmann/json/releases). All you need to do is add - -```cpp -#include "json.hpp" - -// for convenience -using json = nlohmann::json; -``` - -to the files you want to use JSON objects. That's it. Do not forget to set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). - -:beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. - - -## Examples - -Here are some examples to give you an idea how to use the class. - -Assume you want to create the JSON object - -```json -{ - "pi": 3.141, - "happy": true, - "name": "Niels", - "nothing": null, - "answer": { - "everything": 42 - }, - "list": [1, 0, 2], - "object": { - "currency": "USD", - "value": 42.99 - } -} -``` - -With the JSON class, you could write: - -```cpp -// create an empty structure (null) -json j; - -// add a number that is stored as double (note the implicit conversion of j to an object) -j["pi"] = 3.141; - -// add a Boolean that is stored as bool -j["happy"] = true; - -// add a string that is stored as std::string -j["name"] = "Niels"; - -// add another null object by passing nullptr -j["nothing"] = nullptr; - -// add an object inside the object -j["answer"]["everything"] = 42; - -// add an array that is stored as std::vector (using an initializer list) -j["list"] = { 1, 0, 2 }; - -// add another object (using an initializer list of pairs) -j["object"] = { {"currency", "USD"}, {"value", 42.99} }; - -// instead, you could also write (which looks very similar to the JSON above) -json j2 = { - {"pi", 3.141}, - {"happy", true}, - {"name", "Niels"}, - {"nothing", nullptr}, - {"answer", { - {"everything", 42} - }}, - {"list", {1, 0, 2}}, - {"object", { - {"currency", "USD"}, - {"value", 42.99} - }} -}; -``` - -Note that in all these cases, you never need to "tell" the compiler which JSON value you want to use. If you want to be explicit or express some edge cases, the functions `json::array` and `json::object` will help: - -```cpp -// a way to express the empty array [] -json empty_array_explicit = json::array(); - -// ways to express the empty object {} -json empty_object_implicit = json({}); -json empty_object_explicit = json::object(); - -// a way to express an _array_ of key/value pairs [["currency", "USD"], ["value", 42.99]] -json array_not_object = { json::array({"currency", "USD"}), json::array({"value", 42.99}) }; -``` - - -### Serialization / Deserialization - -You can create an object (deserialization) by appending `_json` to a string literal: - -```cpp -// create object from string literal -json j = "{ \"happy\": true, \"pi\": 3.141 }"_json; - -// or even nicer with a raw string literal -auto j2 = R"( - { - "happy": true, - "pi": 3.141 - } -)"_json; - -// or explicitly -auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); -``` - -You can also get a string representation (serialize): - -```cpp -// explicit conversion to string -std::string s = j.dump(); // {\"happy\":true,\"pi\":3.141} - -// serialization with pretty printing -// pass in the amount of spaces to indent -std::cout << j.dump(4) << std::endl; -// { -// "happy": true, -// "pi": 3.141 -// } -``` - -You can also use streams to serialize and deserialize: - -```cpp -// deserialize from standard input -json j; -std::cin >> j; - -// serialize to standard output -std::cout << j; - -// the setw manipulator was overloaded to set the indentation for pretty printing -std::cout << std::setw(4) << j << std::endl; -``` - -These operators work for any subclasses of `std::istream` or `std::ostream`. - -Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. - - -### STL-like access - -We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) requirement. - -```cpp -// create an array using push_back -json j; -j.push_back("foo"); -j.push_back(1); -j.push_back(true); - -// iterate the array -for (json::iterator it = j.begin(); it != j.end(); ++it) { - std::cout << *it << '\n'; -} - -// range-based for -for (auto& element : j) { - std::cout << element << '\n'; -} - -// getter/setter -const std::string tmp = j[0]; -j[1] = 42; -bool foo = j.at(2); - -// other stuff -j.size(); // 3 entries -j.empty(); // false -j.type(); // json::value_t::array -j.clear(); // the array is empty again - -// convenience type checkers -j.is_null(); -j.is_boolean(); -j.is_number(); -j.is_object(); -j.is_array(); -j.is_string(); - -// comparison -j == "[\"foo\", 1, true]"_json; // true - -// create an object -json o; -o["foo"] = 23; -o["bar"] = false; -o["baz"] = 3.141; - -// special iterator member functions for objects -for (json::iterator it = o.begin(); it != o.end(); ++it) { - std::cout << it.key() << " : " << it.value() << "\n"; -} - -// find an entry -if (o.find("foo") != o.end()) { - // there is an entry with key "foo" -} - -// or simpler using count() -int foo_present = o.count("foo"); // 1 -int fob_present = o.count("fob"); // 0 - -// delete an entry -o.erase("foo"); -``` - - -### Conversion from STL containers - -Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON types (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends how the elements are ordered in the respective STL container. - -```cpp -std::vector c_vector {1, 2, 3, 4}; -json j_vec(c_vector); -// [1, 2, 3, 4] - -std::deque c_deque {1.2, 2.3, 3.4, 5.6}; -json j_deque(c_deque); -// [1.2, 2.3, 3.4, 5.6] - -std::list c_list {true, true, false, true}; -json j_list(c_list); -// [true, true, false, true] - -std::forward_list c_flist {12345678909876, 23456789098765, 34567890987654, 45678909876543}; -json j_flist(c_flist); -// [12345678909876, 23456789098765, 34567890987654, 45678909876543] - -std::array c_array {{1, 2, 3, 4}}; -json j_array(c_array); -// [1, 2, 3, 4] - -std::set c_set {"one", "two", "three", "four", "one"}; -json j_set(c_set); // only one entry for "one" is used -// ["four", "one", "three", "two"] - -std::unordered_set c_uset {"one", "two", "three", "four", "one"}; -json j_uset(c_uset); // only one entry for "one" is used -// maybe ["two", "three", "four", "one"] - -std::multiset c_mset {"one", "two", "one", "four"}; -json j_mset(c_mset); // only one entry for "one" is used -// maybe ["one", "two", "four"] - -std::unordered_multiset c_umset {"one", "two", "one", "four"}; -json j_umset(c_umset); // both entries for "one" are used -// maybe ["one", "two", "one", "four"] -``` - -Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON types (see examples above) can be used to to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. - -```cpp -std::map c_map { {"one", 1}, {"two", 2}, {"three", 3} }; -json j_map(c_map); -// {"one": 1, "three": 3, "two": 2 } - -std::unordered_map c_umap { {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; -json j_umap(c_umap); -// {"one": 1.2, "two": 2.3, "three": 3.4} - -std::multimap c_mmap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; -json j_mmap(c_mmap); // only one entry for key "three" is used -// maybe {"one": true, "two": true, "three": true} - -std::unordered_multimap c_ummap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; -json j_ummap(c_ummap); // only one entry for key "three" is used -// maybe {"one": true, "two": true, "three": true} -``` - -### JSON Pointer and JSON Patch - -The library supports **JSON Pointer** ([RFC 6901](https://tools.ietf.org/html/rfc6901)) as alternative means to address structured values. On top of this, **JSON Patch** ([RFC 6902](https://tools.ietf.org/html/rfc6902)) allows to describe differences between two JSON values - effectively allowing patch and diff operations known from Unix. - -```cpp -// a JSON value -json j_original = R"({ - "baz": ["one", "two", "three"], - "foo": "bar" -})"_json; - -// access members with a JSON pointer (RFC 6901) -j_original["/baz/1"_json_pointer]; -// "two" - -// a JSON patch (RFC 6902) -json j_patch = R"([ - { "op": "replace", "path": "/baz", "value": "boo" }, - { "op": "add", "path": "/hello", "value": ["world"] }, - { "op": "remove", "path": "/foo"} -])"_json; - -// apply the patch -json j_result = j_original.patch(j_patch); -// { -// "baz": "boo", -// "hello": ["world"] -// } - -// calculate a JSON patch from two JSON values -json::diff(j_result, j_original); -// [ -// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, -// { "op": "remove","path": "/hello" }, -// { "op": "add", "path": "/foo", "value": "bar" } -// ] -``` - - -### Implicit conversions - -The type of the JSON object is determined automatically by the expression to store. Likewise, the stored value is implicitly converted. - -```cpp -// strings -std::string s1 = "Hello, world!"; -json js = s1; -std::string s2 = js; - -// Booleans -bool b1 = true; -json jb = b1; -bool b2 = jb; - -// numbers -int i = 42; -json jn = i; -double f = jn; - -// etc. -``` - -You can also explicitly ask for the value: - -```cpp -std::string vs = js.get(); -bool vb = jb.get(); -int vi = jn.get(); - -// etc. -``` - - -## Supported compilers - -Though it's 2016 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: - -- GCC 4.9 - 6.0 (and possibly later) -- Clang 3.4 - 3.9 (and possibly later) -- Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) - -I would be happy to learn about other compilers/versions. - -Please note: - -- GCC 4.8 does not work because of two bugs ([55817](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55817) and [57824](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57824)) in the C++11 support. Note there is a [pull request](https://github.com/nlohmann/json/pull/212) to fix some of the issues. -- Android defaults to using very old compilers and C++ libraries. To fix this, add the following to your `Application.mk`. This will switch to the LLVM C++ library, the Clang compiler, and enable C++11 and other features disabled by default. - - ``` - APP_STL := c++_shared - NDK_TOOLCHAIN_VERSION := clang3.6 - APP_CPPFLAGS += -frtti -fexceptions - ``` - - The code compiles successfully with [Android NDK](https://developer.android.com/ndk/index.html?hl=ml), Revision 9 - 11 (and possibly later) and [CrystaX's Android NDK](https://www.crystax.net/en/android/ndk) version 10. - -- For GCC running on MinGW or Android SDK, the error `'to_string' is not a member of 'std'` (or similarly, for `strtod`) may occur. Note this is not an issue with the code, but rather with the compiler itself. On Android, see above to build with a newer environment. For MinGW, please refer to [this site](http://tehsausage.com/mingw-to-string) and [this discussion](https://github.com/nlohmann/json/issues/136) for information on how to fix this bug. For Android NDK using `APP_STL := gnustl_static`, please refer to [this discussion](https://github.com/nlohmann/json/issues/219). - -The following compilers are currently used in continuous integration at [Travis](https://travis-ci.org/nlohmann/json) and [AppVeyor](https://ci.appveyor.com/project/nlohmann/json): - -| Compiler | Operating System | Version String | -|-----------------|------------------------------|----------------| -| GCC 4.9.3 | Ubuntu 14.04.4 LTS | g++-4.9 (Ubuntu 4.9.3-8ubuntu2~14.04) 4.9.3 | -| GCC 5.3.0 | Ubuntu 14.04.4 LTS | g++-5 (Ubuntu 5.3.0-3ubuntu1~14.04) 5.3.0 20151204 | -| GCC 6.1.1 | Ubuntu 14.04.4 LTS | g++-6 (Ubuntu 6.1.1-3ubuntu11~14.04.1) 6.1.1 20160511 | -| Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | -| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | -| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | -| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | -| Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | -| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | -| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | -| Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | -| Clang Xcode 8.0 | Darwin Kernel Version 15.5.0 (OSX 10.11.5) | Apple LLVM version 8.0.0 (clang-800.0.24.1) | -| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | - - -## License - - - -The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): - -Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -## Thanks - -I deeply appreciate the help of the following people. - -- [Teemperor](https://github.com/Teemperor) implemented CMake support and lcov integration, realized escape and Unicode handling in the string parser, and fixed the JSON serialization. -- [elliotgoodrich](https://github.com/elliotgoodrich) fixed an issue with double deletion in the iterator classes. -- [kirkshoop](https://github.com/kirkshoop) made the iterators of the class composable to other libraries. -- [wancw](https://github.com/wanwc) fixed a bug that hindered the class to compile with Clang. -- Tomas Åblad found a bug in the iterator implementation. -- [Joshua C. Randall](https://github.com/jrandall) fixed a bug in the floating-point serialization. -- [Aaron Burghardt](https://github.com/aburgh) implemented code to parse streams incrementally. Furthermore, he greatly improved the parser class by allowing the definition of a filter function to discard undesired elements while parsing. -- [Daniel Kopeček](https://github.com/dkopecek) fixed a bug in the compilation with GCC 5.0. -- [Florian Weber](https://github.com/Florianjw) fixed a bug in and improved the performance of the comparison operators. -- [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. -- [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. -- [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. -- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. -- [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. -- [dariomt](https://github.com/dariomt) fixed some typos in the examples. -- [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. -- [Colin Hirsch](https://github.com/ColinH) took care of a small namespace issue. -- [Huu Nguyen](https://github.com/whoshuu) correct a variable name in the documentation. -- [Silverweed](https://github.com/silverweed) overloaded `parse()` to accept an rvalue reference. -- [dariomt](https://github.com/dariomt) fixed a subtlety in MSVC type support and implemented the `get_ref()` function to get a reference to stored values. -- [ZahlGraf](https://github.com/ZahlGraf) added a workaround that allows compilation using Android NDK. -- [whackashoe](https://github.com/whackashoe) replaced a function that was marked as unsafe by Visual Studio. -- [406345](https://github.com/406345) fixed two small warnings. -- [Glen Fernandes](https://github.com/glenfe) noted a potential portability problem in the `has_mapped_type` function. -- [Corbin Hughes](https://github.com/nibroc) fixed some typos in the contribution guidelines. -- [twelsby](https://github.com/twelsby) fixed the array subscript operator, an issue that failed the MSVC build, and floating-point parsing/dumping. He further added support for unsigned integer numbers and implemented better roundtrip support for parsed numbers. -- [Volker Diels-Grabsch](https://github.com/vog) fixed a link in the README file. -- [msm-](https://github.com/msm-) added support for american fuzzy lop. -- [Annihil](https://github.com/Annihil) fixed an example in the README file. -- [Themercee](https://github.com/Themercee) noted a wrong URL in the README file. -- [Lv Zheng](https://github.com/lv-zheng) fixed a namespace issue with `int64_t` and `uint64_t`. -- [abc100m](https://github.com/abc100m) analyzed the issues with GCC 4.8 and proposed a [partial solution](https://github.com/nlohmann/json/pull/212). -- [zewt](https://github.com/zewt) added useful notes to the README file about Android. -- [Róbert Márki](https://github.com/robertmrk) added a fix to use move iterators and improved the integration via CMake. -- [Chris Kitching](https://github.com/ChrisKitching) cleaned up the CMake files. -- [Tom Needham](https://github.com/06needhamt) fixed a subtle bug with MSVC 2015 which was also proposed by [Michael K.](https://github.com/Epidal). -- [Mário Feroldi](https://github.com/thelostt) fixed a small typo. -- [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. -- [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. - -Thanks a lot for helping out! - - -## Notes - -- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). -- As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions. - - -## Execute unit tests - -To compile and run the tests, you need to execute - -```sh -$ make -$ ./json_unit "*" - -=============================================================================== -All tests passed (8905012 assertions in 32 test cases) -``` - -For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). diff --git a/ext/offbase/json/json.hpp b/ext/offbase/json/json.hpp deleted file mode 100644 index 878fb899..00000000 --- a/ext/offbase/json/json.hpp +++ /dev/null @@ -1,10435 +0,0 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 2.0.2 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -Copyright (c) 2013-2016 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// exclude unsupported compilers -#if defined(__clang__) - #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) - #if CLANG_VERSION < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#elif defined(__GNUC__) - #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - #if GCC_VERSION < 40900 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ - - -/*! -@brief unnamed namespace with internal helper functions -@since version 1.0.0 -*/ -namespace -{ -/*! -@brief Helper to determine whether there's a key_type for T. - -Thus helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. - -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0 -*/ -template -struct has_mapped_type -{ - private: - template static char test(typename C::mapped_type*); - template static char (&test(...))[2]; - public: - static constexpr bool value = sizeof(test(0)) == 1; -}; - -/*! -@brief helper class to create locales with decimal point - -This struct is used a default locale during the JSON serialization. JSON -requires the decimal point to be `.`, so this function overloads the -`do_decimal_point()` function to return `.`. This function is called by -float-to-string conversions to retrieve the decimal separator between integer -and fractional parts. - -@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 -@since version 2.0.0 -*/ -struct DecimalSeparator : std::numpunct -{ - char do_decimal_point() const - { - return '.'; - } -}; - -} - -/*! -@brief a class to store JSON values - -@tparam ObjectType type for JSON objects (`std::map` by default; will be used -in @ref object_t) -@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used -in @ref array_t) -@tparam StringType type for JSON strings and object keys (`std::string` by -default; will be used in @ref string_t) -@tparam BooleanType type for JSON booleans (`bool` by default; will be used -in @ref boolean_t) -@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by -default; will be used in @ref number_integer_t) -@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c -`uint64_t` by default; will be used in @ref number_unsigned_t) -@tparam NumberFloatType type for JSON floating-point numbers (`double` by -default; will be used in @ref number_float_t) -@tparam AllocatorType type of the allocator to use (`std::allocator` by -default) - -@requirement The class satisfies the following concept requirements: -- Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): - JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): - A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): - A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): - A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): - A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): - JSON values can be destructed. -- Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): - JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): - All non-static data members are private and standard layout types, the class - has no virtual functions or (virtual) base classes. -- Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): - JSON values can be compared with `==`, see @ref - operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): - JSON values can be compared with `<`, see @ref - operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): - Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of - other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): - JSON values can be compared against `std::nullptr_t` objects which are used - to model the `null` value. -- Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): - JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); - JSON values can be used like STL containers and provide reverse iterator - access. - -@invariant The member variables @a m_value and @a m_type have the following -relationship: -- If `m_type == value_t::object`, then `m_value.object != nullptr`. -- If `m_type == value_t::array`, then `m_value.array != nullptr`. -- If `m_type == value_t::string`, then `m_value.string != nullptr`. -The invariants are checked by member function assert_invariant(). - -@internal -@note ObjectType trick from http://stackoverflow.com/a/9860911 -@endinternal - -@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange -Format](http://rfc7159.net/rfc7159) - -@since version 1.0.0 - -@nosubgrouping -*/ -template < - template class ObjectType = std::map, - template class ArrayType = std::vector, - class StringType = std::string, - class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator - > -class basic_json -{ - private: - /// workaround type for MSVC - using basic_json_t = basic_json; - - public: - // forward declarations - template class json_reverse_iterator; - class json_pointer; - - ///////////////////// - // container types // - ///////////////////// - - /// @name container types - /// The canonic container types to use @ref basic_json like any other STL - /// container. - /// @{ - - /// the type of elements in a basic_json container - using value_type = basic_json; - - /// the type of an element reference - using reference = value_type&; - /// the type of an element const reference - using const_reference = const value_type&; - - /// a type to represent differences between iterators - using difference_type = std::ptrdiff_t; - /// a type to represent container sizes - using size_type = std::size_t; - - /// the allocator type - using allocator_type = AllocatorType; - - /// the type of an element pointer - using pointer = typename std::allocator_traits::pointer; - /// the type of an element const pointer - using const_pointer = typename std::allocator_traits::const_pointer; - - /// an iterator for a basic_json container - class iterator; - /// a const iterator for a basic_json container - class const_iterator; - /// a reverse iterator for a basic_json container - using reverse_iterator = json_reverse_iterator; - /// a const reverse iterator for a basic_json container - using const_reverse_iterator = json_reverse_iterator; - - /// @} - - - /*! - @brief returns the allocator associated with the container - */ - static allocator_type get_allocator() - { - return allocator_type(); - } - - - /////////////////////////// - // JSON value data types // - /////////////////////////// - - /// @name JSON value data types - /// The data types to store a JSON value. These types are derived from - /// the template arguments passed to class @ref basic_json. - /// @{ - - /*! - @brief a type for an object - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: - > An object is an unordered collection of zero or more name/value pairs, - > where a name is a string and a value is a string, number, boolean, null, - > object, or array. - - To store objects in C++, a type is defined by the template parameters - described below. - - @tparam ObjectType the container to store objects (e.g., `std::map` or - `std::unordered_map`) - @tparam StringType the type of the keys or names (e.g., `std::string`). - The comparison function `std::less` is used to order elements - inside the container. - @tparam AllocatorType the allocator to use for objects (e.g., - `std::allocator`) - - #### Default type - - With the default values for @a ObjectType (`std::map`), @a StringType - (`std::string`), and @a AllocatorType (`std::allocator`), the default - value for @a object_t is: - - @code {.cpp} - std::map< - std::string, // key_type - basic_json, // value_type - std::less, // key_compare - std::allocator> // allocator_type - > - @endcode - - #### Behavior - - The choice of @a object_t influences the behavior of the JSON class. With - the default type, objects have the following behavior: - - - When all names are unique, objects will be interoperable in the sense - that all software implementations receiving that object will agree on - the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. - - Internally, name/value pairs are stored in lexicographical order of the - names. Objects will also be serialized (see @ref dump) in this order. - For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored - and serialized as `{"a": 2, "b": 1}`. - - When comparing objects, the order of the name/value pairs is irrelevant. - This makes objects interoperable in the sense that they will not be - affected by these differences. For instance, `{"b": 1, "a": 2}` and - `{"a": 2, "b": 1}` will be treated as equal. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the object's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON object. - - #### Storage - - Objects are stored as pointers in a @ref basic_json type. That is, for any - access to object values, a pointer of type `object_t*` must be - dereferenced. - - @sa @ref array_t -- type for an array value - - @since version 1.0.0 - - @note The order name/value pairs are added to the object is *not* - preserved by the library. Therefore, iterating an object may return - name/value pairs in a different order than they were originally stored. In - fact, keys will be traversed in alphabetical order as `std::map` with - `std::less` is used by default. Please note this behavior conforms to [RFC - 7159](http://rfc7159.net/rfc7159), because any order implements the - specified "unordered" nature of JSON objects. - */ - using object_t = ObjectType, - AllocatorType>>; - - /*! - @brief a type for an array - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: - > An array is an ordered sequence of zero or more values. - - To store objects in C++, a type is defined by the template parameters - explained below. - - @tparam ArrayType container type to store arrays (e.g., `std::vector` or - `std::list`) - @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) - - #### Default type - - With the default values for @a ArrayType (`std::vector`) and @a - AllocatorType (`std::allocator`), the default value for @a array_t is: - - @code {.cpp} - std::vector< - basic_json, // value_type - std::allocator // allocator_type - > - @endcode - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the array's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON array. - - #### Storage - - Arrays are stored as pointers in a @ref basic_json type. That is, for any - access to array values, a pointer of type `array_t*` must be dereferenced. - - @sa @ref object_t -- type for an object value - - @since version 1.0.0 - */ - using array_t = ArrayType>; - - /*! - @brief a type for a string - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: - > A string is a sequence of zero or more Unicode characters. - - To store objects in C++, a type is defined by the template parameter - described below. Unicode values are split by the JSON class into - byte-sized characters during deserialization. - - @tparam StringType the container to store strings (e.g., `std::string`). - Note this container is used for keys/names in objects, see @ref object_t. - - #### Default type - - With the default values for @a StringType (`std::string`), the default - value for @a string_t is: - - @code {.cpp} - std::string - @endcode - - #### String comparison - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > Software implementations are typically required to test names of object - > members for equality. Implementations that transform the textual - > representation into sequences of Unicode code units and then perform the - > comparison numerically, code unit by code unit, are interoperable in the - > sense that implementations will agree in all cases on equality or - > inequality of two strings. For example, implementations that compare - > strings with escaped characters unconverted may incorrectly find that - > `"a\\b"` and `"a\u005Cb"` are not equal. - - This implementation is interoperable as it does compare strings code unit - by code unit. - - #### Storage - - String values are stored as pointers in a @ref basic_json type. That is, - for any access to string values, a pointer of type `string_t*` must be - dereferenced. - - @since version 1.0.0 - */ - using string_t = StringType; - - /*! - @brief a type for a boolean - - [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a - type which differentiates the two literals `true` and `false`. - - To store objects in C++, a type is defined by the template parameter @a - BooleanType which chooses the type to use. - - #### Default type - - With the default values for @a BooleanType (`bool`), the default value for - @a boolean_t is: - - @code {.cpp} - bool - @endcode - - #### Storage - - Boolean values are stored directly inside a @ref basic_json type. - - @since version 1.0.0 - */ - using boolean_t = BooleanType; - - /*! - @brief a type for a number (integer) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store integer numbers in C++, a type is defined by the template - parameter @a NumberIntegerType which chooses the type to use. - - #### Default type - - With the default values for @a NumberIntegerType (`int64_t`), the default - value for @a number_integer_t is: - - @code {.cpp} - int64_t - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. - - When the default type is used, the maximal integer number that can be - stored is `9223372036854775807` (INT64_MAX) and the minimal integer number - that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers - that are out of range will yield over/underflow when used in a - constructor. During deserialization, too large or small integer numbers - will be automatically be stored as @ref number_unsigned_t or @ref - number_float_t. - - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. - - As this range is a subrange of the exactly supported range [INT64_MIN, - INT64_MAX], this class's integer type is interoperable. - - #### Storage - - Integer number values are stored directly inside a @ref basic_json type. - - @sa @ref number_float_t -- type for number values (floating-point) - - @sa @ref number_unsigned_t -- type for number values (unsigned integer) - - @since version 1.0.0 - */ - using number_integer_t = NumberIntegerType; - - /*! - @brief a type for a number (unsigned) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store unsigned integer numbers in C++, a type is defined by the - template parameter @a NumberUnsignedType which chooses the type to use. - - #### Default type - - With the default values for @a NumberUnsignedType (`uint64_t`), the - default value for @a number_unsigned_t is: - - @code {.cpp} - uint64_t - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. - - When the default type is used, the maximal integer number that can be - stored is `18446744073709551615` (UINT64_MAX) and the minimal integer - number that can be stored is `0`. Integer numbers that are out of range - will yield over/underflow when used in a constructor. During - deserialization, too large or small integer numbers will be automatically - be stored as @ref number_integer_t or @ref number_float_t. - - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. - - As this range is a subrange (when considered in conjunction with the - number_integer_t type) of the exactly supported range [0, UINT64_MAX], - this class's integer type is interoperable. - - #### Storage - - Integer number values are stored directly inside a @ref basic_json type. - - @sa @ref number_float_t -- type for number values (floating-point) - @sa @ref number_integer_t -- type for number values (integer) - - @since version 2.0.0 - */ - using number_unsigned_t = NumberUnsignedType; - - /*! - @brief a type for a number (floating-point) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store floating-point numbers in C++, a type is defined by the template - parameter @a NumberFloatType which chooses the type to use. - - #### Default type - - With the default values for @a NumberFloatType (`double`), the default - value for @a number_float_t is: - - @code {.cpp} - double - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in floating-point literals will be ignored. Internally, - the value will be stored as decimal number. For instance, the C++ - floating-point literal `01.2` will be serialized to `1.2`. During - deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > This specification allows implementations to set limits on the range and - > precision of numbers accepted. Since software that implements IEEE - > 754-2008 binary64 (double precision) numbers is generally available and - > widely used, good interoperability can be achieved by implementations - > that expect no more precision or range than these provide, in the sense - > that implementations will approximate JSON numbers within the expected - > precision. - - This implementation does exactly follow this approach, as it uses double - precision floating-point numbers. Note values smaller than - `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` - will be stored as NaN internally and be serialized to `null`. - - #### Storage - - Floating-point number values are stored directly inside a @ref basic_json - type. - - @sa @ref number_integer_t -- type for number values (integer) - - @sa @ref number_unsigned_t -- type for number values (unsigned integer) - - @since version 1.0.0 - */ - using number_float_t = NumberFloatType; - - /// @} - - - /////////////////////////// - // JSON type enumeration // - /////////////////////////// - - /*! - @brief the JSON type enumeration - - This enumeration collects the different JSON types. It is internally used - to distinguish the stored values, and the functions @ref is_null(), @ref - is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref - is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and - @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and - @ref is_structured() rely on it. - - @note There are three enumeration entries (number_integer, - number_unsigned, and number_float), because the library distinguishes - these three types for numbers: @ref number_unsigned_t is used for unsigned - integers, @ref number_integer_t is used for signed integers, and @ref - number_float_t is used for floating-point numbers or to approximate - integers which do not fit in the limits of their respective type. - - @sa @ref basic_json(const value_t value_type) -- create a JSON value with - the default value for a given type - - @since version 1.0.0 - */ - enum class value_t : uint8_t - { - null, ///< null value - object, ///< object (unordered set of name/value pairs) - array, ///< array (ordered collection of values) - string, ///< string value - boolean, ///< boolean value - number_integer, ///< number value (signed integer) - number_unsigned, ///< number value (unsigned integer) - number_float, ///< number value (floating-point) - discarded ///< discarded by the the parser callback function - }; - - - private: - - /// helper for exception-safe object creation - template - static T* create(Args&& ... args) - { - AllocatorType alloc; - auto deleter = [&](T * object) - { - alloc.deallocate(object, 1); - }; - std::unique_ptr object(alloc.allocate(1), deleter); - alloc.construct(object.get(), std::forward(args)...); - assert(object.get() != nullptr); - return object.release(); - } - - //////////////////////// - // JSON value storage // - //////////////////////// - - /*! - @brief a JSON value - - The actual storage for a JSON value of the @ref basic_json class. This - union combines the different storage types for the JSON value types - defined in @ref value_t. - - JSON type | value_t type | used type - --------- | --------------- | ------------------------ - object | object | pointer to @ref object_t - array | array | pointer to @ref array_t - string | string | pointer to @ref string_t - boolean | boolean | @ref boolean_t - number | number_integer | @ref number_integer_t - number | number_unsigned | @ref number_unsigned_t - number | number_float | @ref number_float_t - null | null | *no value is stored* - - @note Variable-length types (objects, arrays, and strings) are stored as - pointers. The size of the union should not exceed 64 bits if the default - value types are used. - - @since version 1.0.0 - */ - union json_value - { - /// object (stored with pointer to save storage) - object_t* object; - /// array (stored with pointer to save storage) - array_t* array; - /// string (stored with pointer to save storage) - string_t* string; - /// boolean - boolean_t boolean; - /// number (integer) - number_integer_t number_integer; - /// number (unsigned integer) - number_unsigned_t number_unsigned; - /// number (floating-point) - number_float_t number_float; - - /// default constructor (for null values) - json_value() = default; - /// constructor for booleans - json_value(boolean_t v) noexcept : boolean(v) {} - /// constructor for numbers (integer) - json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} - /// constructor for numbers (floating-point) - json_value(number_float_t v) noexcept : number_float(v) {} - /// constructor for empty values of a given type - json_value(value_t t) - { - switch (t) - { - case value_t::object: - { - object = create(); - break; - } - - case value_t::array: - { - array = create(); - break; - } - - case value_t::string: - { - string = create(""); - break; - } - - case value_t::boolean: - { - boolean = boolean_t(false); - break; - } - - case value_t::number_integer: - { - number_integer = number_integer_t(0); - break; - } - - case value_t::number_unsigned: - { - number_unsigned = number_unsigned_t(0); - break; - } - - case value_t::number_float: - { - number_float = number_float_t(0.0); - break; - } - - default: - { - break; - } - } - } - - /// constructor for strings - json_value(const string_t& value) - { - string = create(value); - } - - /// constructor for objects - json_value(const object_t& value) - { - object = create(value); - } - - /// constructor for arrays - json_value(const array_t& value) - { - array = create(value); - } - }; - - /*! - @brief checks the class invariants - - This function asserts the class invariants. It needs to be called at the - end of every constructor to make sure that created objects respect the - invariant. Furthermore, it has to be called each time the type of a JSON - value is changed, because the invariant expresses a relationship between - @a m_type and @a m_value. - */ - void assert_invariant() const - { - assert(m_type != value_t::object or m_value.object != nullptr); - assert(m_type != value_t::array or m_value.array != nullptr); - assert(m_type != value_t::string or m_value.string != nullptr); - } - - public: - ////////////////////////// - // JSON parser callback // - ////////////////////////// - - /*! - @brief JSON callback events - - This enumeration lists the parser events that can trigger calling a - callback function of type @ref parser_callback_t during parsing. - - @image html callback_events.png "Example when certain parse events are triggered" - - @since version 1.0.0 - */ - enum class parse_event_t : uint8_t - { - /// the parser read `{` and started to process a JSON object - object_start, - /// the parser read `}` and finished processing a JSON object - object_end, - /// the parser read `[` and started to process a JSON array - array_start, - /// the parser read `]` and finished processing a JSON array - array_end, - /// the parser read a key of a value in an object - key, - /// the parser finished reading a JSON value - value - }; - - /*! - @brief per-element parser callback type - - With a parser callback function, the result of parsing a JSON text can be - influenced. When passed to @ref parse(std::istream&, const - parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), - it is called on certain events (passed as @ref parse_event_t via parameter - @a event) with a set recursion depth @a depth and context JSON value - @a parsed. The return value of the callback function is a boolean - indicating whether the element that emitted the callback shall be kept or - not. - - We distinguish six scenarios (determined by the event type) in which the - callback function can be called. The following table describes the values - of the parameters @a depth, @a event, and @a parsed. - - parameter @a event | description | parameter @a depth | parameter @a parsed - ------------------ | ----------- | ------------------ | ------------------- - parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded - parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key - parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object - parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded - parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array - parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value - - @image html callback_events.png "Example when certain parse events are triggered" - - Discarding a value (i.e., returning `false`) has different effects - depending on the context in which function was called: - - - Discarded values in structured types are skipped. That is, the parser - will behave as if the discarded value was never read. - - In case a value outside a structured type is skipped, it is replaced - with `null`. This case happens if the top-level element is skipped. - - @param[in] depth the depth of the recursion during parsing - - @param[in] event an event of type parse_event_t indicating the context in - the callback function has been called - - @param[in,out] parsed the current intermediate parse result; note that - writing to this value has no effect for parse_event_t::key events - - @return Whether the JSON value which called the function during parsing - should be kept (`true`) or not (`false`). In the latter case, it is either - skipped completely or replaced by an empty discarded object. - - @sa @ref parse(std::istream&, parser_callback_t) or - @ref parse(const string_t&, parser_callback_t) for examples - - @since version 1.0.0 - */ - using parser_callback_t = std::function; - - - ////////////////// - // constructors // - ////////////////// - - /// @name constructors and destructors - /// Constructors of class @ref basic_json, copy/move constructor, copy - /// assignment, static functions creating objects, and the destructor. - /// @{ - - /*! - @brief create an empty value with a given type - - Create an empty JSON value with a given type. The value will be default - initialized with an empty value which depends on the type: - - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` - - @param[in] value_type the type of the value to create - - @complexity Constant. - - @throw std::bad_alloc if allocation for object, array, or string value - fails - - @liveexample{The following code shows the constructor for different @ref - value_t values,basic_json__value_t} - - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - @sa @ref basic_json(boolean_t value) -- create a boolean value - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const object_t&) -- create a object value - @sa @ref basic_json(const array_t&) -- create a array value - @sa @ref basic_json(const number_float_t) -- create a number - (floating-point) value - @sa @ref basic_json(const number_integer_t) -- create a number (integer) - value - @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) - value - - @since version 1.0.0 - */ - basic_json(const value_t value_type) - : m_type(value_type), m_value(value_type) - { - assert_invariant(); - } - - /*! - @brief create a null object (implicitly) - - Create a `null` JSON value. This is the implicit version of the `null` - value constructor as it takes no parameters. - - @note The class invariant is satisfied, because it poses no requirements - for null values. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - As postcondition, it holds: `basic_json().empty() == true`. - - @liveexample{The following code shows the constructor for a `null` JSON - value.,basic_json} - - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - - @since version 1.0.0 - */ - basic_json() = default; - - /*! - @brief create a null object (explicitly) - - Create a `null` JSON value. This is the explicitly version of the `null` - value constructor as it takes a null pointer as parameter. It allows to - create `null` values by explicitly assigning a `nullptr` to a JSON value. - The passed null pointer itself is not read -- it is only used to choose - the right constructor. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. - - @liveexample{The following code shows the constructor with null pointer - parameter.,basic_json__nullptr_t} - - @sa @ref basic_json() -- default constructor (implicitly creating a `null` - value) - - @since version 1.0.0 - */ - basic_json(std::nullptr_t) noexcept - : basic_json(value_t::null) - { - assert_invariant(); - } - - /*! - @brief create an object (explicit) - - Create an object JSON value with a given content. - - @param[in] val a value for the object - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for object value fails - - @liveexample{The following code shows the constructor with an @ref - object_t parameter.,basic_json__object_t} - - @sa @ref basic_json(const CompatibleObjectType&) -- create an object value - from a compatible STL container - - @since version 1.0.0 - */ - basic_json(const object_t& val) - : m_type(value_t::object), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an object (implicit) - - Create an object JSON value with a given content. This constructor allows - any type @a CompatibleObjectType that can be used to construct values of - type @ref object_t. - - @tparam CompatibleObjectType An object type whose `key_type` and - `value_type` is compatible to @ref object_t. Examples include `std::map`, - `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with - a `key_type` of `std::string`, and a `value_type` from which a @ref - basic_json value can be constructed. - - @param[in] val a value for the object - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for object value fails - - @liveexample{The following code shows the constructor with several - compatible object type parameters.,basic_json__CompatibleObjectType} - - @sa @ref basic_json(const object_t&) -- create an object value - - @since version 1.0.0 - */ - template ::value and - std::is_constructible::value, int>::type - = 0> - basic_json(const CompatibleObjectType& val) - : m_type(value_t::object) - { - using std::begin; - using std::end; - m_value.object = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create an array (explicit) - - Create an array JSON value with a given content. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with an @ref array_t - parameter.,basic_json__array_t} - - @sa @ref basic_json(const CompatibleArrayType&) -- create an array value - from a compatible STL containers - - @since version 1.0.0 - */ - basic_json(const array_t& val) - : m_type(value_t::array), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an array (implicit) - - Create an array JSON value with a given content. This constructor allows - any type @a CompatibleArrayType that can be used to construct values of - type @ref array_t. - - @tparam CompatibleArrayType An object type whose `value_type` is - compatible to @ref array_t. Examples include `std::vector`, `std::deque`, - `std::list`, `std::forward_list`, `std::array`, `std::set`, - `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a - `value_type` from which a @ref basic_json value can be constructed. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with several - compatible array type parameters.,basic_json__CompatibleArrayType} - - @sa @ref basic_json(const array_t&) -- create an array value - - @since version 1.0.0 - */ - template ::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type - = 0> - basic_json(const CompatibleArrayType& val) - : m_type(value_t::array) - { - using std::begin; - using std::end; - m_value.array = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create an string JSON value with a given content. - - @param[in] val a value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with an @ref - string_t parameter.,basic_json__string_t} - - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const string_t& val) - : m_type(value_t::string), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create a string JSON value with a given content. - - @param[in] val a literal value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with string literal - parameter.,basic_json__string_t_value_type} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const typename string_t::value_type* val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a string (implicit) - - Create a string JSON value with a given content. - - @param[in] val a value for the string - - @tparam CompatibleStringType an string type which is compatible to @ref - string_t, for instance `std::string`. - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the construction of a string value - from a compatible type.,basic_json__CompatibleStringType} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - - @since version 1.0.0 - */ - template ::value, int>::type - = 0> - basic_json(const CompatibleStringType& val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a boolean (explicit) - - Creates a JSON boolean type from a given value. - - @param[in] val a boolean value to store - - @complexity Constant. - - @liveexample{The example below demonstrates boolean - values.,basic_json__boolean_t} - - @since version 1.0.0 - */ - basic_json(boolean_t val) noexcept - : m_type(value_t::boolean), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number (explicit) - - Create an integer number JSON value with a given content. - - @tparam T A helper type to remove this function via SFINAE in case @ref - number_integer_t is the same as `int`. In this case, this constructor - would have the same signature as @ref basic_json(const int value). Note - the helper type @a T is not visible in this constructor's interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value.,basic_json__number_integer_t} - - @sa @ref basic_json(const int) -- create a number value (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - template::value) - and std::is_same::value - , int>::type - = 0> - basic_json(const number_integer_t val) noexcept - : m_type(value_t::number_integer), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number from an enum type (explicit) - - Create an integer number JSON value with a given content. - - @param[in] val an integer to create a JSON number from - - @note This constructor allows to pass enums directly to a constructor. As - C++ has no way of specifying the type of an anonymous enum explicitly, we - can only rely on the fact that such values implicitly convert to int. As - int may already be the same type of number_integer_t, we may need to - switch off the constructor @ref basic_json(const number_integer_t). - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value from an anonymous enum.,basic_json__const_int} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const int val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an integer number (implicit) - - Create an integer number JSON value with a given content. This constructor - allows any type @a CompatibleNumberIntegerType that can be used to - construct values of type @ref number_integer_t. - - @tparam CompatibleNumberIntegerType An integer type which is compatible to - @ref number_integer_t. Examples include the types `int`, `int32_t`, - `long`, and `short`. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of several integer - number values from compatible - types.,basic_json__CompatibleIntegerNumberType} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const int) -- create a number value (integer) - - @since version 1.0.0 - */ - template::value and - std::numeric_limits::is_integer and - std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type - = 0> - basic_json(const CompatibleNumberIntegerType val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an unsigned integer number (explicit) - - Create an unsigned integer number JSON value with a given content. - - @tparam T helper type to compare number_unsigned_t and unsigned int (not - visible in) the interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number - value (unsigned integer) from a compatible number type - - @since version 2.0.0 - */ - template::value) - and std::is_same::value - , int>::type - = 0> - basic_json(const number_unsigned_t val) noexcept - : m_type(value_t::number_unsigned), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an unsigned number (implicit) - - Create an unsigned number JSON value with a given content. This - constructor allows any type @a CompatibleNumberUnsignedType that can be - used to construct values of type @ref number_unsigned_t. - - @tparam CompatibleNumberUnsignedType An integer type which is compatible - to @ref number_unsigned_t. Examples may include the types `unsigned int`, - `uint32_t`, or `unsigned short`. - - @param[in] val an unsigned integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const number_unsigned_t) -- create a number value - (unsigned) - - @since version 2.0.0 - */ - template ::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type - = 0> - basic_json(const CompatibleNumberUnsignedType val) noexcept - : m_type(value_t::number_unsigned), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create a floating-point number (explicit) - - Create a floating-point number JSON value with a given content. - - @param[in] val a floating-point value to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is created - instead. - - @complexity Constant. - - @liveexample{The following example creates several floating-point - values.,basic_json__number_float_t} - - @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number - value (floating-point) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const number_float_t val) noexcept - : m_type(value_t::number_float), m_value(val) - { - // replace infinity and NAN by null - if (not std::isfinite(val)) - { - m_type = value_t::null; - m_value = json_value(); - } - - assert_invariant(); - } - - /*! - @brief create an floating-point number (implicit) - - Create an floating-point number JSON value with a given content. This - constructor allows any type @a CompatibleNumberFloatType that can be used - to construct values of type @ref number_float_t. - - @tparam CompatibleNumberFloatType A floating-point type which is - compatible to @ref number_float_t. Examples may include the types `float` - or `double`. - - @param[in] val a floating-point to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is - created instead. - - @complexity Constant. - - @liveexample{The example below shows the construction of several - floating-point number values from compatible - types.,basic_json__CompatibleNumberFloatType} - - @sa @ref basic_json(const number_float_t) -- create a number value - (floating-point) - - @since version 1.0.0 - */ - template::value and - std::is_floating_point::value>::type - > - basic_json(const CompatibleNumberFloatType val) noexcept - : basic_json(number_float_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a container (array or object) from an initializer list - - Creates a JSON value of type array or object from the passed initializer - list @a init. In case @a type_deduction is `true` (default), the type of - the JSON value to be created is deducted from the initializer list @a init - according to the following rules: - - 1. If the list is empty, an empty JSON object value `{}` is created. - 2. If the list consists of pairs whose first element is a string, a JSON - object value is created where the first elements of the pairs are - treated as keys and the second elements are as values. - 3. In all other cases, an array is created. - - The rules aim to create the best fit between a C++ initializer list and - JSON values. The rationale is as follows: - - 1. The empty initializer list is written as `{}` which is exactly an empty - JSON object. - 2. C++ has now way of describing mapped types other than to list a list of - pairs. As JSON requires that keys must be of type string, rule 2 is the - weakest constraint one can pose on initializer lists to interpret them - as an object. - 3. In all other cases, the initializer list could not be interpreted as - JSON object type, so interpreting it as JSON array type is safe. - - With the rules described above, the following JSON values cannot be - expressed by an initializer list: - - - the empty array (`[]`): use @ref array(std::initializer_list) - with an empty initializer list in this case - - arrays whose elements satisfy rule 2: use @ref - array(std::initializer_list) with the same initializer list - in this case - - @note When used without parentheses around an empty initializer list, @ref - basic_json() is called instead of this function, yielding the JSON null - value. - - @param[in] init initializer list with JSON values - - @param[in] type_deduction internal parameter; when set to `true`, the type - of the JSON value is deducted from the initializer list @a init; when set - to `false`, the type provided via @a manual_type is forced. This mode is - used by the functions @ref array(std::initializer_list) and - @ref object(std::initializer_list). - - @param[in] manual_type internal parameter; when @a type_deduction is set - to `false`, the created JSON value will use the provided type (only @ref - value_t::array and @ref value_t::object are valid); when @a type_deduction - is set to `true`, this parameter has no effect - - @throw std::domain_error if @a type_deduction is `false`, @a manual_type - is `value_t::object`, but @a init contains an element which is not a pair - whose first element is a string; example: `"cannot create object from - initializer list"` - - @complexity Linear in the size of the initializer list @a init. - - @liveexample{The example below shows how JSON values are created from - initializer lists.,basic_json__list_init_t} - - @sa @ref array(std::initializer_list) -- create a JSON array - value from an initializer list - @sa @ref object(std::initializer_list) -- create a JSON object - value from an initializer list - - @since version 1.0.0 - */ - basic_json(std::initializer_list init, - bool type_deduction = true, - value_t manual_type = value_t::array) - { - // check if each element is an array with two elements whose first - // element is a string - bool is_an_object = std::all_of(init.begin(), init.end(), - [](const basic_json & element) - { - return element.is_array() and element.size() == 2 and element[0].is_string(); - }); - - // adjust type if type deduction is not wanted - if (not type_deduction) - { - // if array is wanted, do not create an object though possible - if (manual_type == value_t::array) - { - is_an_object = false; - } - - // if object is wanted but impossible, throw an exception - if (manual_type == value_t::object and not is_an_object) - { - throw std::domain_error("cannot create object from initializer list"); - } - } - - if (is_an_object) - { - // the initializer list is a list of pairs -> create object - m_type = value_t::object; - m_value = value_t::object; - - std::for_each(init.begin(), init.end(), [this](const basic_json & element) - { - m_value.object->emplace(*(element[0].m_value.string), element[1]); - }); - } - else - { - // the initializer list describes an array -> create array - m_type = value_t::array; - m_value.array = create(init); - } - - assert_invariant(); - } - - /*! - @brief explicitly create an array from an initializer list - - Creates a JSON array value from a given initializer list. That is, given a - list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the - initializer list is empty, the empty array `[]` is created. - - @note This function is only needed to express two edge cases that cannot - be realized with the initializer list constructor (@ref - basic_json(std::initializer_list, bool, value_t)). These cases - are: - 1. creating an array whose elements are all pairs whose first element is a - string -- in this case, the initializer list constructor would create an - object, taking the first elements as keys - 2. creating an empty array -- passing the empty initializer list to the - initializer list constructor yields an empty object - - @param[in] init initializer list with JSON values to create an array from - (optional) - - @return JSON array value - - @complexity Linear in the size of @a init. - - @liveexample{The following code shows an example for the `array` - function.,array} - - @sa @ref basic_json(std::initializer_list, bool, value_t) -- - create a JSON value from an initializer list - @sa @ref object(std::initializer_list) -- create a JSON object - value from an initializer list - - @since version 1.0.0 - */ - static basic_json array(std::initializer_list init = - std::initializer_list()) - { - return basic_json(init, false, value_t::array); - } - - /*! - @brief explicitly create an object from an initializer list - - Creates a JSON object value from a given initializer list. The initializer - lists elements must be pairs, and their first elements must be strings. If - the initializer list is empty, the empty object `{}` is created. - - @note This function is only added for symmetry reasons. In contrast to the - related function @ref array(std::initializer_list), there are - no cases which can only be expressed by this function. That is, any - initializer list @a init can also be passed to the initializer list - constructor @ref basic_json(std::initializer_list, bool, - value_t). - - @param[in] init initializer list to create an object from (optional) - - @return JSON object value - - @throw std::domain_error if @a init is not a pair whose first elements are - strings; thrown by - @ref basic_json(std::initializer_list, bool, value_t) - - @complexity Linear in the size of @a init. - - @liveexample{The following code shows an example for the `object` - function.,object} - - @sa @ref basic_json(std::initializer_list, bool, value_t) -- - create a JSON value from an initializer list - @sa @ref array(std::initializer_list) -- create a JSON array - value from an initializer list - - @since version 1.0.0 - */ - static basic_json object(std::initializer_list init = - std::initializer_list()) - { - return basic_json(init, false, value_t::object); - } - - /*! - @brief construct an array with count copies of given value - - Constructs a JSON array value by creating @a cnt copies of a passed value. - In case @a cnt is `0`, an empty array is created. As postcondition, - `std::distance(begin(),end()) == cnt` holds. - - @param[in] cnt the number of JSON copies of @a val to create - @param[in] val the JSON value to copy - - @complexity Linear in @a cnt. - - @liveexample{The following code shows examples for the @ref - basic_json(size_type\, const basic_json&) - constructor.,basic_json__size_type_basic_json} - - @since version 1.0.0 - */ - basic_json(size_type cnt, const basic_json& val) - : m_type(value_t::array) - { - m_value.array = create(cnt, val); - assert_invariant(); - } - - /*! - @brief construct a JSON container given an iterator range - - Constructs the JSON value with the contents of the range `[first, last)`. - The semantics depends on the different types a JSON value can have: - - In case of primitive types (number, boolean, or string), @a first must - be `begin()` and @a last must be `end()`. In this case, the value is - copied. Otherwise, std::out_of_range is thrown. - - In case of structured types (array, object), the constructor behaves as - similar versions for `std::vector`. - - In case of a null type, std::domain_error is thrown. - - @tparam InputIT an input iterator type (@ref iterator or @ref - const_iterator) - - @param[in] first begin of the range to copy from (included) - @param[in] last end of the range to copy from (excluded) - - @pre Iterators @a first and @a last must be initialized. - - @throw std::domain_error if iterators are not compatible; that is, do not - belong to the same JSON value; example: `"iterators are not compatible"` - @throw std::out_of_range if iterators are for a primitive type (number, - boolean, or string) where an out of range error can be detected easily; - example: `"iterators out of range"` - @throw std::bad_alloc if allocation for object, array, or string fails - @throw std::domain_error if called with a null value; example: `"cannot - use construct with iterators from null"` - - @complexity Linear in distance between @a first and @a last. - - @liveexample{The example below shows several ways to create JSON values by - specifying a subrange with iterators.,basic_json__InputIt_InputIt} - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - basic_json(InputIT first, InputIT last) - { - assert(first.m_object != nullptr); - assert(last.m_object != nullptr); - - // make sure iterator fits the current value - if (first.m_object != last.m_object) - { - throw std::domain_error("iterators are not compatible"); - } - - // copy type from first iterator - m_type = first.m_object->m_type; - - // check if iterator range is complete for primitive values - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) - { - throw std::out_of_range("iterators out of range"); - } - break; - } - - default: - { - break; - } - } - - switch (m_type) - { - case value_t::number_integer: - { - m_value.number_integer = first.m_object->m_value.number_integer; - break; - } - - case value_t::number_unsigned: - { - m_value.number_unsigned = first.m_object->m_value.number_unsigned; - break; - } - - case value_t::number_float: - { - m_value.number_float = first.m_object->m_value.number_float; - break; - } - - case value_t::boolean: - { - m_value.boolean = first.m_object->m_value.boolean; - break; - } - - case value_t::string: - { - m_value = *first.m_object->m_value.string; - break; - } - - case value_t::object: - { - m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); - break; - } - - case value_t::array: - { - m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); - } - } - - assert_invariant(); - } - - /*! - @brief construct a JSON value given an input stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates constructing a JSON value from - a `std::stringstream` with and without callback - function.,basic_json__istream} - - @since version 2.0.0 - */ - explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) - { - *this = parser(i, cb).parse(); - assert_invariant(); - } - - /////////////////////////////////////// - // other constructors and destructor // - /////////////////////////////////////// - - /*! - @brief copy constructor - - Creates a copy of a given JSON value. - - @param[in] other the JSON value to copy - - @complexity Linear in the size of @a other. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - As postcondition, it holds: `other == basic_json(other)`. - - @throw std::bad_alloc if allocation for object, array, or string fails. - - @liveexample{The following code shows an example for the copy - constructor.,basic_json__basic_json} - - @since version 1.0.0 - */ - basic_json(const basic_json& other) - : m_type(other.m_type) - { - // check of passed value is valid - other.assert_invariant(); - - switch (m_type) - { - case value_t::object: - { - m_value = *other.m_value.object; - break; - } - - case value_t::array: - { - m_value = *other.m_value.array; - break; - } - - case value_t::string: - { - m_value = *other.m_value.string; - break; - } - - case value_t::boolean: - { - m_value = other.m_value.boolean; - break; - } - - case value_t::number_integer: - { - m_value = other.m_value.number_integer; - break; - } - - case value_t::number_unsigned: - { - m_value = other.m_value.number_unsigned; - break; - } - - case value_t::number_float: - { - m_value = other.m_value.number_float; - break; - } - - default: - { - break; - } - } - - assert_invariant(); - } - - /*! - @brief move constructor - - Move constructor. Constructs a JSON value with the contents of the given - value @a other using move semantics. It "steals" the resources from @a - other and leaves it as JSON null value. - - @param[in,out] other value to move to this object - - @post @a other is a JSON null value - - @complexity Constant. - - @liveexample{The code below shows the move constructor explicitly called - via std::move.,basic_json__moveconstructor} - - @since version 1.0.0 - */ - basic_json(basic_json&& other) noexcept - : m_type(std::move(other.m_type)), - m_value(std::move(other.m_value)) - { - // check that passed value is valid - other.assert_invariant(); - - // invalidate payload - other.m_type = value_t::null; - other.m_value = {}; - - assert_invariant(); - } - - /*! - @brief copy assignment - - Copy assignment operator. Copies a JSON value via the "copy and swap" - strategy: It is expressed in terms of the copy constructor, destructor, - and the swap() member function. - - @param[in] other value to copy from - - @complexity Linear. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - @liveexample{The code below shows and example for the copy assignment. It - creates a copy of value `a` which is then swapped with `b`. Finally\, the - copy of `a` (which is the null value after the swap) is - destroyed.,basic_json__copyassignment} - - @since version 1.0.0 - */ - reference& operator=(basic_json other) noexcept ( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - // check that passed value is valid - other.assert_invariant(); - - using std::swap; - swap(m_type, other.m_type); - swap(m_value, other.m_value); - - assert_invariant(); - return *this; - } - - /*! - @brief destructor - - Destroys the JSON value and frees all allocated memory. - - @complexity Linear. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - All stored elements are destroyed and all memory is freed. - - @since version 1.0.0 - */ - ~basic_json() - { - assert_invariant(); - - switch (m_type) - { - case value_t::object: - { - AllocatorType alloc; - alloc.destroy(m_value.object); - alloc.deallocate(m_value.object, 1); - break; - } - - case value_t::array: - { - AllocatorType alloc; - alloc.destroy(m_value.array); - alloc.deallocate(m_value.array, 1); - break; - } - - case value_t::string: - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - break; - } - - default: - { - // all other types need no specific destructor - break; - } - } - } - - /// @} - - public: - /////////////////////// - // object inspection // - /////////////////////// - - /// @name object inspection - /// Functions to inspect the type of a JSON value. - /// @{ - - /*! - @brief serialization - - Serialization function for JSON values. The function tries to mimic - Python's `json.dumps()` function, and currently supports its @a indent - parameter. - - @param[in] indent If indent is nonnegative, then array elements and object - members will be pretty-printed with that indent level. An indent level of - `0` will only insert newlines. `-1` (the default) selects the most compact - representation. - - @return string containing the serialization of the JSON value - - @complexity Linear. - - @liveexample{The following example shows the effect of different @a indent - parameters to the result of the serialization.,dump} - - @see https://docs.python.org/2/library/json.html#json.dump - - @since version 1.0.0 - */ - string_t dump(const int indent = -1) const - { - std::stringstream ss; - // fix locale problems - ss.imbue(std::locale(std::locale(), new DecimalSeparator)); - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - ss.precision(std::numeric_limits::digits10); - - if (indent >= 0) - { - dump(ss, true, static_cast(indent)); - } - else - { - dump(ss, false, 0); - } - - return ss.str(); - } - - /*! - @brief return the type of the JSON value (explicit) - - Return the type of the JSON value as a value from the @ref value_t - enumeration. - - @return the type of the JSON value - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `type()` for all JSON - types.,type} - - @since version 1.0.0 - */ - constexpr value_t type() const noexcept - { - return m_type; - } - - /*! - @brief return whether type is primitive - - This function returns true iff the JSON type is primitive (string, number, - boolean, or null). - - @return `true` if type is primitive (string, number, boolean, or null), - `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_primitive()` for all JSON - types.,is_primitive} - - @sa @ref is_structured() -- returns whether JSON value is structured - @sa @ref is_null() -- returns whether JSON value is `null` - @sa @ref is_string() -- returns whether JSON value is a string - @sa @ref is_boolean() -- returns whether JSON value is a boolean - @sa @ref is_number() -- returns whether JSON value is a number - - @since version 1.0.0 - */ - constexpr bool is_primitive() const noexcept - { - return is_null() or is_string() or is_boolean() or is_number(); - } - - /*! - @brief return whether type is structured - - This function returns true iff the JSON type is structured (array or - object). - - @return `true` if type is structured (array or object), `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_structured()` for all JSON - types.,is_structured} - - @sa @ref is_primitive() -- returns whether value is primitive - @sa @ref is_array() -- returns whether value is an array - @sa @ref is_object() -- returns whether value is an object - - @since version 1.0.0 - */ - constexpr bool is_structured() const noexcept - { - return is_array() or is_object(); - } - - /*! - @brief return whether value is null - - This function returns true iff the JSON value is null. - - @return `true` if type is null, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_null()` for all JSON - types.,is_null} - - @since version 1.0.0 - */ - constexpr bool is_null() const noexcept - { - return m_type == value_t::null; - } - - /*! - @brief return whether value is a boolean - - This function returns true iff the JSON value is a boolean. - - @return `true` if type is boolean, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_boolean()` for all JSON - types.,is_boolean} - - @since version 1.0.0 - */ - constexpr bool is_boolean() const noexcept - { - return m_type == value_t::boolean; - } - - /*! - @brief return whether value is a number - - This function returns true iff the JSON value is a number. This includes - both integer and floating-point values. - - @return `true` if type is number (regardless whether integer, unsigned - integer or floating-type), `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number()` for all JSON - types.,is_number} - - @sa @ref is_number_integer() -- check if value is an integer or unsigned - integer number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 1.0.0 - */ - constexpr bool is_number() const noexcept - { - return is_number_integer() or is_number_float(); - } - - /*! - @brief return whether value is an integer number - - This function returns true iff the JSON value is an integer or unsigned - integer number. This excludes floating-point values. - - @return `true` if type is an integer or unsigned integer number, `false` - otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_integer()` for all - JSON types.,is_number_integer} - - @sa @ref is_number() -- check if value is a number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 1.0.0 - */ - constexpr bool is_number_integer() const noexcept - { - return m_type == value_t::number_integer or m_type == value_t::number_unsigned; - } - - /*! - @brief return whether value is an unsigned integer number - - This function returns true iff the JSON value is an unsigned integer - number. This excludes floating-point and (signed) integer values. - - @return `true` if type is an unsigned integer number, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_unsigned()` for all - JSON types.,is_number_unsigned} - - @sa @ref is_number() -- check if value is a number - @sa @ref is_number_integer() -- check if value is an integer or unsigned - integer number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 2.0.0 - */ - constexpr bool is_number_unsigned() const noexcept - { - return m_type == value_t::number_unsigned; - } - - /*! - @brief return whether value is a floating-point number - - This function returns true iff the JSON value is a floating-point number. - This excludes integer and unsigned integer values. - - @return `true` if type is a floating-point number, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_float()` for all - JSON types.,is_number_float} - - @sa @ref is_number() -- check if value is number - @sa @ref is_number_integer() -- check if value is an integer number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - - @since version 1.0.0 - */ - constexpr bool is_number_float() const noexcept - { - return m_type == value_t::number_float; - } - - /*! - @brief return whether value is an object - - This function returns true iff the JSON value is an object. - - @return `true` if type is object, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_object()` for all JSON - types.,is_object} - - @since version 1.0.0 - */ - constexpr bool is_object() const noexcept - { - return m_type == value_t::object; - } - - /*! - @brief return whether value is an array - - This function returns true iff the JSON value is an array. - - @return `true` if type is array, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_array()` for all JSON - types.,is_array} - - @since version 1.0.0 - */ - constexpr bool is_array() const noexcept - { - return m_type == value_t::array; - } - - /*! - @brief return whether value is a string - - This function returns true iff the JSON value is a string. - - @return `true` if type is string, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_string()` for all JSON - types.,is_string} - - @since version 1.0.0 - */ - constexpr bool is_string() const noexcept - { - return m_type == value_t::string; - } - - /*! - @brief return whether value is discarded - - This function returns true iff the JSON value was discarded during parsing - with a callback function (see @ref parser_callback_t). - - @note This function will always be `false` for JSON values after parsing. - That is, discarded values can only occur during parsing, but will be - removed when inside a structured value or replaced by null in other cases. - - @return `true` if type is discarded, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_discarded()` for all JSON - types.,is_discarded} - - @since version 1.0.0 - */ - constexpr bool is_discarded() const noexcept - { - return m_type == value_t::discarded; - } - - /*! - @brief return the type of the JSON value (implicit) - - Implicitly return the type of the JSON value as a value from the @ref - value_t enumeration. - - @return the type of the JSON value - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies the @ref value_t operator for - all JSON types.,operator__value_t} - - @since version 1.0.0 - */ - constexpr operator value_t() const noexcept - { - return m_type; - } - - /// @} - - private: - ////////////////// - // value access // - ////////////////// - - /// get an object (explicit) - template ::value and - std::is_convertible::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_object()) - { - return T(m_value.object->begin(), m_value.object->end()); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an object (explicit) - object_t get_impl(object_t*) const - { - if (is_object()) - { - return *(m_value.object); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - T to_vector; - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not std::is_same::value - , int>::type = 0> - std::vector get_impl(std::vector*) const - { - if (is_array()) - { - std::vector to_vector; - to_vector.reserve(m_value.array->size()); - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not has_mapped_type::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - return T(m_value.array->begin(), m_value.array->end()); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - array_t get_impl(array_t*) const - { - if (is_array()) - { - return *(m_value.array); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get a string (explicit) - template ::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_string()) - { - return *m_value.string; - } - else - { - throw std::domain_error("type must be string, but is " + type_name()); - } - } - - /// get a number (explicit) - template::value - , int>::type = 0> - T get_impl(T*) const - { - switch (m_type) - { - case value_t::number_integer: - { - return static_cast(m_value.number_integer); - } - - case value_t::number_unsigned: - { - return static_cast(m_value.number_unsigned); - } - - case value_t::number_float: - { - return static_cast(m_value.number_float); - } - - default: - { - throw std::domain_error("type must be number, but is " + type_name()); - } - } - } - - /// get a boolean (explicit) - constexpr boolean_t get_impl(boolean_t*) const - { - return is_boolean() - ? m_value.boolean - : throw std::domain_error("type must be boolean, but is " + type_name()); - } - - /// get a pointer to the value (object) - object_t* get_impl_ptr(object_t*) noexcept - { - return is_object() ? m_value.object : nullptr; - } - - /// get a pointer to the value (object) - constexpr const object_t* get_impl_ptr(const object_t*) const noexcept - { - return is_object() ? m_value.object : nullptr; - } - - /// get a pointer to the value (array) - array_t* get_impl_ptr(array_t*) noexcept - { - return is_array() ? m_value.array : nullptr; - } - - /// get a pointer to the value (array) - constexpr const array_t* get_impl_ptr(const array_t*) const noexcept - { - return is_array() ? m_value.array : nullptr; - } - - /// get a pointer to the value (string) - string_t* get_impl_ptr(string_t*) noexcept - { - return is_string() ? m_value.string : nullptr; - } - - /// get a pointer to the value (string) - constexpr const string_t* get_impl_ptr(const string_t*) const noexcept - { - return is_string() ? m_value.string : nullptr; - } - - /// get a pointer to the value (boolean) - boolean_t* get_impl_ptr(boolean_t*) noexcept - { - return is_boolean() ? &m_value.boolean : nullptr; - } - - /// get a pointer to the value (boolean) - constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept - { - return is_boolean() ? &m_value.boolean : nullptr; - } - - /// get a pointer to the value (integer number) - number_integer_t* get_impl_ptr(number_integer_t*) noexcept - { - return is_number_integer() ? &m_value.number_integer : nullptr; - } - - /// get a pointer to the value (integer number) - constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept - { - return is_number_integer() ? &m_value.number_integer : nullptr; - } - - /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept - { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; - } - - /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept - { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; - } - - /// get a pointer to the value (floating-point number) - number_float_t* get_impl_ptr(number_float_t*) noexcept - { - return is_number_float() ? &m_value.number_float : nullptr; - } - - /// get a pointer to the value (floating-point number) - constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept - { - return is_number_float() ? &m_value.number_float : nullptr; - } - - /*! - @brief helper function to implement get_ref() - - This funcion helps to implement get_ref() without code duplication for - const and non-const overloads - - @tparam ThisType will be deduced as `basic_json` or `const basic_json` - - @throw std::domain_error if ReferenceType does not match underlying value - type of the current JSON - */ - template - static ReferenceType get_ref_impl(ThisType& obj) - { - // helper type - using PointerType = typename std::add_pointer::type; - - // delegate the call to get_ptr<>() - auto ptr = obj.template get_ptr(); - - if (ptr != nullptr) - { - return *ptr; - } - else - { - throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + - obj.type_name()); - } - } - - public: - - /// @name value access - /// Direct access to the stored value of a JSON value. - /// @{ - - /*! - @brief get a value (explicit) - - Explicit type conversion between the JSON value and a compatible value. - - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays - - @return copy of the JSON value, converted to type @a ValueType - - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON; example: `"type must be object, but is null"` - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows several conversions from JSON values - to other types. There a few things to note: (1) Floating-point numbers can - be converted to integers\, (2) A JSON array can be converted to a standard - `std::vector`\, (3) A JSON object can be converted to C++ - associative containers such as `std::unordered_map`.,get__ValueType_const} - - @internal - The idea of using a casted null pointer to choose the correct - implementation is from . - @endinternal - - @sa @ref operator ValueType() const for implicit conversion - @sa @ref get() for pointer-member access - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - ValueType get() const - { - return get_impl(static_cast(nullptr)); - } - - /*! - @brief get a pointer value (explicit) - - Explicit pointer access to the internally stored JSON value. No copies are - made. - - @warning The pointer becomes invalid if the underlying JSON object - changes. - - @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref - object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. - - @return pointer to the internally stored JSON value if the requested - pointer type @a PointerType fits to the JSON value; `nullptr` otherwise - - @complexity Constant. - - @liveexample{The example below shows how pointers to internal values of a - JSON value can be requested. Note that no type conversions are made and a - `nullptr` is returned if the value and the requested pointer type does not - match.,get__PointerType} - - @sa @ref get_ptr() for explicit pointer-member access - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - PointerType get() noexcept - { - // delegate the call to get_ptr - return get_ptr(); - } - - /*! - @brief get a pointer value (explicit) - @copydoc get() - */ - template::value - , int>::type = 0> - constexpr const PointerType get() const noexcept - { - // delegate the call to get_ptr - return get_ptr(); - } - - /*! - @brief get a pointer value (implicit) - - Implicit pointer access to the internally stored JSON value. No copies are - made. - - @warning Writing data to the pointee of the result yields an undefined - state. - - @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref - object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. Enforced by a static - assertion. - - @return pointer to the internally stored JSON value if the requested - pointer type @a PointerType fits to the JSON value; `nullptr` otherwise - - @complexity Constant. - - @liveexample{The example below shows how pointers to internal values of a - JSON value can be requested. Note that no type conversions are made and a - `nullptr` is returned if the value and the requested pointer type does not - match.,get_ptr} - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - PointerType get_ptr() noexcept - { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() - return get_impl_ptr(static_cast(nullptr)); - } - - /*! - @brief get a pointer value (implicit) - @copydoc get_ptr() - */ - template::value - and std::is_const::type>::value - , int>::type = 0> - constexpr const PointerType get_ptr() const noexcept - { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() const - return get_impl_ptr(static_cast(nullptr)); - } - - /*! - @brief get a reference value (implicit) - - Implict reference access to the internally stored JSON value. No copies - are made. - - @warning Writing data to the referee of the result yields an undefined - state. - - @tparam ReferenceType reference type; must be a reference to @ref array_t, - @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or - @ref number_float_t. Enforced by static assertion. - - @return reference to the internally stored JSON value if the requested - reference type @a ReferenceType fits to the JSON value; throws - std::domain_error otherwise - - @throw std::domain_error in case passed type @a ReferenceType is - incompatible with the stored JSON value - - @complexity Constant. - - @liveexample{The example shows several calls to `get_ref()`.,get_ref} - - @since version 1.1.0 - */ - template::value - , int>::type = 0> - ReferenceType get_ref() - { - // delegate call to get_ref_impl - return get_ref_impl(*this); - } - - /*! - @brief get a reference value (implicit) - @copydoc get_ref() - */ - template::value - and std::is_const::type>::value - , int>::type = 0> - ReferenceType get_ref() const - { - // delegate call to get_ref_impl - return get_ref_impl(*this); - } - - /*! - @brief get a value (implicit) - - Implicit type conversion between the JSON value and a compatible value. - The call is realized by calling @ref get() const. - - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays. The character type of @ref string_t - as well as an initializer list of this type is excluded to avoid - ambiguities as these types implicitly convert to `std::string`. - - @return copy of the JSON value, converted to type @a ValueType - - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON, thrown by @ref get() const - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows several conversions from JSON values - to other types. There a few things to note: (1) Floating-point numbers can - be converted to integers\, (2) A JSON array can be converted to a standard - `std::vector`\, (3) A JSON object can be converted to C++ - associative containers such as `std::unordered_map`.,operator__ValueType} - - @since version 1.0.0 - */ - template < typename ValueType, typename - std::enable_if < - not std::is_pointer::value - and not std::is_same::value -#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 - and not std::is_same>::value -#endif - , int >::type = 0 > - operator ValueType() const - { - // delegate the call to get<>() const - return get(); - } - - /// @} - - - //////////////////// - // element access // - //////////////////// - - /// @name element access - /// Access to the JSON value. - /// @{ - - /*! - @brief access specified array element with bounds checking - - Returns a reference to the element at specified location @a idx, with - bounds checking. - - @param[in] idx index of the element to access - - @return reference to the element at index @a idx - - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read and - written using `at()`.,at__size_type} - - @since version 1.0.0 - */ - reference at(size_type idx) - { - // at only works for arrays - if (is_array()) - { - try - { - return m_value.array->at(idx); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified array element with bounds checking - - Returns a const reference to the element at specified location @a idx, - with bounds checking. - - @param[in] idx index of the element to access - - @return const reference to the element at index @a idx - - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read using - `at()`.,at__size_type_const} - - @since version 1.0.0 - */ - const_reference at(size_type idx) const - { - // at only works for arrays - if (is_array()) - { - try - { - return m_value.array->at(idx); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified object element with bounds checking - - Returns a reference to the element at with specified key @a key, with - bounds checking. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using `at()`.,at__object_t_key_type} - - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - reference at(const typename object_t::key_type& key) - { - // at only works for objects - if (is_object()) - { - try - { - return m_value.object->at(key); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified object element with bounds checking - - Returns a const reference to the element at with specified key @a key, - with bounds checking. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - `at()`.,at__object_t_key_type_const} - - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - const_reference at(const typename object_t::key_type& key) const - { - // at only works for objects - if (is_object()) - { - try - { - return m_value.object->at(key); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified array element - - Returns a reference to the element at specified location @a idx. - - @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), - then the array is silently filled up with `null` values to make `idx` a - valid reference to the last stored element. - - @param[in] idx index of the element to access - - @return reference to the element at index @a idx - - @throw std::domain_error if JSON is not an array or null; example: - `"cannot use operator[] with string"` - - @complexity Constant if @a idx is in the range of the array. Otherwise - linear in `idx - size()`. - - @liveexample{The example below shows how array elements can be read and - written using `[]` operator. Note the addition of `null` - values.,operatorarray__size_type} - - @since version 1.0.0 - */ - reference operator[](size_type idx) - { - // implicitly convert null value to an empty array - if (is_null()) - { - m_type = value_t::array; - m_value.array = create(); - assert_invariant(); - } - - // operator[] only works for arrays - if (is_array()) - { - // fill up array with null values if given idx is outside range - if (idx >= m_value.array->size()) - { - m_value.array->insert(m_value.array->end(), - idx - m_value.array->size() + 1, - basic_json()); - } - - return m_value.array->operator[](idx); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified array element - - Returns a const reference to the element at specified location @a idx. - - @param[in] idx index of the element to access - - @return const reference to the element at index @a idx - - @throw std::domain_error if JSON is not an array; example: `"cannot use - operator[] with null"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read using - the `[]` operator.,operatorarray__size_type_const} - - @since version 1.0.0 - */ - const_reference operator[](size_type idx) const - { - // const operator[] only works for arrays - if (is_array()) - { - return m_value.array->operator[](idx); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - reference operator[](const typename object_t::key_type& key) - { - // implicitly convert null value to an empty object - if (is_null()) - { - m_type = value_t::object; - m_value.object = create(); - assert_invariant(); - } - - // operator[] only works for objects - if (is_object()) - { - return m_value.object->operator[](key); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - const_reference operator[](const typename object_t::key_type& key) const - { - // const operator[] only works for objects - if (is_object()) - { - assert(m_value.object->find(key) != m_value.object->end()); - return m_value.object->find(key)->second; - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - template - reference operator[](T * (&key)[n]) - { - return operator[](static_cast(key)); - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @note This function is required for compatibility reasons with Clang. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - template - const_reference operator[](T * (&key)[n]) const - { - return operator[](static_cast(key)); - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.1.0 - */ - template - reference operator[](T* key) - { - // implicitly convert null to object - if (is_null()) - { - m_type = value_t::object; - m_value = value_t::object; - assert_invariant(); - } - - // at only works for objects - if (is_object()) - { - return m_value.object->operator[](key); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.1.0 - */ - template - const_reference operator[](T* key) const - { - // at only works for objects - if (is_object()) - { - assert(m_value.object->find(key) != m_value.object->end()); - return m_value.object->find(key)->second; - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element with default value - - Returns either a copy of an object's element at the specified key @a key - or a given default value if no element with key @a key exists. - - The function is basically equivalent to executing - @code {.cpp} - try { - return at(key); - } catch(std::out_of_range) { - return default_value; - } - @endcode - - @note Unlike @ref at(const typename object_t::key_type&), this function - does not throw if the given key @a key was not found. - - @note Unlike @ref operator[](const typename object_t::key_type& key), this - function does not implicitly add an element to the position defined by @a - key. This function is furthermore also applicable to const objects. - - @param[in] key key of the element to access - @param[in] default_value the value to return if @a key is not found - - @tparam ValueType type compatible to JSON values, for instance `int` for - JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for - JSON arrays. Note the type of the expected value at @a key and the default - value @a default_value must be compatible. - - @return copy of the element at key @a key or @a default_value if @a key - is not found - - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be queried - with a default value.,basic_json__value} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - - @since version 1.0.0 - */ - template ::value - , int>::type = 0> - ValueType value(const typename object_t::key_type& key, ValueType default_value) const - { - // at only works for objects - if (is_object()) - { - // if key is found, return value and given default value otherwise - const auto it = find(key); - if (it != end()) - { - return *it; - } - else - { - return default_value; - } - } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } - } - - /*! - @brief overload for a default value of type const char* - @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const - */ - string_t value(const typename object_t::key_type& key, const char* default_value) const - { - return value(key, string_t(default_value)); - } - - /*! - @brief access specified object element via JSON Pointer with default value - - Returns either a copy of an object's element at the specified key @a key - or a given default value if no element with key @a key exists. - - The function is basically equivalent to executing - @code {.cpp} - try { - return at(ptr); - } catch(std::out_of_range) { - return default_value; - } - @endcode - - @note Unlike @ref at(const json_pointer&), this function does not throw - if the given key @a key was not found. - - @param[in] ptr a JSON pointer to the element to access - @param[in] default_value the value to return if @a ptr found no value - - @tparam ValueType type compatible to JSON values, for instance `int` for - JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for - JSON arrays. Note the type of the expected value at @a key and the default - value @a default_value must be compatible. - - @return copy of the element at key @a key or @a default_value if @a key - is not found - - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be queried - with a default value.,basic_json__value_ptr} - - @sa @ref operator[](const json_pointer&) for unchecked access by reference - - @since version 2.0.2 - */ - template ::value - , int>::type = 0> - ValueType value(const json_pointer& ptr, ValueType default_value) const - { - // at only works for objects - if (is_object()) - { - // if pointer resolves a value, return it or use default value - try - { - return ptr.get_checked(this); - } - catch (std::out_of_range&) - { - return default_value; - } - } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } - } - - /*! - @brief overload for a default value of type const char* - @copydoc basic_json::value(const json_pointer&, ValueType) const - */ - string_t value(const json_pointer& ptr, const char* default_value) const - { - return value(ptr, string_t(default_value)); - } - - /*! - @brief access the first element - - Returns a reference to the first element in the container. For a JSON - container `c`, the expression `c.front()` is equivalent to `*c.begin()`. - - @return In case of a structured type (array or object), a reference to the - first element is returned. In cast of number, string, or boolean values, a - reference to the value is returned. - - @complexity Constant. - - @pre The JSON value must not be `null` (would throw `std::out_of_range`) - or an empty array or object (undefined behavior, guarded by assertions). - @post The JSON value remains unchanged. - - @throw std::out_of_range when called on `null` value - - @liveexample{The following code shows an example for `front()`.,front} - - @sa @ref back() -- access the last element - - @since version 1.0.0 - */ - reference front() - { - return *begin(); - } - - /*! - @copydoc basic_json::front() - */ - const_reference front() const - { - return *cbegin(); - } - - /*! - @brief access the last element - - Returns a reference to the last element in the container. For a JSON - container `c`, the expression `c.back()` is equivalent to - @code {.cpp} - auto tmp = c.end(); - --tmp; - return *tmp; - @endcode - - @return In case of a structured type (array or object), a reference to the - last element is returned. In cast of number, string, or boolean values, a - reference to the value is returned. - - @complexity Constant. - - @pre The JSON value must not be `null` (would throw `std::out_of_range`) - or an empty array or object (undefined behavior, guarded by assertions). - @post The JSON value remains unchanged. - - @throw std::out_of_range when called on `null` value. - - @liveexample{The following code shows an example for `back()`.,back} - - @sa @ref front() -- access the first element - - @since version 1.0.0 - */ - reference back() - { - auto tmp = end(); - --tmp; - return *tmp; - } - - /*! - @copydoc basic_json::back() - */ - const_reference back() const - { - auto tmp = cend(); - --tmp; - return *tmp; - } - - /*! - @brief remove element given an iterator - - Removes the element specified by iterator @a pos. The iterator @a pos must - be valid and dereferenceable. Thus the `end()` iterator (which is valid, - but is not dereferenceable) cannot be used as a value for @a pos. - - If called on a primitive type other than `null`, the resulting JSON value - will be `null`. - - @param[in] pos iterator to the element to remove - @return Iterator following the last removed element. If the iterator @a - pos refers to the last element, the `end()` iterator is returned. - - @tparam InteratorType an @ref iterator or @ref const_iterator - - @post Invalidates iterators and references at or after the point of the - erase, including the `end()` iterator. - - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on an iterator which does not belong to - the current JSON value; example: `"iterator does not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid - iterator (i.e., any iterator which is not `begin()`); example: `"iterator - out of range"` - - @complexity The complexity depends on the type: - - objects: amortized constant - - arrays: linear in distance between pos and the end of the container - - strings: linear in the length of the string - - other types: constant - - @liveexample{The example shows the result of `erase()` for different JSON - types.,erase__IteratorType} - - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - InteratorType erase(InteratorType pos) - { - // make sure iterator fits the current value - if (this != pos.m_object) - { - throw std::domain_error("iterator does not fit current value"); - } - - InteratorType result = end(); - - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not pos.m_it.primitive_iterator.is_begin()) - { - throw std::out_of_range("iterator out of range"); - } - - if (is_string()) - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - m_value.string = nullptr; - } - - m_type = value_t::null; - assert_invariant(); - break; - } - - case value_t::object: - { - result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); - break; - } - - case value_t::array: - { - result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - return result; - } - - /*! - @brief remove elements given an iterator range - - Removes the element specified by the range `[first; last)`. The iterator - @a first does not need to be dereferenceable if `first == last`: erasing - an empty range is a no-op. - - If called on a primitive type other than `null`, the resulting JSON value - will be `null`. - - @param[in] first iterator to the beginning of the range to remove - @param[in] last iterator past the end of the range to remove - @return Iterator following the last removed element. If the iterator @a - second refers to the last element, the `end()` iterator is returned. - - @tparam InteratorType an @ref iterator or @ref const_iterator - - @post Invalidates iterators and references at or after the point of the - erase, including the `end()` iterator. - - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on iterators which does not belong to - the current JSON value; example: `"iterators do not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid - iterators (i.e., if `first != begin()` and `last != end()`); example: - `"iterators out of range"` - - @complexity The complexity depends on the type: - - objects: `log(size()) + std::distance(first, last)` - - arrays: linear in the distance between @a first and @a last, plus linear - in the distance between @a last and end of the container - - strings: linear in the length of the string - - other types: constant - - @liveexample{The example shows the result of `erase()` for different JSON - types.,erase__IteratorType_IteratorType} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - InteratorType erase(InteratorType first, InteratorType last) - { - // make sure iterator fits the current value - if (this != first.m_object or this != last.m_object) - { - throw std::domain_error("iterators do not fit current value"); - } - - InteratorType result = end(); - - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) - { - throw std::out_of_range("iterators out of range"); - } - - if (is_string()) - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - m_value.string = nullptr; - } - - m_type = value_t::null; - assert_invariant(); - break; - } - - case value_t::object: - { - result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, - last.m_it.object_iterator); - break; - } - - case value_t::array: - { - result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, - last.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - return result; - } - - /*! - @brief remove element from a JSON object given a key - - Removes elements from a JSON object with the key value @a key. - - @param[in] key value of the elements to remove - - @return Number of elements removed. If @a ObjectType is the default - `std::map` type, the return value will always be `0` (@a key was not - found) or `1` (@a key was found). - - @post References and iterators to the erased elements are invalidated. - Other references and iterators are not affected. - - @throw std::domain_error when called on a type other than JSON object; - example: `"cannot use erase() with null"` - - @complexity `log(size()) + count(key)` - - @liveexample{The example shows the effect of `erase()`.,erase__key_type} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - size_type erase(const typename object_t::key_type& key) - { - // this erase only works for objects - if (is_object()) - { - return m_value.object->erase(key); - } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - /*! - @brief remove element from a JSON array given an index - - Removes element from a JSON array at the index @a idx. - - @param[in] idx index of the element to remove - - @throw std::domain_error when called on a type other than JSON array; - example: `"cannot use erase() with null"` - @throw std::out_of_range when `idx >= size()`; example: `"array index 17 - is out of range"` - - @complexity Linear in distance between @a idx and the end of the container. - - @liveexample{The example shows the effect of `erase()`.,erase__size_type} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - - @since version 1.0.0 - */ - void erase(const size_type idx) - { - // this erase only works for arrays - if (is_array()) - { - if (idx >= size()) - { - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - - m_value.array->erase(m_value.array->begin() + static_cast(idx)); - } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - /// @} - - - //////////// - // lookup // - //////////// - - /// @name lookup - /// @{ - - /*! - @brief find an element in a JSON object - - Finds an element in a JSON object with key equivalent to @a key. If the - element is not found or the JSON value is not an object, end() is - returned. - - @param[in] key key value of the element to search for - - @return Iterator to an element with key equivalent to @a key. If no such - element is found, past-the-end (see end()) iterator is returned. - - @complexity Logarithmic in the size of the JSON object. - - @liveexample{The example shows how `find()` is used.,find__key_type} - - @since version 1.0.0 - */ - iterator find(typename object_t::key_type key) - { - auto result = end(); - - if (is_object()) - { - result.m_it.object_iterator = m_value.object->find(key); - } - - return result; - } - - /*! - @brief find an element in a JSON object - @copydoc find(typename object_t::key_type) - */ - const_iterator find(typename object_t::key_type key) const - { - auto result = cend(); - - if (is_object()) - { - result.m_it.object_iterator = m_value.object->find(key); - } - - return result; - } - - /*! - @brief returns the number of occurrences of a key in a JSON object - - Returns the number of elements with key @a key. If ObjectType is the - default `std::map` type, the return value will always be `0` (@a key was - not found) or `1` (@a key was found). - - @param[in] key key value of the element to count - - @return Number of elements with key @a key. If the JSON value is not an - object, the return value will be `0`. - - @complexity Logarithmic in the size of the JSON object. - - @liveexample{The example shows how `count()` is used.,count} - - @since version 1.0.0 - */ - size_type count(typename object_t::key_type key) const - { - // return 0 for all nonobject types - return is_object() ? m_value.object->count(key) : 0; - } - - /// @} - - - /////////////// - // iterators // - /////////////// - - /// @name iterators - /// @{ - - /*! - @brief returns an iterator to the first element - - Returns an iterator to the first element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return iterator to the first element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - @liveexample{The following code shows an example for `begin()`.,begin} - - @sa @ref cbegin() -- returns a const iterator to the beginning - @sa @ref end() -- returns an iterator to the end - @sa @ref cend() -- returns a const iterator to the end - - @since version 1.0.0 - */ - iterator begin() noexcept - { - iterator result(this); - result.set_begin(); - return result; - } - - /*! - @copydoc basic_json::cbegin() - */ - const_iterator begin() const noexcept - { - return cbegin(); - } - - /*! - @brief returns a const iterator to the first element - - Returns a const iterator to the first element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return const iterator to the first element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).begin()`. - - @liveexample{The following code shows an example for `cbegin()`.,cbegin} - - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref end() -- returns an iterator to the end - @sa @ref cend() -- returns a const iterator to the end - - @since version 1.0.0 - */ - const_iterator cbegin() const noexcept - { - const_iterator result(this); - result.set_begin(); - return result; - } - - /*! - @brief returns an iterator to one past the last element - - Returns an iterator to one past the last element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return iterator one past the last element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - @liveexample{The following code shows an example for `end()`.,end} - - @sa @ref cend() -- returns a const iterator to the end - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref cbegin() -- returns a const iterator to the beginning - - @since version 1.0.0 - */ - iterator end() noexcept - { - iterator result(this); - result.set_end(); - return result; - } - - /*! - @copydoc basic_json::cend() - */ - const_iterator end() const noexcept - { - return cend(); - } - - /*! - @brief returns a const iterator to one past the last element - - Returns a const iterator to one past the last element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return const iterator one past the last element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).end()`. - - @liveexample{The following code shows an example for `cend()`.,cend} - - @sa @ref end() -- returns an iterator to the end - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref cbegin() -- returns a const iterator to the beginning - - @since version 1.0.0 - */ - const_iterator cend() const noexcept - { - const_iterator result(this); - result.set_end(); - return result; - } - - /*! - @brief returns an iterator to the reverse-beginning - - Returns an iterator to the reverse-beginning; that is, the last element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `reverse_iterator(end())`. - - @liveexample{The following code shows an example for `rbegin()`.,rbegin} - - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref crend() -- returns a const reverse iterator to the end - - @since version 1.0.0 - */ - reverse_iterator rbegin() noexcept - { - return reverse_iterator(end()); - } - - /*! - @copydoc basic_json::crbegin() - */ - const_reverse_iterator rbegin() const noexcept - { - return crbegin(); - } - - /*! - @brief returns an iterator to the reverse-end - - Returns an iterator to the reverse-end; that is, one before the first - element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `reverse_iterator(begin())`. - - @liveexample{The following code shows an example for `rend()`.,rend} - - @sa @ref crend() -- returns a const reverse iterator to the end - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - - @since version 1.0.0 - */ - reverse_iterator rend() noexcept - { - return reverse_iterator(begin()); - } - - /*! - @copydoc basic_json::crend() - */ - const_reverse_iterator rend() const noexcept - { - return crend(); - } - - /*! - @brief returns a const reverse iterator to the last element - - Returns a const iterator to the reverse-beginning; that is, the last - element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).rbegin()`. - - @liveexample{The following code shows an example for `crbegin()`.,crbegin} - - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref crend() -- returns a const reverse iterator to the end - - @since version 1.0.0 - */ - const_reverse_iterator crbegin() const noexcept - { - return const_reverse_iterator(cend()); - } - - /*! - @brief returns a const reverse iterator to one before the first - - Returns a const reverse iterator to the reverse-end; that is, one before - the first element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).rend()`. - - @liveexample{The following code shows an example for `crend()`.,crend} - - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - - @since version 1.0.0 - */ - const_reverse_iterator crend() const noexcept - { - return const_reverse_iterator(cbegin()); - } - - private: - // forward declaration - template class iteration_proxy; - - public: - /*! - @brief wrapper to access iterator member functions in range-based for - - This function allows to access @ref iterator::key() and @ref - iterator::value() during range-based for loops. In these loops, a - reference to the JSON values is returned, so there is no access to the - underlying iterator. - - @note The name of this function is not yet final and may change in the - future. - */ - static iteration_proxy iterator_wrapper(reference cont) - { - return iteration_proxy(cont); - } - - /*! - @copydoc iterator_wrapper(reference) - */ - static iteration_proxy iterator_wrapper(const_reference cont) - { - return iteration_proxy(cont); - } - - /// @} - - - ////////////// - // capacity // - ////////////// - - /// @name capacity - /// @{ - - /*! - @brief checks whether the container is empty - - Checks if a JSON value has no elements. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `true` - boolean | `false` - string | `false` - number | `false` - object | result of function `object_t::empty()` - array | result of function `array_t::empty()` - - @note This function does not return whether a string stored as JSON value - is empty - it returns whether the JSON container itself is empty which is - false in the case of a string. - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their `empty()` functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `begin() == end()`. - - @liveexample{The following code uses `empty()` to check if a JSON - object contains any elements.,empty} - - @sa @ref size() -- returns the number of elements - - @since version 1.0.0 - */ - bool empty() const noexcept - { - switch (m_type) - { - case value_t::null: - { - // null values are empty - return true; - } - - case value_t::array: - { - // delegate call to array_t::empty() - return m_value.array->empty(); - } - - case value_t::object: - { - // delegate call to object_t::empty() - return m_value.object->empty(); - } - - default: - { - // all other types are nonempty - return false; - } - } - } - - /*! - @brief returns the number of elements - - Returns the number of elements in a JSON value. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `0` - boolean | `1` - string | `1` - number | `1` - object | result of function object_t::size() - array | result of function array_t::size() - - @note This function does not return the length of a string stored as JSON - value - it returns the number of elements in the JSON value which is 1 in - the case of a string. - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their size() functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `std::distance(begin(), end())`. - - @liveexample{The following code calls `size()` on the different value - types.,size} - - @sa @ref empty() -- checks whether the container is empty - @sa @ref max_size() -- returns the maximal number of elements - - @since version 1.0.0 - */ - size_type size() const noexcept - { - switch (m_type) - { - case value_t::null: - { - // null values are empty - return 0; - } - - case value_t::array: - { - // delegate call to array_t::size() - return m_value.array->size(); - } - - case value_t::object: - { - // delegate call to object_t::size() - return m_value.object->size(); - } - - default: - { - // all other types have size 1 - return 1; - } - } - } - - /*! - @brief returns the maximum possible number of elements - - Returns the maximum number of elements a JSON value is able to hold due to - system or library implementation limitations, i.e. `std::distance(begin(), - end())` for the JSON value. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `0` (same as `size()`) - boolean | `1` (same as `size()`) - string | `1` (same as `size()`) - number | `1` (same as `size()`) - object | result of function `object_t::max_size()` - array | result of function `array_t::max_size()` - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their `max_size()` functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of returning `b.size()` where `b` is the largest - possible JSON value. - - @liveexample{The following code calls `max_size()` on the different value - types. Note the output is implementation specific.,max_size} - - @sa @ref size() -- returns the number of elements - - @since version 1.0.0 - */ - size_type max_size() const noexcept - { - switch (m_type) - { - case value_t::array: - { - // delegate call to array_t::max_size() - return m_value.array->max_size(); - } - - case value_t::object: - { - // delegate call to object_t::max_size() - return m_value.object->max_size(); - } - - default: - { - // all other types have max_size() == size() - return size(); - } - } - } - - /// @} - - - /////////////// - // modifiers // - /////////////// - - /// @name modifiers - /// @{ - - /*! - @brief clears the contents - - Clears the content of a JSON value and resets it to the default value as - if @ref basic_json(value_t) would have been called: - - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` - - @note Floating-point numbers are set to `0.0` which will be serialized to - `0`. The vale type remains @ref number_float_t. - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows the effect of `clear()` to different - JSON types.,clear} - - @since version 1.0.0 - */ - void clear() noexcept - { - switch (m_type) - { - case value_t::number_integer: - { - m_value.number_integer = 0; - break; - } - - case value_t::number_unsigned: - { - m_value.number_unsigned = 0; - break; - } - - case value_t::number_float: - { - m_value.number_float = 0.0; - break; - } - - case value_t::boolean: - { - m_value.boolean = false; - break; - } - - case value_t::string: - { - m_value.string->clear(); - break; - } - - case value_t::array: - { - m_value.array->clear(); - break; - } - - case value_t::object: - { - m_value.object->clear(); - break; - } - - default: - { - break; - } - } - } - - /*! - @brief add an object to an array - - Appends the given element @a val to the end of the JSON value. If the - function is called on a JSON null value, an empty array is created before - appending @a val. - - @param[in] val the value to add to the JSON array - - @throw std::domain_error when called on a type other than JSON array or - null; example: `"cannot use push_back() with number"` - - @complexity Amortized constant. - - @liveexample{The example shows how `push_back()` and `+=` can be used to - add elements to a JSON array. Note how the `null` value was silently - converted to a JSON array.,push_back} - - @since version 1.0.0 - */ - void push_back(basic_json&& val) - { - // push_back only works for null objects or arrays - if (not(is_null() or is_array())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an array - if (is_null()) - { - m_type = value_t::array; - m_value = value_t::array; - assert_invariant(); - } - - // add element to array (move semantics) - m_value.array->push_back(std::move(val)); - // invalidate object - val.m_type = value_t::null; - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - reference operator+=(basic_json&& val) - { - push_back(std::move(val)); - return *this; - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - void push_back(const basic_json& val) - { - // push_back only works for null objects or arrays - if (not(is_null() or is_array())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an array - if (is_null()) - { - m_type = value_t::array; - m_value = value_t::array; - assert_invariant(); - } - - // add element to array - m_value.array->push_back(val); - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - reference operator+=(const basic_json& val) - { - push_back(val); - return *this; - } - - /*! - @brief add an object to an object - - Inserts the given element @a val to the JSON object. If the function is - called on a JSON null value, an empty object is created before inserting - @a val. - - @param[in] val the value to add to the JSON object - - @throw std::domain_error when called on a type other than JSON object or - null; example: `"cannot use push_back() with number"` - - @complexity Logarithmic in the size of the container, O(log(`size()`)). - - @liveexample{The example shows how `push_back()` and `+=` can be used to - add elements to a JSON object. Note how the `null` value was silently - converted to a JSON object.,push_back__object_t__value} - - @since version 1.0.0 - */ - void push_back(const typename object_t::value_type& val) - { - // push_back only works for null objects or objects - if (not(is_null() or is_object())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an object - if (is_null()) - { - m_type = value_t::object; - m_value = value_t::object; - assert_invariant(); - } - - // add element to array - m_value.object->insert(val); - } - - /*! - @brief add an object to an object - @copydoc push_back(const typename object_t::value_type&) - */ - reference operator+=(const typename object_t::value_type& val) - { - push_back(val); - return *this; - } - - /*! - @brief add an object to an object - - This function allows to use `push_back` with an initializer list. In case - - 1. the current value is an object, - 2. the initializer list @a init contains only two elements, and - 3. the first element of @a init is a string, - - @a init is converted into an object element and added using - @ref push_back(const typename object_t::value_type&). Otherwise, @a init - is converted to a JSON value and added using @ref push_back(basic_json&&). - - @param init an initializer list - - @complexity Linear in the size of the initializer list @a init. - - @note This function is required to resolve an ambiguous overload error, - because pairs like `{"key", "value"}` can be both interpreted as - `object_t::value_type` or `std::initializer_list`, see - https://github.com/nlohmann/json/issues/235 for more information. - - @liveexample{The example shows how initializer lists are treated as - objects when possible.,push_back__initializer_list} - */ - void push_back(std::initializer_list init) - { - if (is_object() and init.size() == 2 and init.begin()->is_string()) - { - const string_t key = *init.begin(); - push_back(typename object_t::value_type(key, *(init.begin() + 1))); - } - else - { - push_back(basic_json(init)); - } - } - - /*! - @brief add an object to an object - @copydoc push_back(std::initializer_list) - */ - reference operator+=(std::initializer_list init) - { - push_back(init); - return *this; - } - - /*! - @brief inserts element - - Inserts element @a val before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] val element to insert - @return iterator pointing to the inserted @a val. - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @complexity Constant plus linear in the distance between pos and end of the - container. - - @liveexample{The example shows how `insert()` is used.,insert} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, const basic_json& val) - { - // insert only works for arrays - if (is_array()) - { - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); - return result; - } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - } - - /*! - @brief inserts element - @copydoc insert(const_iterator, const basic_json&) - */ - iterator insert(const_iterator pos, basic_json&& val) - { - return insert(pos, val); - } - - /*! - @brief inserts elements - - Inserts @a cnt copies of @a val before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] cnt number of copies of @a val to insert - @param[in] val element to insert - @return iterator pointing to the first element inserted, or @a pos if - `cnt==0` - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @complexity Linear in @a cnt plus linear in the distance between @a pos - and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__count} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, size_type cnt, const basic_json& val) - { - // insert only works for arrays - if (is_array()) - { - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); - return result; - } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - } - - /*! - @brief inserts elements - - Inserts elements from range `[first, last)` before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] first begin of the range of elements to insert - @param[in] last end of the range of elements to insert - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - @throw std::domain_error if @a first and @a last do not belong to the same - JSON value; example: `"iterators do not fit"` - @throw std::domain_error if @a first or @a last are iterators into - container for which insert is called; example: `"passed iterators may not - belong to container"` - - @return iterator pointing to the first element inserted, or @a pos if - `first==last` - - @complexity Linear in `std::distance(first, last)` plus linear in the - distance between @a pos and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__range} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, const_iterator first, const_iterator last) - { - // insert only works for arrays - if (not is_array()) - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // check if range iterators belong to the same JSON object - if (first.m_object != last.m_object) - { - throw std::domain_error("iterators do not fit"); - } - - if (first.m_object == this or last.m_object == this) - { - throw std::domain_error("passed iterators may not belong to container"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert( - pos.m_it.array_iterator, - first.m_it.array_iterator, - last.m_it.array_iterator); - return result; - } - - /*! - @brief inserts elements - - Inserts elements from initializer list @a ilist before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] ilist initializer list to insert the values from - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @return iterator pointing to the first element inserted, or @a pos if - `ilist` is empty - - @complexity Linear in `ilist.size()` plus linear in the distance between - @a pos and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__ilist} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, std::initializer_list ilist) - { - // insert only works for arrays - if (not is_array()) - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); - return result; - } - - /*! - @brief exchanges the values - - Exchanges the contents of the JSON value with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other JSON value to exchange the contents with - - @complexity Constant. - - @liveexample{The example below shows how JSON values can be swapped with - `swap()`.,swap__reference} - - @since version 1.0.0 - */ - void swap(reference other) noexcept ( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - std::swap(m_type, other.m_type); - std::swap(m_value, other.m_value); - assert_invariant(); - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON array with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other array to exchange the contents with - - @throw std::domain_error when JSON value is not an array; example: `"cannot - use swap() with string"` - - @complexity Constant. - - @liveexample{The example below shows how arrays can be swapped with - `swap()`.,swap__array_t} - - @since version 1.0.0 - */ - void swap(array_t& other) - { - // swap only works for arrays - if (is_array()) - { - std::swap(*(m_value.array), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON object with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other object to exchange the contents with - - @throw std::domain_error when JSON value is not an object; example: - `"cannot use swap() with string"` - - @complexity Constant. - - @liveexample{The example below shows how objects can be swapped with - `swap()`.,swap__object_t} - - @since version 1.0.0 - */ - void swap(object_t& other) - { - // swap only works for objects - if (is_object()) - { - std::swap(*(m_value.object), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON string with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other string to exchange the contents with - - @throw std::domain_error when JSON value is not a string; example: `"cannot - use swap() with boolean"` - - @complexity Constant. - - @liveexample{The example below shows how strings can be swapped with - `swap()`.,swap__string_t} - - @since version 1.0.0 - */ - void swap(string_t& other) - { - // swap only works for strings - if (is_string()) - { - std::swap(*(m_value.string), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /// @} - - - ////////////////////////////////////////// - // lexicographical comparison operators // - ////////////////////////////////////////// - - /// @name lexicographical comparison operators - /// @{ - - private: - /*! - @brief comparison operator for JSON types - - Returns an ordering that is similar to Python: - - order: null < boolean < number < object < array < string - - furthermore, each type is not smaller than itself - - @since version 1.0.0 - */ - friend bool operator<(const value_t lhs, const value_t rhs) noexcept - { - static constexpr std::array order = {{ - 0, // null - 3, // object - 4, // array - 5, // string - 1, // boolean - 2, // integer - 2, // unsigned - 2, // float - } - }; - - // discarded values are not comparable - if (lhs == value_t::discarded or rhs == value_t::discarded) - { - return false; - } - - return order[static_cast(lhs)] < order[static_cast(rhs)]; - } - - public: - /*! - @brief comparison: equal - - Compares two JSON values for equality according to the following rules: - - Two JSON values are equal if (1) they are from the same type and (2) - their stored values are the same. - - Integer and floating-point numbers are automatically converted before - comparison. Floating-point numbers are compared indirectly: two - floating-point numbers `f1` and `f2` are considered equal if neither - `f1 > f2` nor `f2 > f1` holds. - - Two JSON null values are equal. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are equal - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__equal} - - @since version 1.0.0 - */ - friend bool operator==(const_reference lhs, const_reference rhs) noexcept - { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); - - if (lhs_type == rhs_type) - { - switch (lhs_type) - { - case value_t::array: - { - return *lhs.m_value.array == *rhs.m_value.array; - } - case value_t::object: - { - return *lhs.m_value.object == *rhs.m_value.object; - } - case value_t::null: - { - return true; - } - case value_t::string: - { - return *lhs.m_value.string == *rhs.m_value.string; - } - case value_t::boolean: - { - return lhs.m_value.boolean == rhs.m_value.boolean; - } - case value_t::number_integer: - { - return lhs.m_value.number_integer == rhs.m_value.number_integer; - } - case value_t::number_unsigned: - { - return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; - } - case value_t::number_float: - { - return lhs.m_value.number_float == rhs.m_value.number_float; - } - default: - { - return false; - } - } - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) - { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); - } - - return false; - } - - /*! - @brief comparison: equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__equal__nullptr_t} - - @since version 1.0.0 - */ - friend bool operator==(const_reference v, std::nullptr_t) noexcept - { - return v.is_null(); - } - - /*! - @brief comparison: equal - @copydoc operator==(const_reference, std::nullptr_t) - */ - friend bool operator==(std::nullptr_t, const_reference v) noexcept - { - return v.is_null(); - } - - /*! - @brief comparison: not equal - - Compares two JSON values for inequality by calculating `not (lhs == rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are not equal - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__notequal} - - @since version 1.0.0 - */ - friend bool operator!=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs == rhs); - } - - /*! - @brief comparison: not equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `not v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is not null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__notequal__nullptr_t} - - @since version 1.0.0 - */ - friend bool operator!=(const_reference v, std::nullptr_t) noexcept - { - return not v.is_null(); - } - - /*! - @brief comparison: not equal - @copydoc operator!=(const_reference, std::nullptr_t) - */ - friend bool operator!=(std::nullptr_t, const_reference v) noexcept - { - return not v.is_null(); - } - - /*! - @brief comparison: less than - - Compares whether one JSON value @a lhs is less than another JSON value @a - rhs according to the following rules: - - If @a lhs and @a rhs have the same type, the values are compared using - the default `<` operator. - - Integer and floating-point numbers are automatically converted before - comparison - - In case @a lhs and @a rhs have different types, the values are ignored - and the order of the types is considered, see - @ref operator<(const value_t, const value_t). - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__less} - - @since version 1.0.0 - */ - friend bool operator<(const_reference lhs, const_reference rhs) noexcept - { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); - - if (lhs_type == rhs_type) - { - switch (lhs_type) - { - case value_t::array: - { - return *lhs.m_value.array < *rhs.m_value.array; - } - case value_t::object: - { - return *lhs.m_value.object < *rhs.m_value.object; - } - case value_t::null: - { - return false; - } - case value_t::string: - { - return *lhs.m_value.string < *rhs.m_value.string; - } - case value_t::boolean: - { - return lhs.m_value.boolean < rhs.m_value.boolean; - } - case value_t::number_integer: - { - return lhs.m_value.number_integer < rhs.m_value.number_integer; - } - case value_t::number_unsigned: - { - return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; - } - case value_t::number_float: - { - return lhs.m_value.number_float < rhs.m_value.number_float; - } - default: - { - return false; - } - } - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; - } - - // We only reach this line if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - return operator<(lhs_type, rhs_type); - } - - /*! - @brief comparison: less than or equal - - Compares whether one JSON value @a lhs is less than or equal to another - JSON value by calculating `not (rhs < lhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than or equal to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__greater} - - @since version 1.0.0 - */ - friend bool operator<=(const_reference lhs, const_reference rhs) noexcept - { - return not (rhs < lhs); - } - - /*! - @brief comparison: greater than - - Compares whether one JSON value @a lhs is greater than another - JSON value by calculating `not (lhs <= rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__lessequal} - - @since version 1.0.0 - */ - friend bool operator>(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs <= rhs); - } - - /*! - @brief comparison: greater than or equal - - Compares whether one JSON value @a lhs is greater than or equal to another - JSON value by calculating `not (lhs < rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than or equal to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__greaterequal} - - @since version 1.0.0 - */ - friend bool operator>=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs < rhs); - } - - /// @} - - - /////////////////// - // serialization // - /////////////////// - - /// @name serialization - /// @{ - - /*! - @brief serialize to stream - - Serialize the given JSON value @a j to the output stream @a o. The JSON - value will be serialized using the @ref dump member function. The - indentation of the output can be controlled with the member variable - `width` of the output stream @a o. For instance, using the manipulator - `std::setw(4)` on @a o sets the indentation level to `4` and the - serialization result is the same as calling `dump(4)`. - - @note During serializaion, the locale and the precision of the output - stream @a o are changed. The original values are restored when the - function returns. - - @param[in,out] o stream to serialize to - @param[in] j JSON value to serialize - - @return the stream @a o - - @complexity Linear. - - @liveexample{The example below shows the serialization with different - parameters to `width` to adjust the indentation level.,operator_serialize} - - @since version 1.0.0 - */ - friend std::ostream& operator<<(std::ostream& o, const basic_json& j) - { - // read width member and use it as indentation parameter if nonzero - const bool pretty_print = (o.width() > 0); - const auto indentation = (pretty_print ? o.width() : 0); - - // reset width to 0 for subsequent calls to this stream - o.width(0); - - // fix locale problems - const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator)); - // set precision - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - const auto old_precision = o.precision(std::numeric_limits::digits10); - - // do the actual serialization - j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale and precision - o.imbue(old_locale); - o.precision(old_precision); - return o; - } - - /*! - @brief serialize to stream - @copydoc operator<<(std::ostream&, const basic_json&) - */ - friend std::ostream& operator>>(const basic_json& j, std::ostream& o) - { - return o << j; - } - - /// @} - - - ///////////////////// - // deserialization // - ///////////////////// - - /// @name deserialization - /// @{ - - /*! - @brief deserialize from string - - @param[in] s string to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__string__parser_callback_t} - - @sa @ref parse(std::istream&, const parser_callback_t) for a version that - reads from an input stream - - @since version 1.0.0 - */ - static basic_json parse(const string_t& s, - const parser_callback_t cb = nullptr) - { - return parser(s, cb).parse(); - } - - /*! - @brief deserialize from stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__istream__parser_callback_t} - - @sa @ref parse(const string_t&, const parser_callback_t) for a version - that reads from a string - - @since version 1.0.0 - */ - static basic_json parse(std::istream& i, - const parser_callback_t cb = nullptr) - { - return parser(i, cb).parse(); - } - - /*! - @copydoc parse(std::istream&, const parser_callback_t) - */ - static basic_json parse(std::istream&& i, - const parser_callback_t cb = nullptr) - { - return parser(i, cb).parse(); - } - - /*! - @brief deserialize from stream - - Deserializes an input stream to a JSON value. - - @param[in,out] i input stream to read a serialized JSON value from - @param[in,out] j JSON value to write the deserialized input to - - @throw std::invalid_argument in case of parse errors - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below shows how a JSON value is constructed by - reading a serialization from a stream.,operator_deserialize} - - @sa parse(std::istream&, const parser_callback_t) for a variant with a - parser callback function to filter values while parsing - - @since version 1.0.0 - */ - friend std::istream& operator<<(basic_json& j, std::istream& i) - { - j = parser(i).parse(); - return i; - } - - /*! - @brief deserialize from stream - @copydoc operator<<(basic_json&, std::istream&) - */ - friend std::istream& operator>>(std::istream& i, basic_json& j) - { - j = parser(i).parse(); - return i; - } - - /// @} - - - private: - /////////////////////////// - // convenience functions // - /////////////////////////// - - /*! - @brief return the type as string - - Returns the type name as string to be used in error messages - usually to - indicate that a function was called on a wrong JSON type. - - @return basically a string representation of a the @ref m_type member - - @complexity Constant. - - @since version 1.0.0 - */ - std::string type_name() const - { - switch (m_type) - { - case value_t::null: - return "null"; - case value_t::object: - return "object"; - case value_t::array: - return "array"; - case value_t::string: - return "string"; - case value_t::boolean: - return "boolean"; - case value_t::discarded: - return "discarded"; - default: - return "number"; - } - } - - /*! - @brief calculates the extra space to escape a JSON string - - @param[in] s the string to escape - @return the number of characters required to escape string @a s - - @complexity Linear in the length of string @a s. - */ - static std::size_t extra_space(const string_t& s) noexcept - { - return std::accumulate(s.begin(), s.end(), size_t{}, - [](size_t res, typename string_t::value_type c) - { - switch (c) - { - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - return res + 1; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // from c (1 byte) to \uxxxx (6 bytes) - return res + 5; - } - else - { - return res; - } - } - } - }); - } - - /*! - @brief escape a string - - Escape a string by replacing certain special characters by a sequence of - an escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. - - @param[in] s the string to escape - @return the escaped string - - @complexity Linear in the length of string @a s. - */ - static string_t escape_string(const string_t& s) - { - const auto space = extra_space(s); - if (space == 0) - { - return s; - } - - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; - - for (const auto& c : s) - { - switch (c) - { - // quotation mark (0x22) - case '"': - { - result[pos + 1] = '"'; - pos += 2; - break; - } - - // reverse solidus (0x5c) - case '\\': - { - // nothing to change - pos += 2; - break; - } - - // backspace (0x08) - case '\b': - { - result[pos + 1] = 'b'; - pos += 2; - break; - } - - // formfeed (0x0c) - case '\f': - { - result[pos + 1] = 'f'; - pos += 2; - break; - } - - // newline (0x0a) - case '\n': - { - result[pos + 1] = 'n'; - pos += 2; - break; - } - - // carriage return (0x0d) - case '\r': - { - result[pos + 1] = 'r'; - pos += 2; - break; - } - - // horizontal tab (0x09) - case '\t': - { - result[pos + 1] = 't'; - pos += 2; - break; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // convert a number 0..15 to its hex representation - // (0..f) - static const char hexify[16] = - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - - // print character c as \uxxxx - for (const char m : - { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] - }) - { - result[++pos] = m; - } - - ++pos; - } - else - { - // all other characters are added as-is - result[pos++] = c; - } - break; - } - } - } - - return result; - } - - /*! - @brief internal implementation of the serialization function - - This function is called by the public member function dump and organizes - the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. Note that - - - strings and object keys are escaped using `escape_string()` - - integer numbers are converted implicitly via `operator<<` - - floating-point numbers are converted to a string using `"%g"` format - - @param[out] o stream to write to - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] indent_step the indent level - @param[in] current_indent the current indent level (only used internally) - */ - void dump(std::ostream& o, - const bool pretty_print, - const unsigned int indent_step, - const unsigned int current_indent = 0) const - { - // variable to hold indentation for recursive calls - unsigned int new_indent = current_indent; - - switch (m_type) - { - case value_t::object: - { - if (m_value.object->empty()) - { - o << "{}"; - return; - } - - o << "{"; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) - { - if (i != m_value.object->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' ') << "\"" - << escape_string(i->first) << "\":" - << (pretty_print ? " " : ""); - i->second.dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') + "}"; - return; - } - - case value_t::array: - { - if (m_value.array->empty()) - { - o << "[]"; - return; - } - - o << "["; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) - { - if (i != m_value.array->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' '); - i->dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') << "]"; - return; - } - - case value_t::string: - { - o << string_t("\"") << escape_string(*m_value.string) << "\""; - return; - } - - case value_t::boolean: - { - o << (m_value.boolean ? "true" : "false"); - return; - } - - case value_t::number_integer: - { - o << m_value.number_integer; - return; - } - - case value_t::number_unsigned: - { - o << m_value.number_unsigned; - return; - } - - case value_t::number_float: - { - if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); - } - else - { - o << m_value.number_float; - } - return; - } - - case value_t::discarded: - { - o << ""; - return; - } - - case value_t::null: - { - o << "null"; - return; - } - } - } - - private: - ////////////////////// - // member variables // - ////////////////////// - - /// the type of the current element - value_t m_type = value_t::null; - - /// the value of the current element - json_value m_value = {}; - - - private: - /////////////// - // iterators // - /////////////// - - /*! - @brief an iterator for primitive JSON types - - This class models an iterator for primitive JSON types (boolean, number, - string). It's only purpose is to allow the iterator/const_iterator classes - to "iterate" over primitive values. Internally, the iterator is modeled by - a `difference_type` variable. Value begin_value (`0`) models the begin, - end_value (`1`) models past the end. - */ - class primitive_iterator_t - { - public: - /// set iterator to a defined beginning - void set_begin() noexcept - { - m_it = begin_value; - } - - /// set iterator to a defined past the end - void set_end() noexcept - { - m_it = end_value; - } - - /// return whether the iterator can be dereferenced - constexpr bool is_begin() const noexcept - { - return (m_it == begin_value); - } - - /// return whether the iterator is at end - constexpr bool is_end() const noexcept - { - return (m_it == end_value); - } - - /// return reference to the value to change and compare - operator difference_type& () noexcept - { - return m_it; - } - - /// return value to compare - constexpr operator difference_type () const noexcept - { - return m_it; - } - - private: - static constexpr difference_type begin_value = 0; - static constexpr difference_type end_value = begin_value + 1; - - /// iterator as signed integer type - difference_type m_it = std::numeric_limits::denorm_min(); - }; - - /*! - @brief an iterator value - - @note This structure could easily be a union, but MSVC currently does not - allow unions members with complex constructors, see - https://github.com/nlohmann/json/pull/105. - */ - struct internal_iterator - { - /// iterator for JSON objects - typename object_t::iterator object_iterator; - /// iterator for JSON arrays - typename array_t::iterator array_iterator; - /// generic iterator for all other types - primitive_iterator_t primitive_iterator; - - /// create an uninitialized internal_iterator - internal_iterator() noexcept - : object_iterator(), array_iterator(), primitive_iterator() - {} - }; - - /// proxy class for the iterator_wrapper functions - template - class iteration_proxy - { - private: - /// helper class for iteration - class iteration_proxy_internal - { - private: - /// the iterator - IteratorType anchor; - /// an index for arrays (used to create key names) - size_t array_index = 0; - - public: - explicit iteration_proxy_internal(IteratorType it) noexcept - : anchor(it) - {} - - /// dereference operator (needed for range-based for) - iteration_proxy_internal& operator*() - { - return *this; - } - - /// increment operator (needed for range-based for) - iteration_proxy_internal& operator++() - { - ++anchor; - ++array_index; - - return *this; - } - - /// inequality operator (needed for range-based for) - bool operator!= (const iteration_proxy_internal& o) const - { - return anchor != o.anchor; - } - - /// return key of the iterator - typename basic_json::string_t key() const - { - assert(anchor.m_object != nullptr); - - switch (anchor.m_object->type()) - { - // use integer array index as key - case value_t::array: - { - return std::to_string(array_index); - } - - // use key from the object - case value_t::object: - { - return anchor.key(); - } - - // use an empty key for all primitive types - default: - { - return ""; - } - } - } - - /// return value of the iterator - typename IteratorType::reference value() const - { - return anchor.value(); - } - }; - - /// the container to iterate - typename IteratorType::reference container; - - public: - /// construct iteration proxy from a container - explicit iteration_proxy(typename IteratorType::reference cont) - : container(cont) - {} - - /// return iterator begin (needed for range-based for) - iteration_proxy_internal begin() noexcept - { - return iteration_proxy_internal(container.begin()); - } - - /// return iterator end (needed for range-based for) - iteration_proxy_internal end() noexcept - { - return iteration_proxy_internal(container.end()); - } - }; - - public: - /*! - @brief a const random access iterator for the @ref basic_json class - - This class implements a const iterator for the @ref basic_json class. From - this class, the @ref iterator class is derived. - - @note An iterator is called *initialized* when a pointer to a JSON value - has been set (e.g., by a constructor or a copy assignment). If the - iterator is default-constructed, it is *uninitialized* and most - methods are undefined. The library uses assertions to detect calls - on uninitialized iterators. - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - @since version 1.0.0 - */ - class const_iterator : public std::iterator - { - /// allow basic_json to access private members - friend class basic_json; - - public: - /// the type of the values when the iterator is dereferenced - using value_type = typename basic_json::value_type; - /// a type to represent differences between iterators - using difference_type = typename basic_json::difference_type; - /// defines a pointer to the type iterated over (value_type) - using pointer = typename basic_json::const_pointer; - /// defines a reference to the type iterated over (value_type) - using reference = typename basic_json::const_reference; - /// the category of the iterator - using iterator_category = std::bidirectional_iterator_tag; - - /// default constructor - const_iterator() = default; - - /*! - @brief constructor for a given JSON instance - @param[in] object pointer to a JSON object for this iterator - @pre object != nullptr - @post The iterator is initialized; i.e. `m_object != nullptr`. - */ - explicit const_iterator(pointer object) noexcept - : m_object(object) - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = typename object_t::iterator(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = typename array_t::iterator(); - break; - } - - default: - { - m_it.primitive_iterator = primitive_iterator_t(); - break; - } - } - } - - /*! - @brief copy constructor given a non-const iterator - @param[in] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - explicit const_iterator(const iterator& other) noexcept - : m_object(other.m_object) - { - if (m_object != nullptr) - { - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = other.m_it.object_iterator; - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = other.m_it.array_iterator; - break; - } - - default: - { - m_it.primitive_iterator = other.m_it.primitive_iterator; - break; - } - } - } - } - - /*! - @brief copy constructor - @param[in] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - const_iterator(const const_iterator& other) noexcept - : m_object(other.m_object), m_it(other.m_it) - {} - - /*! - @brief copy assignment - @param[in,out] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - const_iterator& operator=(const_iterator other) noexcept( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - std::swap(m_object, other.m_object); - std::swap(m_it, other.m_it); - return *this; - } - - private: - /*! - @brief set the iterator to the first value - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - void set_begin() noexcept - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = m_object->m_value.object->begin(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = m_object->m_value.array->begin(); - break; - } - - case basic_json::value_t::null: - { - // set to end so begin()==end() is true: null is empty - m_it.primitive_iterator.set_end(); - break; - } - - default: - { - m_it.primitive_iterator.set_begin(); - break; - } - } - } - - /*! - @brief set the iterator past the last value - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - void set_end() noexcept - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = m_object->m_value.object->end(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = m_object->m_value.array->end(); - break; - } - - default: - { - m_it.primitive_iterator.set_end(); - break; - } - } - } - - public: - /*! - @brief return a reference to the value pointed to by the iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference operator*() const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - assert(m_it.object_iterator != m_object->m_value.object->end()); - return m_it.object_iterator->second; - } - - case basic_json::value_t::array: - { - assert(m_it.array_iterator != m_object->m_value.array->end()); - return *m_it.array_iterator; - } - - case basic_json::value_t::null: - { - throw std::out_of_range("cannot get value"); - } - - default: - { - if (m_it.primitive_iterator.is_begin()) - { - return *m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief dereference the iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - pointer operator->() const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - assert(m_it.object_iterator != m_object->m_value.object->end()); - return &(m_it.object_iterator->second); - } - - case basic_json::value_t::array: - { - assert(m_it.array_iterator != m_object->m_value.array->end()); - return &*m_it.array_iterator; - } - - default: - { - if (m_it.primitive_iterator.is_begin()) - { - return m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief post-increment (it++) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator++(int) - { - auto result = *this; - ++(*this); - return result; - } - - /*! - @brief pre-increment (++it) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator++() - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - std::advance(m_it.object_iterator, 1); - break; - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, 1); - break; - } - - default: - { - ++m_it.primitive_iterator; - break; - } - } - - return *this; - } - - /*! - @brief post-decrement (it--) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator--(int) - { - auto result = *this; - --(*this); - return result; - } - - /*! - @brief pre-decrement (--it) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator--() - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - std::advance(m_it.object_iterator, -1); - break; - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, -1); - break; - } - - default: - { - --m_it.primitive_iterator; - break; - } - } - - return *this; - } - - /*! - @brief comparison: equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator==(const const_iterator& other) const - { - // if objects are not the same, the comparison is undefined - if (m_object != other.m_object) - { - throw std::domain_error("cannot compare iterators of different containers"); - } - - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - return (m_it.object_iterator == other.m_it.object_iterator); - } - - case basic_json::value_t::array: - { - return (m_it.array_iterator == other.m_it.array_iterator); - } - - default: - { - return (m_it.primitive_iterator == other.m_it.primitive_iterator); - } - } - } - - /*! - @brief comparison: not equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator!=(const const_iterator& other) const - { - return not operator==(other); - } - - /*! - @brief comparison: smaller - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator<(const const_iterator& other) const - { - // if objects are not the same, the comparison is undefined - if (m_object != other.m_object) - { - throw std::domain_error("cannot compare iterators of different containers"); - } - - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot compare order of object iterators"); - } - - case basic_json::value_t::array: - { - return (m_it.array_iterator < other.m_it.array_iterator); - } - - default: - { - return (m_it.primitive_iterator < other.m_it.primitive_iterator); - } - } - } - - /*! - @brief comparison: less than or equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator<=(const const_iterator& other) const - { - return not other.operator < (*this); - } - - /*! - @brief comparison: greater than - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator>(const const_iterator& other) const - { - return not operator<=(other); - } - - /*! - @brief comparison: greater than or equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator>=(const const_iterator& other) const - { - return not operator<(other); - } - - /*! - @brief add to iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator+=(difference_type i) - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use offsets with object iterators"); - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, i); - break; - } - - default: - { - m_it.primitive_iterator += i; - break; - } - } - - return *this; - } - - /*! - @brief subtract from iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator-=(difference_type i) - { - return operator+=(-i); - } - - /*! - @brief add to iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator+(difference_type i) - { - auto result = *this; - result += i; - return result; - } - - /*! - @brief subtract from iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator-(difference_type i) - { - auto result = *this; - result -= i; - return result; - } - - /*! - @brief return difference - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - difference_type operator-(const const_iterator& other) const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use offsets with object iterators"); - } - - case basic_json::value_t::array: - { - return m_it.array_iterator - other.m_it.array_iterator; - } - - default: - { - return m_it.primitive_iterator - other.m_it.primitive_iterator; - } - } - } - - /*! - @brief access to successor - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference operator[](difference_type n) const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use operator[] for object iterators"); - } - - case basic_json::value_t::array: - { - return *std::next(m_it.array_iterator, n); - } - - case basic_json::value_t::null: - { - throw std::out_of_range("cannot get value"); - } - - default: - { - if (m_it.primitive_iterator == -n) - { - return *m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief return the key of an object iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - typename object_t::key_type key() const - { - assert(m_object != nullptr); - - if (m_object->is_object()) - { - return m_it.object_iterator->first; - } - else - { - throw std::domain_error("cannot use key() for non-object iterators"); - } - } - - /*! - @brief return the value of an iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference value() const - { - return operator*(); - } - - private: - /// associated JSON instance - pointer m_object = nullptr; - /// the actual iterator of the associated instance - internal_iterator m_it = internal_iterator(); - }; - - /*! - @brief a mutable random access iterator for the @ref basic_json class - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): - It is possible to write to the pointed-to element. - - @since version 1.0.0 - */ - class iterator : public const_iterator - { - public: - using base_iterator = const_iterator; - using pointer = typename basic_json::pointer; - using reference = typename basic_json::reference; - - /// default constructor - iterator() = default; - - /// constructor for a given JSON instance - explicit iterator(pointer object) noexcept - : base_iterator(object) - {} - - /// copy constructor - iterator(const iterator& other) noexcept - : base_iterator(other) - {} - - /// copy assignment - iterator& operator=(iterator other) noexcept( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - base_iterator::operator=(other); - return *this; - } - - /// return a reference to the value pointed to by the iterator - reference operator*() const - { - return const_cast(base_iterator::operator*()); - } - - /// dereference the iterator - pointer operator->() const - { - return const_cast(base_iterator::operator->()); - } - - /// post-increment (it++) - iterator operator++(int) - { - iterator result = *this; - base_iterator::operator++(); - return result; - } - - /// pre-increment (++it) - iterator& operator++() - { - base_iterator::operator++(); - return *this; - } - - /// post-decrement (it--) - iterator operator--(int) - { - iterator result = *this; - base_iterator::operator--(); - return result; - } - - /// pre-decrement (--it) - iterator& operator--() - { - base_iterator::operator--(); - return *this; - } - - /// add to iterator - iterator& operator+=(difference_type i) - { - base_iterator::operator+=(i); - return *this; - } - - /// subtract from iterator - iterator& operator-=(difference_type i) - { - base_iterator::operator-=(i); - return *this; - } - - /// add to iterator - iterator operator+(difference_type i) - { - auto result = *this; - result += i; - return result; - } - - /// subtract from iterator - iterator operator-(difference_type i) - { - auto result = *this; - result -= i; - return result; - } - - /// return difference - difference_type operator-(const iterator& other) const - { - return base_iterator::operator-(other); - } - - /// access to successor - reference operator[](difference_type n) const - { - return const_cast(base_iterator::operator[](n)); - } - - /// return the value of an iterator - reference value() const - { - return const_cast(base_iterator::value()); - } - }; - - /*! - @brief a template for a reverse iterator class - - @tparam Base the base iterator type to reverse. Valid types are @ref - iterator (to create @ref reverse_iterator) and @ref const_iterator (to - create @ref const_reverse_iterator). - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): - It is possible to write to the pointed-to element (only if @a Base is - @ref iterator). - - @since version 1.0.0 - */ - template - class json_reverse_iterator : public std::reverse_iterator - { - public: - /// shortcut to the reverse iterator adaptor - using base_iterator = std::reverse_iterator; - /// the reference type for the pointed-to element - using reference = typename Base::reference; - - /// create reverse iterator from iterator - json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept - : base_iterator(it) - {} - - /// create reverse iterator from base class - json_reverse_iterator(const base_iterator& it) noexcept - : base_iterator(it) - {} - - /// post-increment (it++) - json_reverse_iterator operator++(int) - { - return base_iterator::operator++(1); - } - - /// pre-increment (++it) - json_reverse_iterator& operator++() - { - base_iterator::operator++(); - return *this; - } - - /// post-decrement (it--) - json_reverse_iterator operator--(int) - { - return base_iterator::operator--(1); - } - - /// pre-decrement (--it) - json_reverse_iterator& operator--() - { - base_iterator::operator--(); - return *this; - } - - /// add to iterator - json_reverse_iterator& operator+=(difference_type i) - { - base_iterator::operator+=(i); - return *this; - } - - /// add to iterator - json_reverse_iterator operator+(difference_type i) const - { - auto result = *this; - result += i; - return result; - } - - /// subtract from iterator - json_reverse_iterator operator-(difference_type i) const - { - auto result = *this; - result -= i; - return result; - } - - /// return difference - difference_type operator-(const json_reverse_iterator& other) const - { - return this->base() - other.base(); - } - - /// access to successor - reference operator[](difference_type n) const - { - return *(this->operator+(n)); - } - - /// return the key of an object iterator - typename object_t::key_type key() const - { - auto it = --this->base(); - return it.key(); - } - - /// return the value of an iterator - reference value() const - { - auto it = --this->base(); - return it.operator * (); - } - }; - - - private: - ////////////////////// - // lexer and parser // - ////////////////////// - - /*! - @brief lexical analysis - - This class organizes the lexical analysis during JSON deserialization. The - core of it is a scanner generated by [re2c](http://re2c.org) that - processes a buffer and recognizes tokens according to RFC 7159. - */ - class lexer - { - public: - /// token types for the parser - enum class token_type - { - uninitialized, ///< indicating the scanner is uninitialized - literal_true, ///< the `true` literal - literal_false, ///< the `false` literal - literal_null, ///< the `null` literal - value_string, ///< a string -- use get_string() for actual value - value_number, ///< a number -- use get_number() for actual value - begin_array, ///< the character for array begin `[` - begin_object, ///< the character for object begin `{` - end_array, ///< the character for array end `]` - end_object, ///< the character for object end `}` - name_separator, ///< the name separator `:` - value_separator, ///< the value separator `,` - parse_error, ///< indicating a parse error - end_of_input ///< indicating the end of the input buffer - }; - - /// the char type to use in the lexer - using lexer_char_t = unsigned char; - - /// constructor with a given buffer - explicit lexer(const string_t& s) noexcept - : m_stream(nullptr), m_buffer(s) - { - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + s.size(); - } - - /// constructor with a given stream - explicit lexer(std::istream* s) noexcept - : m_stream(s), m_buffer() - { - assert(m_stream != nullptr); - std::getline(*m_stream, m_buffer); - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + m_buffer.size(); - } - - /// default constructor - lexer() = default; - - // switch off unwanted functions - lexer(const lexer&) = delete; - lexer operator=(const lexer&) = delete; - - /*! - @brief create a string from one or two Unicode code points - - There are two cases: (1) @a codepoint1 is in the Basic Multilingual - Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) - @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to - represent a code point above U+FFFF. - - @param[in] codepoint1 the code point (can be high surrogate) - @param[in] codepoint2 the code point (can be low surrogate or 0) - - @return string representation of the code point; the length of the - result string is between 1 and 4 characters. - - @throw std::out_of_range if code point is > 0x10ffff; example: `"code - points above 0x10FFFF are invalid"` - @throw std::invalid_argument if the low surrogate is invalid; example: - `""missing or wrong low surrogate""` - - @complexity Constant. - - @see - */ - static string_t to_unicode(const std::size_t codepoint1, - const std::size_t codepoint2 = 0) - { - // calculate the code point from the given code points - std::size_t codepoint = codepoint1; - - // check if codepoint1 is a high surrogate - if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) - { - // check if codepoint2 is a low surrogate - if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) - { - codepoint = - // high surrogate occupies the most significant 22 bits - (codepoint1 << 10) - // low surrogate occupies the least significant 15 bits - + codepoint2 - // there is still the 0xD800, 0xDC00 and 0x10000 noise - // in the result so we have to subtract with: - // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 - - 0x35FDC00; - } - else - { - throw std::invalid_argument("missing or wrong low surrogate"); - } - } - - string_t result; - - if (codepoint < 0x80) - { - // 1-byte characters: 0xxxxxxx (ASCII) - result.append(1, static_cast(codepoint)); - } - else if (codepoint <= 0x7ff) - { - // 2-byte characters: 110xxxxx 10xxxxxx - result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else if (codepoint <= 0xffff) - { - // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); - result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else if (codepoint <= 0x10ffff) - { - // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); - result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); - result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else - { - throw std::out_of_range("code points above 0x10FFFF are invalid"); - } - - return result; - } - - /// return name of values of type token_type (only used for errors) - static std::string token_type_name(const token_type t) - { - switch (t) - { - case token_type::uninitialized: - return ""; - case token_type::literal_true: - return "true literal"; - case token_type::literal_false: - return "false literal"; - case token_type::literal_null: - return "null literal"; - case token_type::value_string: - return "string literal"; - case token_type::value_number: - return "number literal"; - case token_type::begin_array: - return "'['"; - case token_type::begin_object: - return "'{'"; - case token_type::end_array: - return "']'"; - case token_type::end_object: - return "'}'"; - case token_type::name_separator: - return "':'"; - case token_type::value_separator: - return "','"; - case token_type::parse_error: - return ""; - case token_type::end_of_input: - return "end of input"; - default: - { - // catch non-enum values - return "unknown token"; // LCOV_EXCL_LINE - } - } - } - - /*! - This function implements a scanner for JSON. It is specified using - regular expressions that try to follow RFC 7159 as close as possible. - These regular expressions are then translated into a minimized - deterministic finite automaton (DFA) by the tool - [re2c](http://re2c.org). As a result, the translated code for this - function consists of a large block of code with `goto` jumps. - - @return the class of the next token read from the buffer - - @complexity Linear in the length of the input.\n - - Proposition: The loop below will always terminate for finite input.\n - - Proof (by contradiction): Assume a finite input. To loop forever, the - loop must never hit code with a `break` statement. The only code - snippets without a `break` statement are the continue statements for - whitespace and byte-order-marks. To loop forever, the input must be an - infinite sequence of whitespace or byte-order-marks. This contradicts - the assumption of finite input, q.e.d. - */ - token_type scan() noexcept - { - while (true) - { - // pointer for backtracking information - m_marker = nullptr; - - // remember the begin of the token - m_start = m_cursor; - assert(m_start != nullptr); - - - { - lexer_char_t yych; - unsigned int yyaccept = 0; - static const unsigned char yybm[] = - { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 32, 32, 0, 0, 32, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 160, 128, 0, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 0, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - }; - if ((m_limit - m_cursor) < 5) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 32) - { - goto basic_json_parser_6; - } - if (yych <= '\\') - { - if (yych <= '-') - { - if (yych <= '"') - { - if (yych <= 0x00) - { - goto basic_json_parser_2; - } - if (yych <= '!') - { - goto basic_json_parser_4; - } - goto basic_json_parser_9; - } - else - { - if (yych <= '+') - { - goto basic_json_parser_4; - } - if (yych <= ',') - { - goto basic_json_parser_10; - } - goto basic_json_parser_12; - } - } - else - { - if (yych <= '9') - { - if (yych <= '/') - { - goto basic_json_parser_4; - } - if (yych <= '0') - { - goto basic_json_parser_13; - } - goto basic_json_parser_15; - } - else - { - if (yych <= ':') - { - goto basic_json_parser_17; - } - if (yych == '[') - { - goto basic_json_parser_19; - } - goto basic_json_parser_4; - } - } - } - else - { - if (yych <= 't') - { - if (yych <= 'f') - { - if (yych <= ']') - { - goto basic_json_parser_21; - } - if (yych <= 'e') - { - goto basic_json_parser_4; - } - goto basic_json_parser_23; - } - else - { - if (yych == 'n') - { - goto basic_json_parser_24; - } - if (yych <= 's') - { - goto basic_json_parser_4; - } - goto basic_json_parser_25; - } - } - else - { - if (yych <= '|') - { - if (yych == '{') - { - goto basic_json_parser_26; - } - goto basic_json_parser_4; - } - else - { - if (yych <= '}') - { - goto basic_json_parser_28; - } - if (yych == 0xEF) - { - goto basic_json_parser_30; - } - goto basic_json_parser_4; - } - } - } -basic_json_parser_2: - ++m_cursor; - { - last_token_type = token_type::end_of_input; - break; - } -basic_json_parser_4: - ++m_cursor; -basic_json_parser_5: - { - last_token_type = token_type::parse_error; - break; - } -basic_json_parser_6: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 32) - { - goto basic_json_parser_6; - } - { - continue; - } -basic_json_parser_9: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych <= 0x1F) - { - goto basic_json_parser_5; - } - goto basic_json_parser_32; -basic_json_parser_10: - ++m_cursor; - { - last_token_type = token_type::value_separator; - break; - } -basic_json_parser_12: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_5; - } - if (yych <= '0') - { - goto basic_json_parser_13; - } - if (yych <= '9') - { - goto basic_json_parser_15; - } - goto basic_json_parser_5; -basic_json_parser_13: - yyaccept = 1; - yych = *(m_marker = ++m_cursor); - if (yych <= 'D') - { - if (yych == '.') - { - goto basic_json_parser_37; - } - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - } -basic_json_parser_14: - { - last_token_type = token_type::value_number; - break; - } -basic_json_parser_15: - yyaccept = 1; - m_marker = ++m_cursor; - if ((m_limit - m_cursor) < 3) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 64) - { - goto basic_json_parser_15; - } - if (yych <= 'D') - { - if (yych == '.') - { - goto basic_json_parser_37; - } - goto basic_json_parser_14; - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - goto basic_json_parser_14; - } -basic_json_parser_17: - ++m_cursor; - { - last_token_type = token_type::name_separator; - break; - } -basic_json_parser_19: - ++m_cursor; - { - last_token_type = token_type::begin_array; - break; - } -basic_json_parser_21: - ++m_cursor; - { - last_token_type = token_type::end_array; - break; - } -basic_json_parser_23: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'a') - { - goto basic_json_parser_39; - } - goto basic_json_parser_5; -basic_json_parser_24: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'u') - { - goto basic_json_parser_40; - } - goto basic_json_parser_5; -basic_json_parser_25: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'r') - { - goto basic_json_parser_41; - } - goto basic_json_parser_5; -basic_json_parser_26: - ++m_cursor; - { - last_token_type = token_type::begin_object; - break; - } -basic_json_parser_28: - ++m_cursor; - { - last_token_type = token_type::end_object; - break; - } -basic_json_parser_30: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 0xBB) - { - goto basic_json_parser_42; - } - goto basic_json_parser_5; -basic_json_parser_31: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; -basic_json_parser_32: - if (yybm[0 + yych] & 128) - { - goto basic_json_parser_31; - } - if (yych <= 0x1F) - { - goto basic_json_parser_33; - } - if (yych <= '"') - { - goto basic_json_parser_34; - } - goto basic_json_parser_36; -basic_json_parser_33: - m_cursor = m_marker; - if (yyaccept == 0) - { - goto basic_json_parser_5; - } - else - { - goto basic_json_parser_14; - } -basic_json_parser_34: - ++m_cursor; - { - last_token_type = token_type::value_string; - break; - } -basic_json_parser_36: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= 'e') - { - if (yych <= '/') - { - if (yych == '"') - { - goto basic_json_parser_31; - } - if (yych <= '.') - { - goto basic_json_parser_33; - } - goto basic_json_parser_31; - } - else - { - if (yych <= '\\') - { - if (yych <= '[') - { - goto basic_json_parser_33; - } - goto basic_json_parser_31; - } - else - { - if (yych == 'b') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - } - } - else - { - if (yych <= 'q') - { - if (yych <= 'f') - { - goto basic_json_parser_31; - } - if (yych == 'n') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 's') - { - if (yych <= 'r') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 't') - { - goto basic_json_parser_31; - } - if (yych <= 'u') - { - goto basic_json_parser_43; - } - goto basic_json_parser_33; - } - } - } -basic_json_parser_37: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_44; - } - goto basic_json_parser_33; -basic_json_parser_38: - yych = *++m_cursor; - if (yych <= ',') - { - if (yych == '+') - { - goto basic_json_parser_46; - } - goto basic_json_parser_33; - } - else - { - if (yych <= '-') - { - goto basic_json_parser_46; - } - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_47; - } - goto basic_json_parser_33; - } -basic_json_parser_39: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_49; - } - goto basic_json_parser_33; -basic_json_parser_40: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_50; - } - goto basic_json_parser_33; -basic_json_parser_41: - yych = *++m_cursor; - if (yych == 'u') - { - goto basic_json_parser_51; - } - goto basic_json_parser_33; -basic_json_parser_42: - yych = *++m_cursor; - if (yych == 0xBF) - { - goto basic_json_parser_52; - } - goto basic_json_parser_33; -basic_json_parser_43: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_54; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_54; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_54; - } - goto basic_json_parser_33; - } -basic_json_parser_44: - yyaccept = 1; - m_marker = ++m_cursor; - if ((m_limit - m_cursor) < 3) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= 'D') - { - if (yych <= '/') - { - goto basic_json_parser_14; - } - if (yych <= '9') - { - goto basic_json_parser_44; - } - goto basic_json_parser_14; - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - goto basic_json_parser_14; - } -basic_json_parser_46: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych >= ':') - { - goto basic_json_parser_33; - } -basic_json_parser_47: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '/') - { - goto basic_json_parser_14; - } - if (yych <= '9') - { - goto basic_json_parser_47; - } - goto basic_json_parser_14; -basic_json_parser_49: - yych = *++m_cursor; - if (yych == 's') - { - goto basic_json_parser_55; - } - goto basic_json_parser_33; -basic_json_parser_50: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_56; - } - goto basic_json_parser_33; -basic_json_parser_51: - yych = *++m_cursor; - if (yych == 'e') - { - goto basic_json_parser_58; - } - goto basic_json_parser_33; -basic_json_parser_52: - ++m_cursor; - { - continue; - } -basic_json_parser_54: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_60; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_60; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_60; - } - goto basic_json_parser_33; - } -basic_json_parser_55: - yych = *++m_cursor; - if (yych == 'e') - { - goto basic_json_parser_61; - } - goto basic_json_parser_33; -basic_json_parser_56: - ++m_cursor; - { - last_token_type = token_type::literal_null; - break; - } -basic_json_parser_58: - ++m_cursor; - { - last_token_type = token_type::literal_true; - break; - } -basic_json_parser_60: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_63; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_63; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_63; - } - goto basic_json_parser_33; - } -basic_json_parser_61: - ++m_cursor; - { - last_token_type = token_type::literal_false; - break; - } -basic_json_parser_63: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_31; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - } - - } - - return last_token_type; - } - - /// append data from the stream to the internal buffer - void yyfill() noexcept - { - if (m_stream == nullptr or not * m_stream) - { - return; - } - - const auto offset_start = m_start - m_content; - const auto offset_marker = m_marker - m_start; - const auto offset_cursor = m_cursor - m_start; - - m_buffer.erase(0, static_cast(offset_start)); - std::string line; - assert(m_stream != nullptr); - std::getline(*m_stream, line); - m_buffer += "\n" + line; // add line with newline symbol - - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_content; - m_marker = m_start + offset_marker; - m_cursor = m_start + offset_cursor; - m_limit = m_start + m_buffer.size() - 1; - } - - /// return string representation of last read token - string_t get_token_string() const - { - assert(m_start != nullptr); - return string_t(reinterpret_cast(m_start), - static_cast(m_cursor - m_start)); - } - - /*! - @brief return string value for string tokens - - The function iterates the characters between the opening and closing - quotes of the string value. The complete string is the range - [m_start,m_cursor). Consequently, we iterate from m_start+1 to - m_cursor-1. - - We differentiate two cases: - - 1. Escaped characters. In this case, a new character is constructed - according to the nature of the escape. Some escapes create new - characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied - as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape - `"\\uxxxx"` need special care. In this case, to_unicode takes care - of the construction of the values. - 2. Unescaped characters are copied as is. - - @pre `m_cursor - m_start >= 2`, meaning the length of the last token - is at least 2 bytes which is trivially true for any string (which - consists of at least two quotes). - - " c1 c2 c3 ... " - ^ ^ - m_start m_cursor - - @complexity Linear in the length of the string.\n - - Lemma: The loop body will always terminate.\n - - Proof (by contradiction): Assume the loop body does not terminate. As - the loop body does not contain another loop, one of the called - functions must never return. The called functions are `std::strtoul` - and to_unicode. Neither function can loop forever, so the loop body - will never loop forever which contradicts the assumption that the loop - body does not terminate, q.e.d.\n - - Lemma: The loop condition for the for loop is eventually false.\n - - Proof (by contradiction): Assume the loop does not terminate. Due to - the above lemma, this can only be due to a tautological loop - condition; that is, the loop condition i < m_cursor - 1 must always be - true. Let x be the change of i for any loop iteration. Then - m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This - can be rephrased to m_cursor - m_start - 2 > x. With the - precondition, we x <= 0, meaning that the loop condition holds - indefinitly if i is always decreased. However, observe that the value - of i is strictly increasing with each iteration, as it is incremented - by 1 in the iteration expression and never decremented inside the loop - body. Hence, the loop condition will eventually be false which - contradicts the assumption that the loop condition is a tautology, - q.e.d. - - @return string value of current token without opening and closing - quotes - @throw std::out_of_range if to_unicode fails - */ - string_t get_string() const - { - assert(m_cursor - m_start >= 2); - - string_t result; - result.reserve(static_cast(m_cursor - m_start - 2)); - - // iterate the result between the quotes - for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) - { - // process escaped characters - if (*i == '\\') - { - // read next character - ++i; - - switch (*i) - { - // the default escapes - case 't': - { - result += "\t"; - break; - } - case 'b': - { - result += "\b"; - break; - } - case 'f': - { - result += "\f"; - break; - } - case 'n': - { - result += "\n"; - break; - } - case 'r': - { - result += "\r"; - break; - } - case '\\': - { - result += "\\"; - break; - } - case '/': - { - result += "/"; - break; - } - case '"': - { - result += "\""; - break; - } - - // unicode - case 'u': - { - // get code xxxx from uxxxx - auto codepoint = std::strtoul(std::string(reinterpret_cast(i + 1), - 4).c_str(), nullptr, 16); - - // check if codepoint is a high surrogate - if (codepoint >= 0xD800 and codepoint <= 0xDBFF) - { - // make sure there is a subsequent unicode - if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') - { - throw std::invalid_argument("missing low surrogate"); - } - - // get code yyyy from uxxxx\uyyyy - auto codepoint2 = std::strtoul(std::string(reinterpret_cast - (i + 7), 4).c_str(), nullptr, 16); - result += to_unicode(codepoint, codepoint2); - // skip the next 10 characters (xxxx\uyyyy) - i += 10; - } - else - { - // add unicode character(s) - result += to_unicode(codepoint); - // skip the next four characters (xxxx) - i += 4; - } - break; - } - } - } - else - { - // all other characters are just copied to the end of the - // string - result.append(1, static_cast(*i)); - } - } - - return result; - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - long double str_to_float_t(long double* /* type */, char** endptr) const - { - return std::strtold(reinterpret_cast(m_start), endptr); - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - double str_to_float_t(double* /* type */, char** endptr) const - { - return std::strtod(reinterpret_cast(m_start), endptr); - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } - - /*! - @brief return number value for number tokens - - This function translates the last token into the most appropriate - number type (either integer, unsigned integer or floating point), - which is passed back to the caller via the result parameter. - - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. - - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). - - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. - */ - void get_number(basic_json& result) const - { - assert(m_start != nullptr); - - const lexer::lexer_char_t* curptr = m_start; - - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; - - // maximum absolute value of the relevant integer type - number_unsigned_t max; - - // temporarily store the type to avoid unecessary bitfield access - value_t type; - - // look for sign - if (*curptr == '-') - { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; - } - else - { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); - } - - // count the significant figures - for (; curptr < m_cursor; curptr++) - { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') - { - if (*curptr == '.') - { - // don't count '.' but change to float - type = value_t::number_float; - continue; - } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; - break; - } - - // skip if definitely not an integer - if (type != value_t::number_float) - { - // multiply last value by ten and add the new digit - auto temp = value * 10 + *curptr - '0'; - - // test for overflow - if (temp < value || temp > max) - { - // overflow - type = value_t::number_float; - } - else - { - // no overflow - save it - value = temp; - } - } - } - - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - result.m_value.number_integer = -static_cast(value); - } - else - { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); - } - - // save the type - result.m_type = type; - } - - private: - /// optional input stream - std::istream* m_stream = nullptr; - /// the buffer - string_t m_buffer; - /// the buffer pointer - const lexer_char_t* m_content = nullptr; - /// pointer to the beginning of the current symbol - const lexer_char_t* m_start = nullptr; - /// pointer for backtracking information - const lexer_char_t* m_marker = nullptr; - /// pointer to the current symbol - const lexer_char_t* m_cursor = nullptr; - /// pointer to the end of the buffer - const lexer_char_t* m_limit = nullptr; - /// the last token type - token_type last_token_type = token_type::end_of_input; - }; - - /*! - @brief syntax analysis - - This class implements a recursive decent parser. - */ - class parser - { - public: - /// constructor for strings - parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(s) - { - // read first token - get_token(); - } - - /// a parser reading from an input stream - parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(&_is) - { - // read first token - get_token(); - } - - /// public parser interface - basic_json parse() - { - basic_json result = parse_internal(true); - result.assert_invariant(); - - expect(lexer::token_type::end_of_input); - - // return parser result and replace it with null in case the - // top-level value was discarded by the callback function - return result.is_discarded() ? basic_json() : std::move(result); - } - - private: - /// the actual parser - basic_json parse_internal(bool keep) - { - auto result = basic_json(value_t::discarded); - - switch (last_token) - { - case lexer::token_type::begin_object: - { - if (keep and (not callback or (keep = callback(depth++, parse_event_t::object_start, result)))) - { - // explicitly set result to object to cope with {} - result.m_type = value_t::object; - result.m_value = value_t::object; - } - - // read next token - get_token(); - - // closing } -> we are done - if (last_token == lexer::token_type::end_object) - { - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - // no comma is expected here - unexpect(lexer::token_type::value_separator); - - // otherwise: parse key-value pairs - do - { - // ugly, but could be fixed with loop reorganization - if (last_token == lexer::token_type::value_separator) - { - get_token(); - } - - // store key - expect(lexer::token_type::value_string); - const auto key = m_lexer.get_string(); - - bool keep_tag = false; - if (keep) - { - if (callback) - { - basic_json k(key); - keep_tag = callback(depth, parse_event_t::key, k); - } - else - { - keep_tag = true; - } - } - - // parse separator (:) - get_token(); - expect(lexer::token_type::name_separator); - - // parse and add value - get_token(); - auto value = parse_internal(keep); - if (keep and keep_tag and not value.is_discarded()) - { - result[key] = std::move(value); - } - } - while (last_token == lexer::token_type::value_separator); - - // closing } - expect(lexer::token_type::end_object); - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result = basic_json(value_t::discarded); - } - - return result; - } - - case lexer::token_type::begin_array: - { - if (keep and (not callback or (keep = callback(depth++, parse_event_t::array_start, result)))) - { - // explicitly set result to object to cope with [] - result.m_type = value_t::array; - result.m_value = value_t::array; - } - - // read next token - get_token(); - - // closing ] -> we are done - if (last_token == lexer::token_type::end_array) - { - get_token(); - if (callback and not callback(--depth, parse_event_t::array_end, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - // no comma is expected here - unexpect(lexer::token_type::value_separator); - - // otherwise: parse values - do - { - // ugly, but could be fixed with loop reorganization - if (last_token == lexer::token_type::value_separator) - { - get_token(); - } - - // parse value - auto value = parse_internal(keep); - if (keep and not value.is_discarded()) - { - result.push_back(std::move(value)); - } - } - while (last_token == lexer::token_type::value_separator); - - // closing ] - expect(lexer::token_type::end_array); - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) - { - result = basic_json(value_t::discarded); - } - - return result; - } - - case lexer::token_type::literal_null: - { - get_token(); - result.m_type = value_t::null; - break; - } - - case lexer::token_type::value_string: - { - const auto s = m_lexer.get_string(); - get_token(); - result = basic_json(s); - break; - } - - case lexer::token_type::literal_true: - { - get_token(); - result.m_type = value_t::boolean; - result.m_value = true; - break; - } - - case lexer::token_type::literal_false: - { - get_token(); - result.m_type = value_t::boolean; - result.m_value = false; - break; - } - - case lexer::token_type::value_number: - { - m_lexer.get_number(result); - get_token(); - break; - } - - default: - { - // the last token was unexpected - unexpect(last_token); - } - } - - if (keep and callback and not callback(depth, parse_event_t::value, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - /// get next token from lexer - typename lexer::token_type get_token() noexcept - { - last_token = m_lexer.scan(); - return last_token; - } - - void expect(typename lexer::token_type t) const - { - if (t != last_token) - { - std::string error_msg = "parse error - unexpected "; - error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + - "'") : - lexer::token_type_name(last_token)); - error_msg += "; expected " + lexer::token_type_name(t); - throw std::invalid_argument(error_msg); - } - } - - void unexpect(typename lexer::token_type t) const - { - if (t == last_token) - { - std::string error_msg = "parse error - unexpected "; - error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + - "'") : - lexer::token_type_name(last_token)); - throw std::invalid_argument(error_msg); - } - } - - private: - /// current level of recursion - int depth = 0; - /// callback function - const parser_callback_t callback = nullptr; - /// the type of the last read token - typename lexer::token_type last_token = lexer::token_type::uninitialized; - /// the lexer - lexer m_lexer; - }; - - public: - /*! - @brief JSON Pointer - - A JSON pointer defines a string syntax for identifying a specific value - within a JSON document. It can be used with functions `at` and - `operator[]`. Furthermore, JSON pointers are the base for JSON patches. - - @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - - @since version 2.0.0 - */ - class json_pointer - { - /// allow basic_json to access private members - friend class basic_json; - - public: - /*! - @brief create JSON pointer - - Create a JSON pointer according to the syntax described in - [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). - - @param[in] s string representing the JSON pointer; if omitted, the - empty string is assumed which references the whole JSON - value - - @throw std::domain_error if reference token is nonempty and does not - begin with a slash (`/`); example: `"JSON pointer must be empty or - begin with /"` - @throw std::domain_error if a tilde (`~`) is not followed by `0` - (representing `~`) or `1` (representing `/`); example: `"escape error: - ~ must be followed with 0 or 1"` - - @liveexample{The example shows the construction several valid JSON - pointers as well as the exceptional behavior.,json_pointer} - - @since version 2.0.0 - */ - explicit json_pointer(const std::string& s = "") - : reference_tokens(split(s)) - {} - - /*! - @brief return a string representation of the JSON pointer - - @invariant For each JSON pointer `ptr`, it holds: - @code {.cpp} - ptr == json_pointer(ptr.to_string()); - @endcode - - @return a string representation of the JSON pointer - - @liveexample{The example shows the result of `to_string`., - json_pointer__to_string} - - @since version 2.0.0 - */ - std::string to_string() const noexcept - { - return std::accumulate(reference_tokens.begin(), - reference_tokens.end(), std::string{}, - [](const std::string & a, const std::string & b) - { - return a + "/" + escape(b); - }); - } - - /// @copydoc to_string() - operator std::string() const - { - return to_string(); - } - - private: - /// remove and return last reference pointer - std::string pop_back() - { - if (is_root()) - { - throw std::domain_error("JSON pointer has no parent"); - } - - auto last = reference_tokens.back(); - reference_tokens.pop_back(); - return last; - } - - /// return whether pointer points to the root document - bool is_root() const - { - return reference_tokens.empty(); - } - - json_pointer top() const - { - if (is_root()) - { - throw std::domain_error("JSON pointer has no parent"); - } - - json_pointer result = *this; - result.reference_tokens = {reference_tokens[0]}; - return result; - } - - /*! - @brief create and return a reference to the pointed to value - - @complexity Linear in the number of reference tokens. - */ - reference get_and_create(reference j) const - { - pointer result = &j; - - // in case no reference tokens exist, return a reference to the - // JSON value j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) - { - switch (result->m_type) - { - case value_t::null: - { - if (reference_token == "0") - { - // start a new array if reference token is 0 - result = &result->operator[](0); - } - else - { - // start a new object otherwise - result = &result->operator[](reference_token); - } - break; - } - - case value_t::object: - { - // create an entry in the object - result = &result->operator[](reference_token); - break; - } - - case value_t::array: - { - // create an entry in the array - result = &result->operator[](static_cast(std::stoi(reference_token))); - break; - } - - /* - The following code is only reached if there exists a - reference token _and_ the current value is primitive. In - this case, we have an error situation, because primitive - values may only occur as single value; that is, with an - empty list of reference tokens. - */ - default: - { - throw std::domain_error("invalid value to unflatten"); - } - } - } - - return *result; - } - - /*! - @brief return a reference to the pointed to value - - @param[in] ptr a JSON value - - @return reference to the JSON value pointed to by the JSON pointer - - @complexity Linear in the length of the JSON pointer. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - */ - reference get_unchecked(pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case value_t::array: - { - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - if (reference_token == "-") - { - // explicityly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); - } - else - { - // convert array index to number; unchecked access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); - } - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - reference get_checked(pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - /*! - @brief return a const reference to the pointed to value - - @param[in] ptr a JSON value - - @return const reference to the JSON value pointed to by the JSON - pointer - */ - const_reference get_unchecked(const_pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" cannot be used for const access - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // use unchecked array access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - const_reference get_checked(const_pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - /// split the string input to reference tokens - static std::vector split(std::string reference_string) - { - std::vector result; - - // special case: empty reference string -> no reference tokens - if (reference_string.empty()) - { - return result; - } - - // check if nonempty reference string begins with slash - if (reference_string[0] != '/') - { - throw std::domain_error("JSON pointer must be empty or begin with '/'"); - } - - // extract the reference tokens: - // - slash: position of the last read slash (or end of string) - // - start: position after the previous slash - for ( - // search for the first slash after the first character - size_t slash = reference_string.find_first_of("/", 1), - // set the beginning of the first reference token - start = 1; - // we can stop if start == string::npos+1 = 0 - start != 0; - // set the beginning of the next reference token - // (will eventually be 0 if slash == std::string::npos) - start = slash + 1, - // find next slash - slash = reference_string.find_first_of("/", start)) - { - // use the text between the beginning of the reference token - // (start) and the last slash (slash). - auto reference_token = reference_string.substr(start, slash - start); - - // check reference tokens are properly escaped - for (size_t pos = reference_token.find_first_of("~"); - pos != std::string::npos; - pos = reference_token.find_first_of("~", pos + 1)) - { - assert(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (pos == reference_token.size() - 1 or - (reference_token[pos + 1] != '0' and - reference_token[pos + 1] != '1')) - { - throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); - } - } - - // finally, store the reference token - unescape(reference_token); - result.push_back(reference_token); - } - - return result; - } - - private: - /*! - @brief replace all occurrences of a substring by another string - - @param[in,out] s the string to manipulate - @param[in] f the substring to replace with @a t - @param[in] t the string to replace @a f - - @return The string @a s where all occurrences of @a f are replaced - with @a t. - - @pre The search string @a f must not be empty. - - @since version 2.0.0 - */ - static void replace_substring(std::string& s, - const std::string& f, - const std::string& t) - { - assert(not f.empty()); - - for ( - size_t pos = s.find(f); // find first occurrence of f - pos != std::string::npos; // make sure f was found - s.replace(pos, f.size(), t), // replace with t - pos = s.find(f, pos + t.size()) // find next occurrence of f - ); - } - - /// escape tilde and slash - static std::string escape(std::string s) - { - // escape "~"" to "~0" and "/" to "~1" - replace_substring(s, "~", "~0"); - replace_substring(s, "/", "~1"); - return s; - } - - /// unescape tilde and slash - static void unescape(std::string& s) - { - // first transform any occurrence of the sequence '~1' to '/' - replace_substring(s, "~1", "/"); - // then transform any occurrence of the sequence '~0' to '~' - replace_substring(s, "~0", "~"); - } - - /*! - @param[in] reference_string the reference string to the current value - @param[in] value the value to consider - @param[in,out] result the result object to insert values to - - @note Empty objects or arrays are flattened to `null`. - */ - static void flatten(const std::string& reference_string, - const basic_json& value, - basic_json& result) - { - switch (value.m_type) - { - case value_t::array: - { - if (value.m_value.array->empty()) - { - // flatten empty array as null - result[reference_string] = nullptr; - } - else - { - // iterate array and use index as reference string - for (size_t i = 0; i < value.m_value.array->size(); ++i) - { - flatten(reference_string + "/" + std::to_string(i), - value.m_value.array->operator[](i), result); - } - } - break; - } - - case value_t::object: - { - if (value.m_value.object->empty()) - { - // flatten empty object as null - result[reference_string] = nullptr; - } - else - { - // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) - { - flatten(reference_string + "/" + escape(element.first), - element.second, result); - } - } - break; - } - - default: - { - // add primitive value with its reference string - result[reference_string] = value; - break; - } - } - } - - /*! - @param[in] value flattened JSON - - @return unflattened JSON - */ - static basic_json unflatten(const basic_json& value) - { - if (not value.is_object()) - { - throw std::domain_error("only objects can be unflattened"); - } - - basic_json result; - - // iterate the JSON object values - for (const auto& element : *value.m_value.object) - { - if (not element.second.is_primitive()) - { - throw std::domain_error("values in object must be primitive"); - } - - // assign value to reference pointed to by JSON pointer; Note - // that if the JSON pointer is "" (i.e., points to the whole - // value), function get_and_create returns a reference to - // result itself. An assignment will then create a primitive - // value. - json_pointer(element.first).get_and_create(result) = element.second; - } - - return result; - } - - private: - /// the reference tokens - std::vector reference_tokens {}; - }; - - ////////////////////////// - // JSON Pointer support // - ////////////////////////// - - /// @name JSON Pointer functions - /// @{ - - /*! - @brief access specified element via JSON Pointer - - Uses a JSON pointer to retrieve a reference to the respective JSON value. - No bound checking is performed. Similar to @ref operator[](const typename - object_t::key_type&), `null` values are created in arrays and objects if - necessary. - - In particular: - - If the JSON pointer points to an object key that does not exist, it - is created an filled with a `null` value before a reference to it - is returned. - - If the JSON pointer points to an array index that does not exist, it - is created an filled with a `null` value before a reference to it - is returned. All indices between the current maximum and the given - index are also filled with `null`. - - The special value `-` is treated as a synonym for the index past the - end. - - @param[in] ptr a JSON pointer - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,operatorjson_pointer} - - @since version 2.0.0 - */ - reference operator[](const json_pointer& ptr) - { - return ptr.get_unchecked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Uses a JSON pointer to retrieve a reference to the respective JSON value. - No bound checking is performed. The function does not change the JSON - value; no `null` values are created. In particular, the the special value - `-` yields an exception. - - @param[in] ptr JSON pointer to the desired element - - @return const reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} - - @since version 2.0.0 - */ - const_reference operator[](const json_pointer& ptr) const - { - return ptr.get_unchecked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Returns a reference to the element at with specified JSON pointer @a ptr, - with bounds checking. - - @param[in] ptr JSON pointer to the desired element - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,at_json_pointer} - - @since version 2.0.0 - */ - reference at(const json_pointer& ptr) - { - return ptr.get_checked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Returns a const reference to the element at with specified JSON pointer @a - ptr, with bounds checking. - - @param[in] ptr JSON pointer to the desired element - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,at_json_pointer_const} - - @since version 2.0.0 - */ - const_reference at(const json_pointer& ptr) const - { - return ptr.get_checked(this); - } - - /*! - @brief return flattened JSON value - - The function creates a JSON object whose keys are JSON pointers (see [RFC - 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all - primitive. The original JSON value can be restored using the @ref - unflatten() function. - - @return an object that maps JSON pointers to primitve values - - @note Empty objects and arrays are flattened to `null` and will not be - reconstructed correctly by the @ref unflatten() function. - - @complexity Linear in the size the JSON value. - - @liveexample{The following code shows how a JSON object is flattened to an - object whose keys consist of JSON pointers.,flatten} - - @sa @ref unflatten() for the reverse function - - @since version 2.0.0 - */ - basic_json flatten() const - { - basic_json result(value_t::object); - json_pointer::flatten("", *this, result); - return result; - } - - /*! - @brief unflatten a previously flattened JSON value - - The function restores the arbitrary nesting of a JSON value that has been - flattened before using the @ref flatten() function. The JSON value must - meet certain constraints: - 1. The value must be an object. - 2. The keys must be JSON pointers (see - [RFC 6901](https://tools.ietf.org/html/rfc6901)) - 3. The mapped values must be primitive JSON types. - - @return the original JSON from a flattened version - - @note Empty objects and arrays are flattened by @ref flatten() to `null` - values and can not unflattened to their original type. Apart from - this example, for a JSON value `j`, the following is always true: - `j == j.flatten().unflatten()`. - - @complexity Linear in the size the JSON value. - - @liveexample{The following code shows how a flattened JSON object is - unflattened into the original nested JSON object.,unflatten} - - @sa @ref flatten() for the reverse function - - @since version 2.0.0 - */ - basic_json unflatten() const - { - return json_pointer::unflatten(*this); - } - - /// @} - - ////////////////////////// - // JSON Patch functions // - ////////////////////////// - - /// @name JSON Patch functions - /// @{ - - /*! - @brief applies a JSON patch - - [JSON Patch](http://jsonpatch.com) defines a JSON document structure for - expressing a sequence of operations to apply to a JSON) document. With - this funcion, a JSON Patch is applied to the current JSON value by - executing all operations from the patch. - - @param[in] json_patch JSON patch document - @return patched document - - @note The application of a patch is atomic: Either all operations succeed - and the patched document is returned or an exception is thrown. In - any case, the original value is not changed: the patch is applied - to a copy of the value. - - @throw std::out_of_range if a JSON pointer inside the patch could not - be resolved successfully in the current JSON value; example: `"key baz - not found"` - @throw invalid_argument if the JSON patch is malformed (e.g., mandatory - attributes are missing); example: `"operation add must have member path"` - - @complexity Linear in the size of the JSON value and the length of the - JSON patch. As usually only a fraction of the JSON value is affected by - the patch, the complexity can usually be neglected. - - @liveexample{The following code shows how a JSON patch is applied to a - value.,patch} - - @sa @ref diff -- create a JSON patch by comparing two JSON values - - @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) - @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) - - @since version 2.0.0 - */ - basic_json patch(const basic_json& json_patch) const - { - // make a working copy to apply the patch to - basic_json result = *this; - - // the valid JSON Patch operations - enum class patch_operations {add, remove, replace, move, copy, test, invalid}; - - const auto get_op = [](const std::string op) - { - if (op == "add") - { - return patch_operations::add; - } - if (op == "remove") - { - return patch_operations::remove; - } - if (op == "replace") - { - return patch_operations::replace; - } - if (op == "move") - { - return patch_operations::move; - } - if (op == "copy") - { - return patch_operations::copy; - } - if (op == "test") - { - return patch_operations::test; - } - - return patch_operations::invalid; - }; - - // wrapper for "add" operation; add value at ptr - const auto operation_add = [&result](json_pointer & ptr, basic_json val) - { - // adding to the root of the target document means replacing it - if (ptr.is_root()) - { - result = val; - } - else - { - // make sure the top element of the pointer exists - json_pointer top_pointer = ptr.top(); - if (top_pointer != ptr) - { - basic_json& x = result.at(top_pointer); - } - - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result[ptr]; - - switch (parent.m_type) - { - case value_t::null: - case value_t::object: - { - // use operator[] to add value - parent[last_path] = val; - break; - } - - case value_t::array: - { - if (last_path == "-") - { - // special case: append to back - parent.push_back(val); - } - else - { - const auto idx = std::stoi(last_path); - if (static_cast(idx) > parent.size()) - { - // avoid undefined behavior - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - else - { - // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); - } - } - break; - } - - default: - { - // if there exists a parent it cannot be primitive - assert(false); // LCOV_EXCL_LINE - } - } - } - }; - - // wrapper for "remove" operation; remove value at ptr - const auto operation_remove = [&result](json_pointer & ptr) - { - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result.at(ptr); - - // remove child - if (parent.is_object()) - { - // perform range check - auto it = parent.find(last_path); - if (it != parent.end()) - { - parent.erase(it); - } - else - { - throw std::out_of_range("key '" + last_path + "' not found"); - } - } - else if (parent.is_array()) - { - // note erase performs range check - parent.erase(static_cast(std::stoi(last_path))); - } - }; - - // type check - if (not json_patch.is_array()) - { - // a JSON patch must be an array of objects - throw std::invalid_argument("JSON patch must be an array of objects"); - } - - // iterate and apply th eoperations - for (const auto& val : json_patch) - { - // wrapper to get a value for an operation - const auto get_value = [&val](const std::string & op, - const std::string & member, - bool string_type) -> basic_json& - { - // find value - auto it = val.m_value.object->find(member); - - // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; - - // check if desired value is present - if (it == val.m_value.object->end()) - { - throw std::invalid_argument(error_msg + " must have member '" + member + "'"); - } - - // check if result is of type string - if (string_type and not it->second.is_string()) - { - throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); - } - - // no error: return value - return it->second; - }; - - // type check - if (not val.is_object()) - { - throw std::invalid_argument("JSON patch must be an array of objects"); - } - - // collect mandatory members - const std::string op = get_value("op", "op", true); - const std::string path = get_value(op, "path", true); - json_pointer ptr(path); - - switch (get_op(op)) - { - case patch_operations::add: - { - operation_add(ptr, get_value("add", "value", false)); - break; - } - - case patch_operations::remove: - { - operation_remove(ptr); - break; - } - - case patch_operations::replace: - { - // the "path" location must exist - use at() - result.at(ptr) = get_value("replace", "value", false); - break; - } - - case patch_operations::move: - { - const std::string from_path = get_value("move", "from", true); - json_pointer from_ptr(from_path); - - // the "from" location must exist - use at() - basic_json v = result.at(from_ptr); - - // The move operation is functionally identical to a - // "remove" operation on the "from" location, followed - // immediately by an "add" operation at the target - // location with the value that was just removed. - operation_remove(from_ptr); - operation_add(ptr, v); - break; - } - - case patch_operations::copy: - { - const std::string from_path = get_value("copy", "from", true);; - const json_pointer from_ptr(from_path); - - // the "from" location must exist - use at() - result[ptr] = result.at(from_ptr); - break; - } - - case patch_operations::test: - { - bool success = false; - try - { - // check if "value" matches the one at "path" - // the "path" location must exist - use at() - success = (result.at(ptr) == get_value("test", "value", false)); - } - catch (std::out_of_range&) - { - // ignore out of range errors: success remains false - } - - // throw an exception if test fails - if (not success) - { - throw std::domain_error("unsuccessful: " + val.dump()); - } - - break; - } - - case patch_operations::invalid: - { - // op must be "add", "remove", "replace", "move", "copy", or - // "test" - throw std::invalid_argument("operation value '" + op + "' is invalid"); - } - } - } - - return result; - } - - /*! - @brief creates a diff as a JSON patch - - Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can - be changed into the value @a target by calling @ref patch function. - - @invariant For two JSON values @a source and @a target, the following code - yields always `true`: - @code {.cpp} - source.patch(diff(source, target)) == target; - @endcode - - @note Currently, only `remove`, `add`, and `replace` operations are - generated. - - @param[in] source JSON value to copare from - @param[in] target JSON value to copare against - @param[in] path helper value to create JSON pointers - - @return a JSON patch to convert the @a source to @a target - - @complexity Linear in the lengths of @a source and @a target. - - @liveexample{The following code shows how a JSON patch is created as a - diff for two JSON values.,diff} - - @sa @ref patch -- apply a JSON patch - - @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) - - @since version 2.0.0 - */ - static basic_json diff(const basic_json& source, - const basic_json& target, - std::string path = "") - { - // the patch - basic_json result(value_t::array); - - // if the values are the same, return empty patch - if (source == target) - { - return result; - } - - if (source.type() != target.type()) - { - // different types: replace value - result.push_back( - { - {"op", "replace"}, - {"path", path}, - {"value", target} - }); - } - else - { - switch (source.type()) - { - case value_t::array: - { - // first pass: traverse common elements - size_t i = 0; - while (i < source.size() and i < target.size()) - { - // recursive call to compare array values at index i - auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - ++i; - } - - // i now reached the end of at least one array - // in a second pass, traverse the remaining elements - - // remove my remaining elements - const auto end_index = static_cast(result.size()); - while (i < source.size()) - { - // add operations in reverse order to avoid invalid - // indices - result.insert(result.begin() + end_index, object( - { - {"op", "remove"}, - {"path", path + "/" + std::to_string(i)} - })); - ++i; - } - - // add other remaining elements - while (i < target.size()) - { - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + std::to_string(i)}, - {"value", target[i]} - }); - ++i; - } - - break; - } - - case value_t::object: - { - // first pass: traverse this object's elements - for (auto it = source.begin(); it != source.end(); ++it) - { - // escape the key name to be used in a JSON patch - const auto key = json_pointer::escape(it.key()); - - if (target.find(it.key()) != target.end()) - { - // recursive call to compare object values at key it - auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - } - else - { - // found a key that is not in o -> remove it - result.push_back(object( - { - {"op", "remove"}, - {"path", path + "/" + key} - })); - } - } - - // second pass: traverse other object's elements - for (auto it = target.begin(); it != target.end(); ++it) - { - if (source.find(it.key()) == source.end()) - { - // found a key that is not in this -> add it - const auto key = json_pointer::escape(it.key()); - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + key}, - {"value", it.value()} - }); - } - } - - break; - } - - default: - { - // both primitive type: replace value - result.push_back( - { - {"op", "replace"}, - {"path", path}, - {"value", target} - }); - break; - } - } - } - - return result; - } - - /// @} -}; - - -///////////// -// presets // -///////////// - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; -} - - -/////////////////////// -// nonmember support // -/////////////////////// - -// specialization of std::swap, and std::hash -namespace std -{ -/*! -@brief exchanges the values of two JSON objects - -@since version 1.0.0 -*/ -template <> -inline void swap(nlohmann::json& j1, - nlohmann::json& j2) noexcept( - is_nothrow_move_constructible::value and - is_nothrow_move_assignable::value - ) -{ - j1.swap(j2); -} - -/// hash value for JSON objects -template <> -struct hash -{ - /*! - @brief return a hash value for a JSON object - - @since version 1.0.0 - */ - std::size_t operator()(const nlohmann::json& j) const - { - // a naive hashing via the string representation - const auto& h = hash(); - return h(j.dump()); - } -}; -} - -/*! -@brief user-defined string literal for JSON values - -This operator implements a user-defined string literal for JSON objects. It -can be used by adding `"_json"` to a string literal and returns a JSON object -if no parse error occurred. - -@param[in] s a string representation of a JSON object -@return a JSON object - -@since version 1.0.0 -*/ -inline nlohmann::json operator "" _json(const char* s, std::size_t) -{ - return nlohmann::json::parse(reinterpret_cast(s)); -} - -/*! -@brief user-defined string literal for JSON pointer - -This operator implements a user-defined string literal for JSON Pointers. It -can be used by adding `"_json"` to a string literal and returns a JSON pointer -object if no parse error occurred. - -@param[in] s a string representation of a JSON Pointer -@return a JSON pointer object - -@since version 2.0.0 -*/ -inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t) -{ - return nlohmann::json::json_pointer(s); -} - -// restore GCC/clang diagnostic settings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic pop -#endif - -#endif diff --git a/ext/offbase/offbase.hpp b/ext/offbase/offbase.hpp deleted file mode 100644 index acce8c4a..00000000 --- a/ext/offbase/offbase.hpp +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Offbase: a super-minimal in-filesystem JSON object persistence store - */ - -#ifndef OFFBASE_HPP__ -#define OFFBASE_HPP__ - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include "json/json.hpp" - -#define OFFBASE_PATH_SEP "/" - -/** - * A super-minimal in-filesystem JSON object persistence store - */ -class offbase : public nlohmann::json -{ -public: - offbase(const char *p) : - nlohmann::json(nlohmann::json::object()), - _path(p), - _saved(nlohmann::json::object()) - { - this->load(); - } - - ~offbase() - { - this->commit(); - } - - /** - * Load this instance from disk, clearing any existing contents first - * - * If the 'errors' vector is NULL, false is returned and reading aborts - * on any error. If this parameter is non-NULL the paths of errors will - * be added to the vector and reading will continue. False will only be - * returned on really big errors like no path being defined. - * - * @param errors If specified, fill this vector with the paths to any objects that fail read - * @return True on success, false on fatal error - */ - inline bool load(std::vector *errors = (std::vector *)0) - { - if (!_path.length()) - return false; - *this = nlohmann::json::object(); - if (!_loadObj(_path,*this,errors)) - return false; - _saved = *(reinterpret_cast(this)); - return true; - } - - /** - * Commit any pending changes to this object to disk - * - * @return True on success or false if an I/O error occurred - */ - inline bool commit(std::vector *errors = (std::vector *)0) - { - if (!_path.length()) - return false; - if (!_commitObj(_path,*this,&_saved,errors)) - return false; - _saved = *(reinterpret_cast(this)); - return true; - } - - static inline std::string escapeKey(const std::string &k) - { - std::string e; - const char *ptr = k.data(); - const char *eof = ptr + k.length(); - char tmp[8]; - while (ptr != eof) { - if ( ((*ptr >= 'a')&&(*ptr <= 'z')) || ((*ptr >= 'A')&&(*ptr <= 'Z')) || ((*ptr >= '0')&&(*ptr <= '9')) || (*ptr == '.') || (*ptr == '_') || (*ptr == '-') || (*ptr == ',') ) - e.push_back(*ptr); - else { - snprintf(tmp,sizeof(tmp),"~%.2x",(unsigned int)*ptr); - e.append(tmp); - } - ++ptr; - } - return e; - } - - static inline std::string unescapeKey(const std::string &k) - { - std::string u; - const char *ptr = k.data(); - const char *eof = ptr + k.length(); - char tmp[8]; - while (ptr != eof) { - if (*ptr == '~') { - if (++ptr == eof) break; - tmp[0] = *ptr; - if (++ptr == eof) break; - tmp[1] = *(ptr++); - tmp[2] = (char)0; - u.push_back((char)strtol(tmp,(char **)0,16)); - } else { - u.push_back(*(ptr++)); - } - } - return u; - } - -private: - static inline bool _readFile(const char *path,std::string &buf) - { - char tmp[4096]; - FILE *f = fopen(path,"rb"); - if (f) { - for(;;) { - long n = (long)fread(tmp,1,sizeof(tmp),f); - if (n > 0) - buf.append(tmp,n); - else break; - } - fclose(f); - return true; - } - return false; - } - - static inline bool _loadArr(const std::string &path,nlohmann::json &arr,std::vector *errors) - { - std::map atmp; // place into an ordered container first because filesystem does not guarantee this - - struct dirent dbuf; - struct dirent *de; - DIR *d = opendir(path.c_str()); - if (d) { - while (!readdir_d(d,&dbuf,&de)) { - if (!de) break; - const std::string name(de->d_name); - if (name.length() != 12) continue; // array entries are XXXXXXXXXX.T - if (name[name.length()-2] == '.') { - if (name[name.length()-1] == 'V') { - std::string buf; - if (_readFile((path + OFFBASE_PATH_SEP + name).c_str(),buf)) { - try { - atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::parse(buf); - } catch ( ... ) { - if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else { - return false; - } - } - } else if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else return false; - } else if (name[name.length()-1] == 'O') { - if (!_loadObj(path + OFFBASE_PATH_SEP + name,atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::object(),errors)) - return false; - } else if (name[name.length()-1] == 'A') { - if (!_loadArr(path + OFFBASE_PATH_SEP + name,atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::array(),errors)) - return false; - } - } - } - closedir(d); - } else if (errors) { - errors->push_back(path); - } else return false; - - if (atmp.size() > 0) { - unsigned long lasti = 0; - for(std::map::iterator i(atmp.begin());i!=atmp.end();++i) { - for(unsigned long k=lasti;kfirst;++k) // fill any gaps with nulls - arr.push_back(nlohmann::json(std::nullptr_t)); - lasti = i->first; - arr.push_back(i->second); - } - } - - return true; - } - - static inline bool _loadObj(const std::string &path,nlohmann::json &obj,std::vector *errors) - { - struct dirent dbuf; - struct dirent *de; - DIR *d = opendir(path.c_str()); - if (d) { - while (!readdir_d(d,&dbuf,&de)) { - if (!de) break; - if ((strcmp(de->d_name,".") == 0)||(strcmp(de->d_name,"..") == 0)) continue; // sanity check - const std::string name(de->d_name); - if (name.length() <= 2) continue; - if (name[name.length()-2] == '.') { - if (name[name.length()-1] == 'V') { - std::string buf; - if (_readFile((path + OFFBASE_PATH_SEP + name).c_str(),buf)) { - try { - obj[unescapeKey(name)] = nlohmann::json::parse(buf); - } catch ( ... ) { - if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else { - return false; - } - } - } else if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else return false; - } else if (name[name.length()-1] == 'O') { - if (!_loadObj(path + OFFBASE_PATH_SEP + name,obj[unescapeKey(name)] = nlohmann::json::object(),errors)) - return false; - } else if (name[name.length()-1] == 'A') { - if (!_loadArr(path + OFFBASE_PATH_SEP + name,obj[unescapeKey(name)] = nlohmann::json::array(),errors)) - return false; - } - } - } - closedir(d); - } else if (errors) { - errors->push_back(path); - } else return false; - return true; - } - - static inline void _rmDashRf(const std::string &path) - { - struct dirent dbuf; - struct dirent *de; - DIR *d = opendir(path.c_str()); - if (d) { - while (!readdir_r(d,&dbuf,&de)) { - if (!de) break; - if ((strcmp(de->d_name,".") == 0)||(strcmp(de->d_name,"..") == 0)) continue; // sanity check - const std::string full(path + OFFBASE_PATH_SEP + de->d_name); - if (unlink(full.c_str())) { - _rmDashRf(full); - rmdir(full.c_str()); - } - } - closedir(d); - } - rmdir(path.c_str()); - } - - static inline bool _commitArr(const std::string &path,const nlohmann::json &arr,const nlohmann::json *previous,std::vector *errors) - { - char tmp[32]; - - if (!arr.is_array()) - return false; - - mkdir(path.c_str(),0755); - - for(unsigned long i=0;i<(unsigned long)arr.size();++i) { - const nlohmann::json &value = arr[i]; - - const nlohmann::json *next = (const nlohmann::json *)0; - if ((previous)&&(previous->is_array())&&(i < previous->size())) { - next = &((*previous)[i]); - if (*next == value) - continue; - } - - if (value.is_object()) { - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - if (!_commitObj(path + tmp,value,next,errors)) - return false; - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - unlink((path + tmp).c_str()); - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - } else if (value.is_array()) { - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - if (!_commitArr(path + tmp,value,next,errors)) - return false; - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - unlink((path + tmp).c_str()); - } else { - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - FILE *f = fopen((path + tmp).c_str(),"w"); - if (f) { - const std::string v(value.dump()); - if (fwrite(v.c_str(),v.length(),1,f) != 1) { - fclose(f); - return false; - } else { - fclose(f); - } - } else { - return false; - } - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - } - } - - if ((previous)&&(previous->is_array())) { - for(unsigned long i=(unsigned long)arr.size();i<(unsigned long)previous->size();++i) { - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - unlink((path + tmp).c_str()); - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - } - } - - return true; - } - - static inline bool _commitObj(const std::string &path,const nlohmann::json &obj,const nlohmann::json *previous,std::vector *errors) - { - if (!obj.is_object()) - return false; - - mkdir(path.c_str(),0755); - - for(nlohmann::json::const_iterator i(obj.begin());i!=obj.end();++i) { - if (i.key().length() == 0) - continue; - - const nlohmann::json *next = (const nlohmann::json *)0; - if ((previous)&&(previous->is_object())) { - nlohmann::json::const_iterator saved(previous->find(i.key())); - if (saved != previous->end()) { - next = &(saved.value()); - if (i.value() == *next) - continue; - } - } - - const std::string keyp(path + OFFBASE_PATH_SEP + escapeKey(i.key())); - if (i.value().is_object()) { - if (!_commitObj(keyp + ".O",i.value(),next,errors)) - return false; - unlink((keyp + ".V").c_str()); - _rmDashRf(keyp + ".A"); - } else if (i.value().is_array()) { - if (!_commitArr(keyp + ".A",i.value(),next,errors)) - return false; - unlink((keyp + ".V").c_str()); - _rmDashRf(keyp + ".O"); - } else { - FILE *f = fopen((keyp + ".V").c_str(),"w"); - if (f) { - const std::string v(i.value().dump()); - if (fwrite(v.c_str(),v.length(),1,f) != 1) { - fclose(f); - return false; - } else { - fclose(f); - } - } else { - return false; - } - _rmDashRf(keyp + ".A"); - _rmDashRf(keyp + ".O"); - } - } - - if ((previous)&&(previous->is_object())) { - for(nlohmann::json::const_iterator i(previous->begin());i!=previous->end();++i) { - if ((i.key().length() > 0)&&(obj.find(i.key()) == obj.end())) { - const std::string keyp(path + OFFBASE_PATH_SEP + escapeKey(i.key())); - unlink((keyp + ".V").c_str()); - _rmDashRf(keyp + ".A"); - _rmDashRf(keyp + ".O"); - } - } - } - - return true; - } - - std::string _path; - nlohmann::json _saved; -}; - -#endif diff --git a/make-mac.mk b/make-mac.mk index e821c4cf..09e04eab 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -65,7 +65,7 @@ else STRIP=strip endif -CXXFLAGS=$(CFLAGS) -fno-rtti +CXXFLAGS=$(CFLAGS) -fno-rtti -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ all: one @@ -78,7 +78,7 @@ one: $(OBJS) service/OneService.o one.o $(CODESIGN) -vvv zerotier-one cli: FORCE - $(CXX) -Os -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl + $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl $(STRIP) zerotier selftest: $(OBJS) selftest.o diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 3f6b9be6..12446909 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -113,7 +113,7 @@ void InetAddress::set(const std::string &ip,unsigned int port) sin6->sin6_port = Utils::hton((uint16_t)port); if (inet_pton(AF_INET6,ip.c_str(),(void *)&(sin6->sin6_addr.s6_addr)) <= 0) memset(this,0,sizeof(InetAddress)); - } else { + } else if (ip.find('.') != std::string::npos) { struct sockaddr_in *sin = reinterpret_cast(this); ss_family = AF_INET; sin->sin_port = Utils::hton((uint16_t)port); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 3a04308b..086bb269 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -107,6 +107,86 @@ std::vector OSUtils::listDirectory(const char *path) return r; } +std::vector OSUtils::listSubdirectories(const char *path) +{ + std::vector r; + +#ifdef __WINDOWS__ + HANDLE hFind; + WIN32_FIND_DATAA ffd; + if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { + do { + if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))&&((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)) + r.push_back(std::string(ffd.cFileName)); + } while (FindNextFileA(hFind,&ffd)); + FindClose(hFind); + } +#else + struct dirent de; + struct dirent *dptr; + DIR *d = opendir(path); + if (!d) + return r; + dptr = (struct dirent *)0; + for(;;) { + if (readdir_r(d,&de,&dptr)) + break; + if (dptr) { + if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_DIR)) + r.push_back(std::string(dptr->d_name)); + } else break; + } + closedir(d); +#endif + + return r; +} + +bool OSUtils::rmDashRf(const char *path) +{ +#ifdef __WINDOWS__ + HANDLE hFind; + WIN32_FIND_DATAA ffd; + if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { + do { + if ((strcmp(ffd.cFileName,".") != 0)&&(strcmp(ffd.cFileName,"..") != 0)) { + if ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) { + if (DeleteFileA((std::string(path) + ZT_PATH_SEPARATOR_S + ffd.cFileName).c_str()) == FALSE) + return false; + } else { + if (!rmDashRf((std::string(path) + ZT_PATH_SEPARATOR_S + ffd.cFileName).c_str())) + return false; + } + } + } while (FindNextFileA(hFind,&ffd)); + FindClose(hFind); + } + return (RemoveDirectoryA(path) != FALSE); +#else + struct dirent de; + struct dirent *dptr; + DIR *d = opendir(path); + if (!d) + return true; + dptr = (struct dirent *)0; + for(;;) { + if (readdir_r(d,&de,&dptr)) + break; + if ((dptr)&&(strcmp(dptr->d_name,".") != 0)&&(strcmp(dptr->d_name,"..") != 0)) { + std::string p(path); + p.push_back(ZT_PATH_SEPARATOR); + p.append(dptr->d_name); + if (unlink(p.c_str()) != 0) { + if (!rmDashRf(p.c_str())) + return false; + } + } else break; + } + closedir(d); + return (rmdir(path) == 0); +#endif +} + void OSUtils::lockDownFile(const char *path,bool isDir) { #ifdef __UNIX_LIKE__ diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 25bed9fe..4f74344f 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -105,10 +105,26 @@ public: * This returns only files, not sub-directories. * * @param path Path to list - * @return Names of files in directory + * @return Names of files in directory (without path prepended) */ static std::vector listDirectory(const char *path); + /** + * List a directory's subdirectories + * + * @param path Path to list + * @return Names of subdirectories (without path prepended) + */ + static std::vector listSubdirectories(const char *path); + + /** + * Delete a directory and all its files and subdirectories recursively + * + * @param path Path to delete + * @return True on success + */ + static bool rmDashRf(const char *path); + /** * Set modes on a file to something secure * -- cgit v1.2.3 From 58701c1ca8adc774dd1a6d179d8431579a59f85b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 16 Aug 2016 14:08:08 -0700 Subject: . --- attic/old-controller-schema.sql | 112 +++++++++++++++++++++++++++++++++ controller/SqliteNetworkController.cpp | 7 --- controller/schema.sql | 112 --------------------------------- controller/schema.sql.c | 112 --------------------------------- controller/schema2c.sh | 8 --- 5 files changed, 112 insertions(+), 239 deletions(-) create mode 100644 attic/old-controller-schema.sql delete mode 100644 controller/schema.sql delete mode 100644 controller/schema.sql.c delete mode 100755 controller/schema2c.sh (limited to 'controller') diff --git a/attic/old-controller-schema.sql b/attic/old-controller-schema.sql new file mode 100644 index 00000000..d1daf8d0 --- /dev/null +++ b/attic/old-controller-schema.sql @@ -0,0 +1,112 @@ +CREATE TABLE Config ( + k varchar(16) PRIMARY KEY NOT NULL, + v varchar(1024) NOT NULL +); + +CREATE TABLE Network ( + id char(16) PRIMARY KEY NOT NULL, + name varchar(128) NOT NULL, + private integer NOT NULL DEFAULT(1), + enableBroadcast integer NOT NULL DEFAULT(1), + allowPassiveBridging integer NOT NULL DEFAULT(0), + multicastLimit integer NOT NULL DEFAULT(32), + creationTime integer NOT NULL DEFAULT(0), + revision integer NOT NULL DEFAULT(1), + memberRevisionCounter integer NOT NULL DEFAULT(1), + flags integer NOT NULL DEFAULT(0) +); + +CREATE TABLE AuthToken ( + id integer PRIMARY KEY NOT NULL, + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + authMode integer NOT NULL DEFAULT(1), + useCount integer NOT NULL DEFAULT(0), + maxUses integer NOT NULL DEFAULT(0), + expiresAt integer NOT NULL DEFAULT(0), + token varchar(256) NOT NULL +); + +CREATE INDEX AuthToken_networkId_token ON AuthToken(networkId,token); + +CREATE TABLE Node ( + id char(10) PRIMARY KEY NOT NULL, + identity varchar(4096) NOT NULL +); + +CREATE TABLE IpAssignment ( + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + nodeId char(10) REFERENCES Node(id) ON DELETE CASCADE, + type integer NOT NULL DEFAULT(0), + ip blob(16) NOT NULL, + ipNetmaskBits integer NOT NULL DEFAULT(0), + ipVersion integer NOT NULL DEFAULT(4) +); + +CREATE UNIQUE INDEX IpAssignment_networkId_ip ON IpAssignment (networkId, ip); + +CREATE INDEX IpAssignment_networkId_nodeId ON IpAssignment (networkId, nodeId); + +CREATE TABLE IpAssignmentPool ( + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + ipRangeStart blob(16) NOT NULL, + ipRangeEnd blob(16) NOT NULL, + ipVersion integer NOT NULL DEFAULT(4) +); + +CREATE UNIQUE INDEX IpAssignmentPool_networkId_ipRangeStart ON IpAssignmentPool (networkId,ipRangeStart); + +CREATE TABLE Member ( + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE, + authorized integer NOT NULL DEFAULT(0), + activeBridge integer NOT NULL DEFAULT(0), + memberRevision integer NOT NULL DEFAULT(0), + flags integer NOT NULL DEFAULT(0), + lastRequestTime integer NOT NULL DEFAULT(0), + lastPowDifficulty integer NOT NULL DEFAULT(0), + lastPowTime integer NOT NULL DEFAULT(0), + recentHistory blob, + PRIMARY KEY (networkId, nodeId) +); + +CREATE INDEX Member_networkId_nodeId ON Member(networkId,nodeId); +CREATE INDEX Member_networkId_activeBridge ON Member(networkId, activeBridge); +CREATE INDEX Member_networkId_memberRevision ON Member(networkId, memberRevision); +CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime); + +CREATE TABLE Route ( + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + target blob(16) NOT NULL, + via blob(16), + targetNetmaskBits integer NOT NULL, + ipVersion integer NOT NULL, + flags integer NOT NULL, + metric integer NOT NULL +); + +CREATE INDEX Route_networkId ON Route (networkId); + +CREATE TABLE Rule ( + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + capId integer, + ruleNo integer NOT NULL, + ruleType integer NOT NULL DEFAULT(0), + "addr" blob(16), + "int1" integer, + "int2" integer, + "int3" integer, + "int4" integer +); + +CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId); + +CREATE TABLE MemberTC ( + networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE, + tagId integer, + tagValue integer, + capId integer, + capMaxCustodyChainLength integer NOT NULL DEFAULT(1) +); + +CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId); diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 94d2e2e6..cbfa8891 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -53,15 +53,8 @@ #include "../node/MAC.hpp" #include "../node/Address.hpp" -// offbase includes and builds upon nlohmann::json using json = nlohmann::json; -// Stored in database as schemaVersion key in Config. -// If not present, database is assumed to be empty and at the current schema version -// and this key/value is added automatically. -//#define ZT_NETCONF_SQLITE_SCHEMA_VERSION 5 -//#define ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR "5" - // API version reported via JSON control plane #define ZT_NETCONF_CONTROLLER_API_VERSION 3 diff --git a/controller/schema.sql b/controller/schema.sql deleted file mode 100644 index d1daf8d0..00000000 --- a/controller/schema.sql +++ /dev/null @@ -1,112 +0,0 @@ -CREATE TABLE Config ( - k varchar(16) PRIMARY KEY NOT NULL, - v varchar(1024) NOT NULL -); - -CREATE TABLE Network ( - id char(16) PRIMARY KEY NOT NULL, - name varchar(128) NOT NULL, - private integer NOT NULL DEFAULT(1), - enableBroadcast integer NOT NULL DEFAULT(1), - allowPassiveBridging integer NOT NULL DEFAULT(0), - multicastLimit integer NOT NULL DEFAULT(32), - creationTime integer NOT NULL DEFAULT(0), - revision integer NOT NULL DEFAULT(1), - memberRevisionCounter integer NOT NULL DEFAULT(1), - flags integer NOT NULL DEFAULT(0) -); - -CREATE TABLE AuthToken ( - id integer PRIMARY KEY NOT NULL, - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - authMode integer NOT NULL DEFAULT(1), - useCount integer NOT NULL DEFAULT(0), - maxUses integer NOT NULL DEFAULT(0), - expiresAt integer NOT NULL DEFAULT(0), - token varchar(256) NOT NULL -); - -CREATE INDEX AuthToken_networkId_token ON AuthToken(networkId,token); - -CREATE TABLE Node ( - id char(10) PRIMARY KEY NOT NULL, - identity varchar(4096) NOT NULL -); - -CREATE TABLE IpAssignment ( - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - nodeId char(10) REFERENCES Node(id) ON DELETE CASCADE, - type integer NOT NULL DEFAULT(0), - ip blob(16) NOT NULL, - ipNetmaskBits integer NOT NULL DEFAULT(0), - ipVersion integer NOT NULL DEFAULT(4) -); - -CREATE UNIQUE INDEX IpAssignment_networkId_ip ON IpAssignment (networkId, ip); - -CREATE INDEX IpAssignment_networkId_nodeId ON IpAssignment (networkId, nodeId); - -CREATE TABLE IpAssignmentPool ( - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - ipRangeStart blob(16) NOT NULL, - ipRangeEnd blob(16) NOT NULL, - ipVersion integer NOT NULL DEFAULT(4) -); - -CREATE UNIQUE INDEX IpAssignmentPool_networkId_ipRangeStart ON IpAssignmentPool (networkId,ipRangeStart); - -CREATE TABLE Member ( - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE, - authorized integer NOT NULL DEFAULT(0), - activeBridge integer NOT NULL DEFAULT(0), - memberRevision integer NOT NULL DEFAULT(0), - flags integer NOT NULL DEFAULT(0), - lastRequestTime integer NOT NULL DEFAULT(0), - lastPowDifficulty integer NOT NULL DEFAULT(0), - lastPowTime integer NOT NULL DEFAULT(0), - recentHistory blob, - PRIMARY KEY (networkId, nodeId) -); - -CREATE INDEX Member_networkId_nodeId ON Member(networkId,nodeId); -CREATE INDEX Member_networkId_activeBridge ON Member(networkId, activeBridge); -CREATE INDEX Member_networkId_memberRevision ON Member(networkId, memberRevision); -CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime); - -CREATE TABLE Route ( - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - target blob(16) NOT NULL, - via blob(16), - targetNetmaskBits integer NOT NULL, - ipVersion integer NOT NULL, - flags integer NOT NULL, - metric integer NOT NULL -); - -CREATE INDEX Route_networkId ON Route (networkId); - -CREATE TABLE Rule ( - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - capId integer, - ruleNo integer NOT NULL, - ruleType integer NOT NULL DEFAULT(0), - "addr" blob(16), - "int1" integer, - "int2" integer, - "int3" integer, - "int4" integer -); - -CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId); - -CREATE TABLE MemberTC ( - networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, - nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE, - tagId integer, - tagValue integer, - capId integer, - capMaxCustodyChainLength integer NOT NULL DEFAULT(1) -); - -CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId); diff --git a/controller/schema.sql.c b/controller/schema.sql.c deleted file mode 100644 index e84ee766..00000000 --- a/controller/schema.sql.c +++ /dev/null @@ -1,112 +0,0 @@ -#define ZT_NETCONF_SCHEMA_SQL \ -"CREATE TABLE Config (\n"\ -" k varchar(16) PRIMARY KEY NOT NULL,\n"\ -" v varchar(1024) NOT NULL\n"\ -");\n"\ -"\n"\ -"CREATE TABLE Network (\n"\ -" id char(16) PRIMARY KEY NOT NULL,\n"\ -" name varchar(128) NOT NULL,\n"\ -" private integer NOT NULL DEFAULT(1),\n"\ -" enableBroadcast integer NOT NULL DEFAULT(1),\n"\ -" allowPassiveBridging integer NOT NULL DEFAULT(0),\n"\ -" multicastLimit integer NOT NULL DEFAULT(32),\n"\ -" creationTime integer NOT NULL DEFAULT(0),\n"\ -" revision integer NOT NULL DEFAULT(1),\n"\ -" memberRevisionCounter integer NOT NULL DEFAULT(1),\n"\ -" flags integer NOT NULL DEFAULT(0)\n"\ -");\n"\ -"\n"\ -"CREATE TABLE AuthToken (\n"\ -" id integer PRIMARY KEY NOT NULL,\n"\ -" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ -" authMode integer NOT NULL DEFAULT(1),\n"\ -" useCount integer NOT NULL DEFAULT(0),\n"\ -" maxUses integer NOT NULL DEFAULT(0),\n"\ -" expiresAt integer NOT NULL DEFAULT(0),\n"\ -" token varchar(256) NOT NULL\n"\ -");\n"\ -"\n"\ -"CREATE INDEX AuthToken_networkId_token ON AuthToken(networkId,token);\n"\ -"\n"\ -"CREATE TABLE Node (\n"\ -" id char(10) PRIMARY KEY NOT NULL,\n"\ -" identity varchar(4096) NOT NULL\n"\ -");\n"\ -"\n"\ -"CREATE TABLE IpAssignment (\n"\ -" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ -" nodeId char(10) REFERENCES Node(id) ON DELETE CASCADE,\n"\ -" type integer NOT NULL DEFAULT(0),\n"\ -" ip blob(16) NOT NULL,\n"\ -" ipNetmaskBits integer NOT NULL DEFAULT(0),\n"\ -" ipVersion integer NOT NULL DEFAULT(4)\n"\ -");\n"\ -"\n"\ -"CREATE UNIQUE INDEX IpAssignment_networkId_ip ON IpAssignment (networkId, ip);\n"\ -"\n"\ -"CREATE INDEX IpAssignment_networkId_nodeId ON IpAssignment (networkId, nodeId);\n"\ -"\n"\ -"CREATE TABLE IpAssignmentPool (\n"\ -" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ -" ipRangeStart blob(16) NOT NULL,\n"\ -" ipRangeEnd blob(16) NOT NULL,\n"\ -" ipVersion integer NOT NULL DEFAULT(4)\n"\ -");\n"\ -"\n"\ -"CREATE UNIQUE INDEX IpAssignmentPool_networkId_ipRangeStart ON IpAssignmentPool (networkId,ipRangeStart);\n"\ -"\n"\ -"CREATE TABLE Member (\n"\ -" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ -" nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n"\ -" authorized integer NOT NULL DEFAULT(0),\n"\ -" activeBridge integer NOT NULL DEFAULT(0),\n"\ -" memberRevision integer NOT NULL DEFAULT(0),\n"\ -" flags integer NOT NULL DEFAULT(0),\n"\ -" lastRequestTime integer NOT NULL DEFAULT(0),\n"\ -" lastPowDifficulty integer NOT NULL DEFAULT(0),\n"\ -" lastPowTime integer NOT NULL DEFAULT(0),\n"\ -" recentHistory blob,\n"\ -" PRIMARY KEY (networkId, nodeId)\n"\ -");\n"\ -"\n"\ -"CREATE INDEX Member_networkId_nodeId ON Member(networkId,nodeId);\n"\ -"CREATE INDEX Member_networkId_activeBridge ON Member(networkId, activeBridge);\n"\ -"CREATE INDEX Member_networkId_memberRevision ON Member(networkId, memberRevision);\n"\ -"CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n"\ -"\n"\ -"CREATE TABLE Route (\n"\ -" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ -" target blob(16) NOT NULL,\n"\ -" via blob(16),\n"\ -" targetNetmaskBits integer NOT NULL,\n"\ -" ipVersion integer NOT NULL,\n"\ -" flags integer NOT NULL,\n"\ -" metric integer NOT NULL\n"\ -");\n"\ -"\n"\ -"CREATE INDEX Route_networkId ON Route (networkId);\n"\ -"\n"\ -"CREATE TABLE Relay (\n"\ -" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ -" address char(10) NOT NULL,\n"\ -" phyAddress varchar(64) NOT NULL\n"\ -");\n"\ -"\n"\ -"CREATE UNIQUE INDEX Relay_networkId_address ON Relay (networkId,address);\n"\ -"\n"\ -"CREATE TABLE Rule (\n"\ -" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ -" policyId varchar(32),\n"\ -" ruleNo integer NOT NULL,\n"\ -" ruleType integer NOT NULL DEFAULT(0),\n"\ -" \"addr\" blob(16),\n"\ -" \"int1\" integer,\n"\ -" \"int2\" integer,\n"\ -" \"int3\" integer,\n"\ -" \"int4\" integer\n"\ -");\n"\ -"\n"\ -"CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n"\ -"CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId);\n"\ -"" diff --git a/controller/schema2c.sh b/controller/schema2c.sh deleted file mode 100755 index 4f4f1647..00000000 --- a/controller/schema2c.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Run this file to package the .sql file into a .c file whenever the SQL changes. - -rm -f schema.sql.c -echo '#define ZT_NETCONF_SCHEMA_SQL \' >schema.sql.c -cat schema.sql | sed 's/"/\\"/g' | sed 's/^/"/' | sed 's/$/\\n"\\/' >>schema.sql.c -echo '""' >>schema.sql.c -- cgit v1.2.3 From c0639ccd37d9904ad0411a38f0a38f5d7d329f1e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 16 Aug 2016 16:46:08 -0700 Subject: Just about ready to test. --- controller/SqliteNetworkController.cpp | 1120 ++++++++++++-------------------- controller/SqliteNetworkController.hpp | 32 +- 2 files changed, 456 insertions(+), 696 deletions(-) (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index cbfa8891..885d12af 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -604,271 +604,153 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; } - char nwids[24],nodeIds[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - Utils::snprintf(nodeIds,sizeof(nodeIds),"%.10llx",(unsigned long long)identity.address().toInt()); - const uint64_t now = OSUtils::now(); - /* - { // begin lock - Mutex::Lock _l(_lock); - - // Check rate limit circuit breaker to prevent flooding - { - uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; - if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) - return NetworkController::NETCONF_QUERY_IGNORE; - lrt = now; - } - - _backupNeeded = true; + // Check rate limit circuit breaker to prevent flooding + { + Mutex::Lock _l(_lastRequestTime_m); + uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; + if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + return NetworkController::NETCONF_QUERY_IGNORE; + lrt = now; + } - // Create Node record or do full identity check if we already have one + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; + const std::string memberJP(_memberJP(nwid,identity.address(),false)); + json member(_readJson(memberJP)); - sqlite3_reset(_sGetNodeIdentity); - sqlite3_bind_text(_sGetNodeIdentity,1,member.nodeId,10,SQLITE_STATIC); - if (sqlite3_step(_sGetNodeIdentity) == SQLITE_ROW) { + { + std::string haveIdStr = member.value("identity",""); + if (haveIdStr.length() > 0) { try { - Identity alreadyKnownIdentity((const char *)sqlite3_column_text(_sGetNodeIdentity,0)); - if (alreadyKnownIdentity != identity) + if (Identity(haveIdStr.c_str()) != identity) return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } catch ( ... ) { // identity stored in database is not valid or is NULL + } catch ( ... ) { return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } } else { - std::string idstr(identity.toString(false)); - sqlite3_reset(_sCreateOrReplaceNode); - sqlite3_bind_text(_sCreateOrReplaceNode,1,member.nodeId,10,SQLITE_STATIC); - sqlite3_bind_text(_sCreateOrReplaceNode,2,idstr.c_str(),-1,SQLITE_STATIC); - if (sqlite3_step(_sCreateOrReplaceNode) != SQLITE_DONE) { - return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } - } - - // Fetch Network record - - sqlite3_reset(_sGetNetworkById); - sqlite3_bind_text(_sGetNetworkById,1,network.id,16,SQLITE_STATIC); - if (sqlite3_step(_sGetNetworkById) == SQLITE_ROW) { - network.name = (const char *)sqlite3_column_text(_sGetNetworkById,0); - network.isPrivate = (sqlite3_column_int(_sGetNetworkById,1) > 0); - network.enableBroadcast = (sqlite3_column_int(_sGetNetworkById,2) > 0); - network.allowPassiveBridging = (sqlite3_column_int(_sGetNetworkById,3) > 0); - network.flags = sqlite3_column_int(_sGetNetworkById,4); - network.multicastLimit = sqlite3_column_int(_sGetNetworkById,5); - network.creationTime = (uint64_t)sqlite3_column_int64(_sGetNetworkById,6); - network.revision = (uint64_t)sqlite3_column_int64(_sGetNetworkById,7); - network.memberRevisionCounter = (uint64_t)sqlite3_column_int64(_sGetNetworkById,8); - } else { - return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; - } - - // Fetch or create Member record - - sqlite3_reset(_sGetMember); - sqlite3_bind_text(_sGetMember,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetMember,2,member.nodeId,10,SQLITE_STATIC); - if (sqlite3_step(_sGetMember) == SQLITE_ROW) { - member.rowid = sqlite3_column_int64(_sGetMember,0); - member.authorized = (sqlite3_column_int(_sGetMember,1) > 0); - member.activeBridge = (sqlite3_column_int(_sGetMember,2) > 0); - member.lastRequestTime = (uint64_t)sqlite3_column_int64(_sGetMember,5); - const char *rhblob = (const char *)sqlite3_column_blob(_sGetMember,6); - if (rhblob) - member.recentHistory.fromBlob(rhblob,(unsigned int)sqlite3_column_bytes(_sGetMember,6)); - } else { - member.authorized = (network.isPrivate ? false : true); - member.activeBridge = false; - sqlite3_reset(_sCreateMember); - sqlite3_bind_text(_sCreateMember,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_text(_sCreateMember,2,member.nodeId,10,SQLITE_STATIC); - sqlite3_bind_int(_sCreateMember,3,(member.authorized ? 1 : 0)); - sqlite3_bind_text(_sCreateMember,4,network.id,16,SQLITE_STATIC); - if (sqlite3_step(_sCreateMember) != SQLITE_DONE) { - return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } - member.rowid = sqlite3_last_insert_rowid(_db); - - sqlite3_reset(_sIncrementMemberRevisionCounter); - sqlite3_bind_text(_sIncrementMemberRevisionCounter,1,network.id,16,SQLITE_STATIC); - sqlite3_step(_sIncrementMemberRevisionCounter); + member["identity"] = identity.toString(false); } + } - // Update Member.history - - { - char mh[1024]; - Utils::snprintf(mh,sizeof(mh), - "{\"ts\":%llu,\"authorized\":%s,\"clientMajorVersion\":%u,\"clientMinorVersion\":%u,\"clientRevision\":%u,\"fromAddr\":", - (unsigned long long)now, - ((member.authorized) ? "true" : "false"), - metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0), - metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0), - metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0)); - member.recentHistory.push_front(std::string(mh)); - if (fromAddr) { - member.recentHistory.front().push_back('"'); - member.recentHistory.front().append(_jsonEscape(fromAddr.toString())); - member.recentHistory.front().append("\"}"); - } else { - member.recentHistory.front().append("null}"); + // Make sure these are always present no matter what, and increment member revision since we will always at least log something + member["id"] = identity.address().toString(); + member["address"] = member["id"]; + member["nwid"] = network["id"]; + member["lastModified"] = now; + member["memberRevision"] = member.value("memberRevision",0ULL) + 1; + + // Update member log + { + json recentLog = json::array(); + json rlEntry = json::object(); + rlEntry["ts"] = now; + rlEntry["authorized"] = member["authorized"]; + rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); + if (fromAddr) + rlEntry["fromAddr"] = fromAddr.toString(); + recentLog.push_back(rlEntry); + json oldLog = member["recentLog"]; + if (oldLog.is_array()) { + for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) + break; } - - while (member.recentHistory.size() > ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) - member.recentHistory.pop_back(); - std::string rhblob(member.recentHistory.toBlob()); - - sqlite3_reset(_sUpdateMemberHistory); - sqlite3_clear_bindings(_sUpdateMemberHistory); - sqlite3_bind_int64(_sUpdateMemberHistory,1,(sqlite3_int64)now); - sqlite3_bind_blob(_sUpdateMemberHistory,2,(const void *)rhblob.data(),(int)rhblob.length(),SQLITE_STATIC); - sqlite3_bind_int64(_sUpdateMemberHistory,3,member.rowid); - sqlite3_step(_sUpdateMemberHistory); } + member["recentLog"] = recentLog; + } - // Don't proceed if member is not authorized! --------------------------- - - if (!member.authorized) + if (!member.value("authorized",false)) { + if (network.value("private",true)) { + _writeJson(memberJP,member); return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - - // Create network configuration -- we create both legacy and new types and send both for backward compatibility - - // New network config structure - nc.networkId = Utils::hexStrToU64(network.id); - nc.type = network.isPrivate ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; - nc.timestamp = now; - nc.revision = network.revision; - nc.issuedTo = member.nodeId; - if (network.enableBroadcast) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (network.allowPassiveBridging) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - memcpy(nc.name,network.name,std::min((unsigned int)ZT_MAX_NETWORK_SHORT_NAME_LENGTH,(unsigned int)strlen(network.name))); - - { // TODO: right now only etherTypes are supported in rules - std::vector allowedEtherTypes; - sqlite3_reset(_sGetEtherTypesFromRuleTable); - sqlite3_bind_text(_sGetEtherTypesFromRuleTable,1,network.id,16,SQLITE_STATIC); - while (sqlite3_step(_sGetEtherTypesFromRuleTable) == SQLITE_ROW) { - if (sqlite3_column_type(_sGetEtherTypesFromRuleTable,0) == SQLITE_NULL) { - allowedEtherTypes.clear(); - allowedEtherTypes.push_back(0); // NULL 'allow' matches ANY - break; - } else { - int et = sqlite3_column_int(_sGetEtherTypesFromRuleTable,0); - if ((et >= 0)&&(et <= 0xffff)) - allowedEtherTypes.push_back(et); - } - } - std::sort(allowedEtherTypes.begin(),allowedEtherTypes.end()); - allowedEtherTypes.erase(std::unique(allowedEtherTypes.begin(),allowedEtherTypes.end()),allowedEtherTypes.end()); - - for(long i=0;i<(long)allowedEtherTypes.size();++i) { - if ((nc.ruleCount + 2) > ZT_MAX_NETWORK_RULES) - break; - if (allowedEtherTypes[i] > 0) { - nc.rules[nc.ruleCount].t = ZT_NETWORK_RULE_MATCH_ETHERTYPE; - nc.rules[nc.ruleCount].v.etherType = (uint16_t)allowedEtherTypes[i]; - ++nc.ruleCount; - } - nc.rules[nc.ruleCount++].t = ZT_NETWORK_RULE_ACTION_ACCEPT; - } + } else { + member["authorized"] = true; // auto-authorize on public networks } + } - nc.multicastLimit = network.multicastLimit; - - bool amActiveBridge = false; - { - sqlite3_reset(_sGetActiveBridges); - sqlite3_bind_text(_sGetActiveBridges,1,network.id,16,SQLITE_STATIC); - while (sqlite3_step(_sGetActiveBridges) == SQLITE_ROW) { - const char *ab = (const char *)sqlite3_column_text(_sGetActiveBridges,0); - if ((ab)&&(strlen(ab) == 10)) { - const uint64_t ab2 = Utils::hexStrToU64(ab); + nc.networkId = nwid; + nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc.timestamp = now; + nc.revision = network.value("revision",0ULL); + nc.issuedTo = identity.address(); + if (network.value("enableBroadcast",true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (network.value("allowPassiveBridging",false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc.name,sizeof(nc.name),network.value("name","").c_str()); + nc.multicastLimit = (unsigned int)network.value("multicastLimit",32ULL); + + bool amActiveBridge = false; + { + json ab = network["activeBridges"]; + if (ab.is_array()) { + for(unsigned long i=0;i(&(r->target))) = InetAddress((const void *)((const char *)sqlite3_column_blob(_sGetRoutes,0) + 12),4,(unsigned int)sqlite3_column_int(_sGetRoutes,2)); - break; - case 6: - *(reinterpret_cast(&(r->target))) = InetAddress((const void *)sqlite3_column_blob(_sGetRoutes,0),16,(unsigned int)sqlite3_column_int(_sGetRoutes,2)); - break; - default: - continue; - } - if (sqlite3_column_type(_sGetRoutes,1) != SQLITE_NULL) { - switch(sqlite3_column_int(_sGetRoutes,3)) { // ipVersion - case 4: - *(reinterpret_cast(&(r->via))) = InetAddress((const void *)((const char *)sqlite3_column_blob(_sGetRoutes,1) + 12),4,0); - break; - case 6: - *(reinterpret_cast(&(r->via))) = InetAddress((const void *)sqlite3_column_blob(_sGetRoutes,1),16,0); - break; - default: - continue; - } - } - r->flags = (uint16_t)sqlite3_column_int(_sGetRoutes,4); - r->metric = (uint16_t)sqlite3_column_int(_sGetRoutes,5); - ++nc.routeCount; - } + auto v4AssignMode = network["v4AssignMode"]; + auto v6AssignMode = network["v6AssignMode"]; + auto ipAssignmentPools = network["ipAssignmentPools"]; + auto routes = network["routes"]; + auto rules = network["rules"]; - // Assign special IPv6 addresses if these are enabled - if (((network.flags & ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_RFC4193) != 0)&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if (v6AssignMode.is_object()) { + if ((v6AssignMode.value("rfc4193",false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } - if (((network.flags & ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_6PLANE) != 0)&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if ((v6AssignMode.value("6plane",false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } + } + + if (rules.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + break; + auto rule = rules[i]; + if (_parseRule(rule,nc.rules[nc.ruleCount])) + ++nc.ruleCount; + } + } + + if (routes.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) + break; + auto route = routes[i]; + InetAddress t(route.value("target","")); + InetAddress v(route.value("via","")); + if ((t)&&(v)&&(t.ss_family == v.ss_family)) { + ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + *(reinterpret_cast(&(r->target))) = t; + *(reinterpret_cast(&(r->via))) = v; + ++nc.routeCount; + } + } + } - // Get managed addresses that are assigned to this member - bool haveManagedIpv4AutoAssignment = false; - bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count - sqlite3_reset(_sGetIpAssignmentsForNode); - sqlite3_bind_text(_sGetIpAssignmentsForNode,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetIpAssignmentsForNode,2,member.nodeId,10,SQLITE_STATIC); - while (sqlite3_step(_sGetIpAssignmentsForNode) == SQLITE_ROW) { - const unsigned char *const ipbytes = (const unsigned char *)sqlite3_column_blob(_sGetIpAssignmentsForNode,0); - if ((!ipbytes)||(sqlite3_column_bytes(_sGetIpAssignmentsForNode,0) != 16)) - continue; - //const int ipNetmaskBits = sqlite3_column_int(_sGetIpAssignmentsForNode,1); - const int ipVersion = sqlite3_column_int(_sGetIpAssignmentsForNode,2); - - InetAddress ip; - if (ipVersion == 4) - ip = InetAddress(ipbytes + 12,4,0); - else if (ipVersion == 6) - ip = InetAddress(ipbytes,16,0); - else continue; + bool haveManagedIpv4AutoAssignment = false; + bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count + json ipAssignments = member["ipAssignments"]; + if (ipAssignments.is_array()) { + for(unsigned long i=0;i(sqlite3_column_blob(_sGetIpAssignmentPools,0)); - const uint8_t *const ipRangeEndB = reinterpret_cast(sqlite3_column_blob(_sGetIpAssignmentPools,1)); - if ((!ipRangeStartB)||(!ipRangeEndB)||(sqlite3_column_bytes(_sGetIpAssignmentPools,0) != 16)||(sqlite3_column_bytes(_sGetIpAssignmentPools,1) != 16)) - continue; - - uint64_t s[2],e[2],x[2],xx[2]; - memcpy(s,ipRangeStartB,16); - memcpy(e,ipRangeEndB,16); - s[0] = Utils::ntoh(s[0]); - s[1] = Utils::ntoh(s[1]); - e[0] = Utils::ntoh(e[0]); - e[1] = Utils::ntoh(e[1]); - x[0] = s[0]; - x[1] = s[1]; - - for(unsigned int trialCount=0;trialCount<1000;++trialCount) { - if ((trialCount == 0)&&(e[1] > s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { - // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. - xx[0] = Utils::hton(x[0]); - xx[1] = Utils::hton(x[1] + identity.address().toInt()); - } else { - // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway - Utils::getSecureRandom((void *)xx,16); - if ((e[0] > s[0])) - xx[0] %= (e[0] - s[0]); - else xx[0] = 0; - if ((e[1] > s[1])) - xx[1] %= (e[1] - s[1]); - else xx[1] = 0; - xx[0] = Utils::hton(x[0] + xx[0]); - xx[1] = Utils::hton(x[1] + xx[1]); - } + std::set allocatedIps; + bool allocatedIpsLoaded = false; + + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(v6AssignMode.value("zt",false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { + if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + for(unsigned long p=0;((p s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { + // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. + xx[0] = Utils::hton(x[0]); + xx[1] = Utils::hton(x[1] + identity.address().toInt()); + } else { + // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway + Utils::getSecureRandom((void *)xx,16); + if ((e[0] > s[0])) + xx[0] %= (e[0] - s[0]); + else xx[0] = 0; + if ((e[1] > s[1])) + xx[1] %= (e[1] - s[1]); + else xx[1] = 0; + xx[0] = Utils::hton(x[0] + xx[0]); + xx[1] = Utils::hton(x[1] + xx[1]); + } - InetAddress ip6((const void *)xx,16,0); + InetAddress ip6((const void *)xx,16,0); - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); - } + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = 0; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) + routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + } - // If it's routed, then try to claim and assign it and if successful end loop - if (routedNetmaskBits > 0) { - sqlite3_reset(_sCheckIfIpIsAllocated); - sqlite3_bind_text(_sCheckIfIpIsAllocated,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_blob(_sCheckIfIpIsAllocated,2,(const void *)ip6.rawIpData(),16,SQLITE_STATIC); - sqlite3_bind_int(_sCheckIfIpIsAllocated,3,6); // 6 == IPv6 - sqlite3_bind_int(_sCheckIfIpIsAllocated,4,(int)0); - if (sqlite3_step(_sCheckIfIpIsAllocated) != SQLITE_ROW) { - // No rows returned, so the IP is available - sqlite3_reset(_sAllocateIp); - sqlite3_bind_text(_sAllocateIp,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_text(_sAllocateIp,2,member.nodeId,10,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,3,(int)0); - sqlite3_bind_blob(_sAllocateIp,4,(const void *)ip6.rawIpData(),16,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,5,routedNetmaskBits); // IP netmask bits from matching route - sqlite3_bind_int(_sAllocateIp,6,6); // 6 == IPv6 - if (sqlite3_step(_sAllocateIp) == SQLITE_DONE) { - ip6.setPort(routedNetmaskBits); - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) - nc.staticIps[nc.staticIpCount++] = ip6; - break; - } + // If it's routed, then try to claim and assign it and if successful end loop + if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip6))) { + ipAssignments.push_back(ip6.toIpString()); + member["ipAssignments"] = ipAssignments; + ip6.setPort((unsigned int)routedNetmaskBits); + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) + nc.staticIps[nc.staticIpCount++] = ip6; + haveManagedIpv4AutoAssignment = true; + break; } } } } } + } - // Auto-assign IPv4 address if auto-assignment is enabled and it's needed - if ( ((network.flags & ZT_DB_NETWORK_FLAG_ZT_MANAGED_V4_AUTO_ASSIGN) != 0) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { - sqlite3_reset(_sGetIpAssignmentPools); - sqlite3_bind_text(_sGetIpAssignmentPools,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_int(_sGetIpAssignmentPools,2,4); // 4 == IPv4 - while (sqlite3_step(_sGetIpAssignmentPools) == SQLITE_ROW) { - const unsigned char *ipRangeStartB = reinterpret_cast(sqlite3_column_blob(_sGetIpAssignmentPools,0)); - const unsigned char *ipRangeEndB = reinterpret_cast(sqlite3_column_blob(_sGetIpAssignmentPools,1)); - if ((!ipRangeStartB)||(!ipRangeEndB)||(sqlite3_column_bytes(_sGetIpAssignmentPools,0) != 16)||(sqlite3_column_bytes(_sGetIpAssignmentPools,1) != 16)) - continue; - - uint32_t ipRangeStart = Utils::ntoh(*(reinterpret_cast(ipRangeStartB + 12))); - uint32_t ipRangeEnd = Utils::ntoh(*(reinterpret_cast(ipRangeEndB + 12))); - if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) - continue; - uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; - - // Start with the LSB of the member's address - uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); - - for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { - uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; - ++ipTrialCounter; - if ((ip & 0x000000ff) == 0x000000ff) - continue; // don't allow addresses that end in .255 - - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); - if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { - routedNetmaskBits = targetBits; - break; + if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(v4AssignMode.value("zt",false))) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { + if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + for(unsigned long p=0;((p(&ipRangeStart)->sin_addr.s_addr)); + uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); + if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) + continue; + uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; + + // Start with the LSB of the member's address + uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); + + for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { + uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; + ++ipTrialCounter; + if ((ip & 0x000000ff) == 0x000000ff) + continue; // don't allow addresses that end in .255 + + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = 0; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); + int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); + if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { + routedNetmaskBits = targetBits; + break; + } } } - } - // If it's routed, then try to claim and assign it and if successful end loop - if (routedNetmaskBits > 0) { - uint32_t ipBlob[4]; // actually a 16-byte blob, we put IPv4s in the last 4 bytes - ipBlob[0] = 0; ipBlob[1] = 0; ipBlob[2] = 0; ipBlob[3] = Utils::hton(ip); - sqlite3_reset(_sCheckIfIpIsAllocated); - sqlite3_bind_text(_sCheckIfIpIsAllocated,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_blob(_sCheckIfIpIsAllocated,2,(const void *)ipBlob,16,SQLITE_STATIC); - sqlite3_bind_int(_sCheckIfIpIsAllocated,3,4); // 4 == IPv4 - sqlite3_bind_int(_sCheckIfIpIsAllocated,4,(int)0); - if (sqlite3_step(_sCheckIfIpIsAllocated) != SQLITE_ROW) { - // No rows returned, so the IP is available - sqlite3_reset(_sAllocateIp); - sqlite3_bind_text(_sAllocateIp,1,network.id,16,SQLITE_STATIC); - sqlite3_bind_text(_sAllocateIp,2,member.nodeId,10,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,3,(int)0); - sqlite3_bind_blob(_sAllocateIp,4,(const void *)ipBlob,16,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,5,routedNetmaskBits); // IP netmask bits from matching route - sqlite3_bind_int(_sAllocateIp,6,4); // 4 == IPv4 - if (sqlite3_step(_sAllocateIp) == SQLITE_DONE) { - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); - v4ip->sin_family = AF_INET; - v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); - v4ip->sin_addr.s_addr = Utils::hton(ip); - } - break; + InetAddress ip4(Utils::hton(ip),0); + + // If it's routed, then try to claim and assign it and if successful end loop + if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip4))) { + ipAssignments.push_back(ip4.toIpString()); + member["ipAssignments"] = ipAssignments; + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); + v4ip->sin_family = AF_INET; + v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); + v4ip->sin_addr.s_addr = Utils::hton(ip); } + haveManagedIpv4AutoAssignment = true; + break; } } } } } - } // end lock + } - // Perform signing outside lock to enable concurrency - if (network.isPrivate) { + if (network.value("private",true)) { CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); if (com.sign(signingId)) { nc.com = com; @@ -1050,8 +906,8 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co } } + _writeJson(memberJP,member); return NetworkController::NETCONF_QUERY_OK; - */ } unsigned int SqliteNetworkController::handleControlPlaneHttpGET( @@ -1062,7 +918,138 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpGET( std::string &responseBody, std::string &responseContentType) { - return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType); + if ((path.size() > 0)&&(path[0] == "network")) { + + if ((path.size() >= 2)&&(path[1].length() == 16)) { + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; + + if (path.size() >= 3) { + + if (path[2] == "member") { + + if (path.size() >= 4) { + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + + char addrs[24]; + Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + + member["clock"] = OSUtils::now(); + responseBody = member.dump(2); + responseContentType = "application/json"; + + return 200; + } else { + + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + const std::string rc = member.value("memberRevision","0"); + responseBody.append(rc); + } + } + } + responseBody.push_back('}'); + responseContentType = "application/json"; + + return 200; + } + + } else if ((path[2] == "active")&&(path.size() == 3)) { + + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + auto recentLog = member["recentLog"]; + if ((recentLog.is_array())&&(recentLog.size() > 0)) { + auto mostRecentLog = recentLog[0]; + if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + responseBody.append(mostRecentLog.dump()); + } + } + } + } + } + responseBody.push_back('}'); + responseContentType = "application/json"; + return 200; + + } else if ((path[2] == "test")&&(path.size() >= 4)) { + + Mutex::Lock _l(_circuitTests_m); + std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); + if ((cte != _circuitTests.end())&&(cte->second.test)) { + + responseBody = "["; + responseBody.append(cte->second.jsonResults); + responseBody.push_back(']'); + responseContentType = "application/json"; + + return 200; + + } // else 404 + + } // else 404 + + } else { + + nlohmann::json o(network); + o["clock"] = OSUtils::now(); + responseBody = o.dump(2); + responseContentType = "application/json"; + return 200; + + } + } else if (path.size() == 1) { + + responseBody = "["; + std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); + for(auto i(networks.begin());i!=networks.end();++i) { + if (i->length() == 16) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\""); + } + } + responseBody.push_back(']'); + responseContentType = "application/json"; + return 200; + + } // else 404 + + } else { + + char tmp[4096]; + Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + responseBody = tmp; + responseContentType = "application/json"; + return 200; + + } + + return 404; } unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( @@ -1076,6 +1063,15 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( if (path.empty()) return 404; + json b; + try { + b = json::parse(body); + if (!b.is_object()) + return 400; + } catch ( ... ) { + return 400; + } + if (path[0] == "network") { if ((path.size() >= 2)&&(path[1].length() == 16)) { @@ -1090,144 +1086,48 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - /* - int64_t addToNetworkRevision = 0; - - int64_t memberRowId = 0; - sqlite3_reset(_sGetMember); - sqlite3_bind_text(_sGetMember,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetMember,2,addrs,10,SQLITE_STATIC); - bool memberExists = false; - if (sqlite3_step(_sGetMember) == SQLITE_ROW) { - memberExists = true; - memberRowId = sqlite3_column_int64(_sGetMember,0); - } - - if (!memberExists) { - sqlite3_reset(_sCreateMember); - sqlite3_bind_text(_sCreateMember,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sCreateMember,2,addrs,10,SQLITE_STATIC); - sqlite3_bind_int(_sCreateMember,3,0); - sqlite3_bind_text(_sCreateMember,4,nwids,16,SQLITE_STATIC); - if (sqlite3_step(_sCreateMember) != SQLITE_DONE) - return 500; - memberRowId = (int64_t)sqlite3_last_insert_rowid(_db); - - sqlite3_reset(_sIncrementMemberRevisionCounter); - sqlite3_bind_text(_sIncrementMemberRevisionCounter,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sIncrementMemberRevisionCounter); - addToNetworkRevision = 1; - } - - json_value *j = json_parse(body.c_str(),body.length()); - if (j) { - if (j->type == json_object) { - for(unsigned int k=0;ku.object.length;++k) { - - if (!strcmp(j->u.object.values[k].name,"authorized")) { - if (j->u.object.values[k].value->type == json_boolean) { - sqlite3_reset(_sUpdateMemberAuthorized); - sqlite3_bind_int(_sUpdateMemberAuthorized,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - sqlite3_bind_text(_sUpdateMemberAuthorized,2,nwids,16,SQLITE_STATIC); - sqlite3_bind_int64(_sUpdateMemberAuthorized,3,memberRowId); - if (sqlite3_step(_sUpdateMemberAuthorized) != SQLITE_DONE) - return 500; - - sqlite3_reset(_sIncrementMemberRevisionCounter); - sqlite3_bind_text(_sIncrementMemberRevisionCounter,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sIncrementMemberRevisionCounter); - addToNetworkRevision = 1; - } - } else if (!strcmp(j->u.object.values[k].name,"activeBridge")) { - if (j->u.object.values[k].value->type == json_boolean) { - sqlite3_reset(_sUpdateMemberActiveBridge); - sqlite3_bind_int(_sUpdateMemberActiveBridge,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - sqlite3_bind_text(_sUpdateMemberActiveBridge,2,nwids,16,SQLITE_STATIC); - sqlite3_bind_int64(_sUpdateMemberActiveBridge,3,memberRowId); - if (sqlite3_step(_sUpdateMemberActiveBridge) != SQLITE_DONE) - return 500; - - sqlite3_reset(_sIncrementMemberRevisionCounter); - sqlite3_bind_text(_sIncrementMemberRevisionCounter,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sIncrementMemberRevisionCounter); - addToNetworkRevision = 1; - } - } else if (!strcmp(j->u.object.values[k].name,"ipAssignments")) { - if (j->u.object.values[k].value->type == json_array) { - sqlite3_reset(_sDeleteIpAllocations); - sqlite3_bind_text(_sDeleteIpAllocations,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sDeleteIpAllocations,2,addrs,10,SQLITE_STATIC); - sqlite3_bind_int(_sDeleteIpAllocations,3,(int)0); - if (sqlite3_step(_sDeleteIpAllocations) != SQLITE_DONE) - return 500; - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *ipalloc = j->u.object.values[k].value->u.array.values[kk]; - if (ipalloc->type == json_string) { - InetAddress a(ipalloc->u.string.ptr); - char ipBlob[16]; - int ipVersion = 0; - _ipToBlob(a,ipBlob,ipVersion); - if (ipVersion > 0) { - sqlite3_reset(_sAllocateIp); - sqlite3_bind_text(_sAllocateIp,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sAllocateIp,2,addrs,10,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,3,(int)0); - sqlite3_bind_blob(_sAllocateIp,4,(const void *)ipBlob,16,SQLITE_STATIC); - sqlite3_bind_int(_sAllocateIp,5,(int)a.netmaskBits()); // NOTE: this field is now ignored but set it anyway - sqlite3_bind_int(_sAllocateIp,6,ipVersion); - if (sqlite3_step(_sAllocateIp) != SQLITE_DONE) - return 500; - } - } - } - addToNetworkRevision = 1; - } - } else if (!strcmp(j->u.object.values[k].name,"identity")) { - // Identity is technically an immutable field, but if the member's Node has - // no identity we allow it to be populated. This is primarily for migrating - // node data from another controller. - json_value *idstr = j->u.object.values[k].value; - if (idstr->type == json_string) { - bool alreadyHaveIdentity = false; - - sqlite3_reset(_sGetNodeIdentity); - sqlite3_bind_text(_sGetNodeIdentity,1,addrs,10,SQLITE_STATIC); - if (sqlite3_step(_sGetNodeIdentity) == SQLITE_ROW) { - const char *tmp2 = (const char *)sqlite3_column_text(_sGetNodeIdentity,0); - if ((tmp2)&&(tmp2[0])) - alreadyHaveIdentity = true; - } - - if (!alreadyHaveIdentity) { - try { - Identity id2(idstr->u.string.ptr); - if (id2) { - std::string idstr2(id2.toString(false)); // object must persist until after sqlite3_step() for SQLITE_STATIC - sqlite3_reset(_sCreateOrReplaceNode); - sqlite3_bind_text(_sCreateOrReplaceNode,1,addrs,10,SQLITE_STATIC); - sqlite3_bind_text(_sCreateOrReplaceNode,2,idstr2.c_str(),-1,SQLITE_STATIC); - sqlite3_step(_sCreateOrReplaceNode); - } - } catch ( ... ) {} // ignore invalid identities - } + char addrs[24]; + Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); + + json member(_readJson(_memberJP(nwid,Address(address),true))); + + try { + if (b.count("authorized")) member["authorized"] = b.value("authorized",false); + if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known + if (b.count("ipAssignments")) { + auto ipa = b["ipAssignments"]; + if (ipa.is_array()) { + json mipa(json::array()); + for(unsigned long i=0;i 0)&&(revision > 0)) { - sqlite3_reset(_sSetNetworkRevision); - sqlite3_bind_int64(_sSetNetworkRevision,1,revision + addToNetworkRevision); - sqlite3_bind_text(_sSetNetworkRevision,2,nwids,16,SQLITE_STATIC); - sqlite3_step(_sSetNetworkRevision); - } - */ + if (!member.count("authorized")) member["authorized"] = false; + if (!member.count("ipAssignments")) member["ipAssignments"] = json::array(); + if (!member.count("recentLog")) member["recentLog"] = json::array(); + + member["id"] = addrs; + member["address"] = addrs; // legacy + member["nwid"] = nwids; + member["lastModified"] = OSUtils::now(); + member["memberRevision"] = member.value("memberRevision",0ULL) + 1; - return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType); + _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); + + member["clock"] = OSUtils::now(); + responseBody = member.dump(2); + responseContentType = "application/json"; + return 200; } else if ((path.size() == 3)&&(path[2] == "test")) { Mutex::Lock _l(_circuitTests_m); @@ -1238,42 +1138,26 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( Utils::getSecureRandom(&(test->testId),sizeof(test->testId)); test->credentialNetworkId = nwid; test->ptr = (void *)this; - - // TODO TODO - /* - json_value *j = json_parse(body.c_str(),body.length()); - if (j) { - if (j->type == json_object) { - for(unsigned int k=0;ku.object.length;++k) { - - if (!strcmp(j->u.object.values[k].name,"hops")) { - if (j->u.object.values[k].value->type == json_array) { - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *hop = j->u.object.values[k].value->u.array.values[kk]; - if (hop->type == json_array) { - for(unsigned int kkk=0;kkku.array.length;++kkk) { - if (hop->u.array.values[kkk]->type == json_string) { - test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(hop->u.array.values[kkk]->u.string.ptr) & 0xffffffffffULL; - } - } - ++test->hopCount; - } - } - } - } else if (!strcmp(j->u.object.values[k].name,"reportAtEveryHop")) { - if (j->u.object.values[k].value->type == json_boolean) - test->reportAtEveryHop = (j->u.object.values[k].value->u.boolean == 0) ? 0 : 1; + json hops = b["hops"]; + if (hops.is_array()) { + for(unsigned long i=0;ihops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(hop.c_str()) & 0xffffffffffULL; } - + } else if (hops2.is_string()) { + std::string hop = hops2; + test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(hop.c_str()) & 0xffffffffffULL; } } - json_value_free(j); } - */ + test->reportAtEveryHop = (b.value("reportAtEveryHop",true) ? 1 : 0); if (!test->hopCount) { ::free((void *)test); - return 500; + return 400; } test->timestamp = OSUtils::now(); @@ -1295,13 +1179,6 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } else { // POST to network ID - json b; - try { - b = json::parse(body); - } catch ( ... ) { - return 403; - } - // Magic ID ending with ______ picks a random unused network ID if (path[1].substr(10) == "______") { nwid = 0; @@ -1330,6 +1207,19 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); + if (b.count("activeBridges")) { + auto ab = b["activeBridges"]; + if (ab.is_array()) { + json ab2 = json::array(); + for(unsigned long i=0;i &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - // Assumes _lock is locked - if ((path.size() > 0)&&(path[0] == "network")) { - - if ((path.size() >= 2)&&(path[1].length() == 16)) { - const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return 404; - - if (path.size() >= 3) { - - if (path[2] == "member") { - - if (path.size() >= 4) { - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - json member(_readJson(_memberJP(nwid,Address(address),false))); - if (!member.size()) - return 404; - - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - - json o(member); - o["nwid"] = nwids; - o["address"] = addrs; - o["clock"] = OSUtils::now(); - responseBody = o.dump(2); - responseContentType = "application/json"; - - return 200; - } else { - - responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - const std::string rc = member.value("memberRevision","0"); - responseBody.append(rc); - } - } - } - responseBody.push_back('}'); - responseContentType = "application/json"; - - return 200; - } - - } else if ((path[2] == "active")&&(path.size() == 3)) { - - responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - auto recentLog = member["recentLog"]; - if ((recentLog.is_array())&&(recentLog.size() > 0)) { - auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - responseBody.append(mostRecentLog.dump()); - } - } - } - } - } - responseBody.push_back('}'); - responseContentType = "application/json"; - return 200; - - } else if ((path[2] == "test")&&(path.size() >= 4)) { - - Mutex::Lock _l(_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); - if ((cte != _circuitTests.end())&&(cte->second.test)) { - - responseBody = "["; - responseBody.append(cte->second.jsonResults); - responseBody.push_back(']'); - responseContentType = "application/json"; - - return 200; - - } // else 404 - - } // else 404 - - } else { - - nlohmann::json o(network); - o["nwid"] = nwids; - o["clock"] = OSUtils::now(); - responseBody = o.dump(2); - responseContentType = "application/json"; - return 200; - - } - } else if (path.size() == 1) { - - responseBody = "["; - std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); - for(auto i(networks.begin());i!=networks.end();++i) { - if (i->length() == 16) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\""); - } - } - responseBody.push_back(']'); - responseContentType = "application/json"; - return 200; - - } // else 404 - - } else { - - char tmp[4096]; - Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); - responseBody = tmp; - responseContentType = "application/json"; - return 200; - - } - - return 404; -} - void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) { char tmp[65535]; diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp index 15e0968f..288ea23a 100644 --- a/controller/SqliteNetworkController.hpp +++ b/controller/SqliteNetworkController.hpp @@ -24,12 +24,14 @@ #include #include #include +#include #include "../node/Constants.hpp" #include "../node/NetworkController.hpp" #include "../node/Mutex.hpp" #include "../node/Utils.hpp" +#include "../node/InetAddress.hpp" #include "../osdep/OSUtils.hpp" @@ -76,14 +78,6 @@ public: std::string &responseContentType); private: - unsigned int _doCPGet( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType); - static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); inline nlohmann::json _readJson(const std::string &path) @@ -134,6 +128,28 @@ private: return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); } + inline std::set _getAlreadyAllocatedIps(const uint64_t nwid) + { + std::set ips; + std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); + std::vector members(OSUtils::listSubdirectories(bp.c_str())); + for(std::vector::iterator m(members.begin());m!=members.end();++m) { + if (m->length() == ZT_ADDRESS_LENGTH_HEX) { + nlohmann::json mj = _readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"); + auto ipAssignments = mj["ipAssignments"]; + if (ipAssignments.is_array()) { + for(unsigned long i=0;i Date: Tue, 16 Aug 2016 16:57:45 -0700 Subject: . --- controller/SqliteNetworkController.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 885d12af..05cee7c7 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -644,7 +644,6 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co // Update member log { - json recentLog = json::array(); json rlEntry = json::object(); rlEntry["ts"] = now; rlEntry["authorized"] = member["authorized"]; @@ -654,8 +653,9 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); if (fromAddr) rlEntry["fromAddr"] = fromAddr.toString(); + json recentLog = json::array(); recentLog.push_back(rlEntry); - json oldLog = member["recentLog"]; + auto oldLog = member["recentLog"]; if (oldLog.is_array()) { for(unsigned long i=0;i Date: Wed, 17 Aug 2016 10:25:25 -0700 Subject: Rules parsing stuff. --- controller/SqliteNetworkController.cpp | 51 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 24 deletions(-) (limited to 'controller') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 05cee7c7..863f93f3 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -14,15 +14,6 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ */ #include @@ -639,7 +630,6 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co member["id"] = identity.address().toString(); member["address"] = member["id"]; member["nwid"] = network["id"]; - member["lastModified"] = now; member["memberRevision"] = member.value("memberRevision",0ULL) + 1; // Update member log @@ -666,14 +656,12 @@ NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(co member["recentLog"] = recentLog; } - if (!member.value("authorized",false)) { - if (network.value("private",true)) { - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } else { - member["authorized"] = true; // auto-authorize on public networks - } + // Stop if network is private and member is not authorized + if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } + // Else compose and send network config nc.networkId = nwid; nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; @@ -1094,6 +1082,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( try { if (b.count("authorized")) member["authorized"] = b.value("authorized",false); if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known + if (b.count("ipAssignments")) { auto ipa = b["ipAssignments"]; if (ipa.is_array()) { @@ -1119,8 +1108,8 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( member["id"] = addrs; member["address"] = addrs; // legacy member["nwid"] = nwids; - member["lastModified"] = OSUtils::now(); member["memberRevision"] = member.value("memberRevision",0ULL) + 1; + member["objtype"] = "member"; _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); @@ -1144,12 +1133,12 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( auto hops2 = hops[i]; if (hops2.is_array()) { for(unsigned long j=0;jhops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(hop.c_str()) & 0xffffffffffULL; + std::string s = hops2[j]; + test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; } } else if (hops2.is_string()) { - std::string hop = hops2; - test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(hop.c_str()) & 0xffffffffffULL; + std::string s = hops2; + test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; } } } @@ -1303,11 +1292,20 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( if (b.count("rules")) { auto rules = b["rules"]; if (rules.is_array()) { + json nrules = json::array(); for(unsigned long i=0;i Date: Wed, 17 Aug 2016 10:42:32 -0700 Subject: We now always build the controller in ZeroTier One, at least for desktop and server targets. Also means that ZeroTier One now requires C++11. (Still keeping C++11 out of the core in node/ though.) --- controller/EmbeddedNetworkController.cpp | 1462 ++++++++++++++++++++++++++++++ controller/EmbeddedNetworkController.hpp | 173 ++++ controller/SqliteNetworkController.cpp | 1462 ------------------------------ controller/SqliteNetworkController.hpp | 173 ---- make-linux.mk | 6 - make-mac.mk | 6 - objects.mk | 1 + selftest.cpp | 4 - service/ControlPlane.cpp | 20 +- service/ControlPlane.hpp | 10 +- service/OneService.cpp | 25 +- service/OneService.hpp | 12 - 12 files changed, 1646 insertions(+), 1708 deletions(-) create mode 100644 controller/EmbeddedNetworkController.cpp create mode 100644 controller/EmbeddedNetworkController.hpp delete mode 100644 controller/SqliteNetworkController.cpp delete mode 100644 controller/SqliteNetworkController.hpp (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp new file mode 100644 index 00000000..479a65f5 --- /dev/null +++ b/controller/EmbeddedNetworkController.cpp @@ -0,0 +1,1462 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../include/ZeroTierOne.h" +#include "../node/Constants.hpp" + +#include "EmbeddedNetworkController.hpp" + +#include "../node/Node.hpp" +#include "../node/Utils.hpp" +#include "../node/CertificateOfMembership.hpp" +#include "../node/NetworkConfig.hpp" +#include "../node/Dictionary.hpp" +#include "../node/InetAddress.hpp" +#include "../node/MAC.hpp" +#include "../node/Address.hpp" + +using json = nlohmann::json; + +// API version reported via JSON control plane +#define ZT_NETCONF_CONTROLLER_API_VERSION 3 + +// Number of requests to remember in member history +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 8 + +// Min duration between requests for an address/nwid combo to prevent floods +#define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 + +// Nodes are considered active if they've queried in less than this long +#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) + +namespace ZeroTier { + +static json _renderRule(ZT_VirtualNetworkRule &rule) +{ + char tmp[128]; + json r = json::object(); + r["not"] = ((rule.t & 0x80) != 0); + switch((rule.t) & 0x7f) { + case ZT_NETWORK_RULE_ACTION_DROP: + r["type"] = "ACTION_DROP"; + break; + case ZT_NETWORK_RULE_ACTION_ACCEPT: + r["type"] = "ACTION_ACCEPT"; + break; + case ZT_NETWORK_RULE_ACTION_TEE: + r["type"] = "ACTION_TEE"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_ACTION_REDIRECT: + r["type"] = "ACTION_REDIRECT"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + r["type"] = "MATCH_VLAN_ID"; + r["vlanId"] = (uint64_t)rule.v.vlanId; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + r["type"] = "MATCH_VLAN_PCP"; + r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + r["type"] = "MATCH_VLAN_DEI"; + r["vlanDei"] = (uint64_t)rule.v.vlanDei; + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + r["type"] = "MATCH_ETHERTYPE"; + r["etherType"] = (uint64_t)rule.v.etherType; + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + r["type"] = "MATCH_MAC_SOURCE"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + r["type"] = "MATCH_MAC_DEST"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + r["type"] = "MATCH_IPV4_SOURCE"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + r["type"] = "MATCH_IPV4_DEST"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + r["type"] = "MATCH_IPV6_SOURCE"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + r["type"] = "MATCH_IPV6_DEST"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + r["type"] = "MATCH_IP_TOS"; + r["ipTos"] = (uint64_t)rule.v.ipTos; + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + r["type"] = "MATCH_IP_PROTOCOL"; + r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + r["type"] = "MATCH_CHARACTERISTICS"; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); + r["mask"] = tmp; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); + r["value"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["start"] = (uint64_t)rule.v.frameSize[0]; + r["end"] = (uint64_t)rule.v.frameSize[1]; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + r["type"] = "MATCH_TAGS_SAMENESS"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + } + return r; +} + +static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) +{ + if (r.is_object()) + return false; + std::string t = r["type"]; + memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); + if (r.value("not",false)) + rule.t = 0x80; + else rule.t = 0x00; + if (t == "ACTION_DROP") { + rule.t |= ZT_NETWORK_RULE_ACTION_DROP; + return true; + } else if (t == "ACTION_ACCEPT") { + rule.t |= ZT_NETWORK_RULE_ACTION_ACCEPT; + return true; + } else if (t == "ACTION_TEE") { + rule.t |= ZT_NETWORK_RULE_ACTION_TEE; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "ACTION_REDIRECT") { + rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_VLAN_ID") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; + rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_VLAN_PCP") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; + rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_VLAN_DEI") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; + rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_ETHERTYPE") { + rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; + rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_MAC_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_MAC_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_IPV4_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV4_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV6_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IPV6_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IP_TOS") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; + rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_PROTOCOL") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; + rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_IP_DEST_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_CHARACTERISTICS") { + rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; + if (r.count("mask")) { + auto v = r["mask"]; + if (v.is_number()) { + rule.v.characteristics[0] = v; + } else { + std::string tmp = v; + rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); + } + } + if (r.count("value")) { + auto v = r["value"]; + if (v.is_number()) { + rule.v.characteristics[1] = v; + } else { + std::string tmp = v; + rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); + } + } + return true; + } else if (t == "MATCH_FRAME_SIZE_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; + rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0").c_str()) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0").c_str()) & 0xffffULL); + return true; + } else if (t == "MATCH_TAGS_SAMENESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_AND") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_OR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_XOR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } + return false; +} + +EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : + _node(node), + _path(dbPath) +{ + OSUtils::mkdir(dbPath); + /* + if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) + throw std::runtime_error("SqliteNetworkController cannot open database file"); + sqlite3_busy_timeout(_db,10000); + + sqlite3_exec(_db,"PRAGMA synchronous = OFF",0,0,0); + sqlite3_exec(_db,"PRAGMA journal_mode = MEMORY",0,0,0); + + sqlite3_stmt *s = (sqlite3_stmt *)0; + if ((sqlite3_prepare_v2(_db,"SELECT v FROM Config WHERE k = 'schemaVersion';",-1,&s,(const char **)0) == SQLITE_OK)&&(s)) { + int schemaVersion = -1234; + if (sqlite3_step(s) == SQLITE_ROW) { + schemaVersion = sqlite3_column_int(s,0); + } + + sqlite3_finalize(s); + + if (schemaVersion == -1234) { + sqlite3_close(_db); + throw std::runtime_error("SqliteNetworkController schemaVersion not found in Config table (init failure?)"); + } + + if (schemaVersion < 2) { + // Create NodeHistory table to upgrade from version 1 to version 2 + if (sqlite3_exec(_db, + "CREATE TABLE NodeHistory (\n" + " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " networkVisitCounter INTEGER NOT NULL DEFAULT(0),\n" + " networkRequestAuthorized INTEGER NOT NULL DEFAULT(0),\n" + " requestTime INTEGER NOT NULL DEFAULT(0),\n" + " clientMajorVersion INTEGER NOT NULL DEFAULT(0),\n" + " clientMinorVersion INTEGER NOT NULL DEFAULT(0),\n" + " clientRevision INTEGER NOT NULL DEFAULT(0),\n" + " networkRequestMetaData VARCHAR(1024),\n" + " fromAddress VARCHAR(128)\n" + ");\n" + "CREATE INDEX NodeHistory_nodeId ON NodeHistory (nodeId);\n" + "CREATE INDEX NodeHistory_networkId ON NodeHistory (networkId);\n" + "CREATE INDEX NodeHistory_requestTime ON NodeHistory (requestTime);\n" + "UPDATE \"Config\" SET \"v\" = 2 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 2: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 2; + } + } + + if (schemaVersion < 3) { + // Create Route table to upgrade from version 2 to version 3 and migrate old + // data. Also delete obsolete Gateway table that was never actually used, and + // migrate Network flags to a bitwise flags field instead of ASCII cruft. + if (sqlite3_exec(_db, + "DROP TABLE Gateway;\n" + "CREATE TABLE Route (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " target blob(16) NOT NULL,\n" + " via blob(16),\n" + " targetNetmaskBits integer NOT NULL,\n" + " ipVersion integer NOT NULL,\n" + " flags integer NOT NULL,\n" + " metric integer NOT NULL\n" + ");\n" + "CREATE INDEX Route_networkId ON Route (networkId);\n" + "INSERT INTO Route SELECT DISTINCT networkId,\"ip\" AS \"target\",NULL AS \"via\",ipNetmaskBits AS targetNetmaskBits,ipVersion,0 AS \"flags\",0 AS \"metric\" FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" + "ALTER TABLE Network ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" + "UPDATE Network SET \"flags\" = (\"flags\" | 1) WHERE v4AssignMode = 'zt';\n" + "UPDATE Network SET \"flags\" = (\"flags\" | 2) WHERE v6AssignMode = 'rfc4193';\n" + "UPDATE Network SET \"flags\" = (\"flags\" | 4) WHERE v6AssignMode = '6plane';\n" + "ALTER TABLE Member ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" + "DELETE FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" + "UPDATE \"Config\" SET \"v\" = 3 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 3; + } + } + + if (schemaVersion < 4) { + // Turns out this was overkill and a huge performance drag. Will be revisiting this + // more later but for now a brief snapshot of recent history stored in Member is fine. + // Also prepare for implementation of proof of work requests. + if (sqlite3_exec(_db, + "DROP TABLE NodeHistory;\n" + "ALTER TABLE Member ADD COLUMN lastRequestTime integer NOT NULL DEFAULT(0);\n" + "ALTER TABLE Member ADD COLUMN lastPowDifficulty integer NOT NULL DEFAULT(0);\n" + "ALTER TABLE Member ADD COLUMN lastPowTime integer NOT NULL DEFAULT(0);\n" + "ALTER TABLE Member ADD COLUMN recentHistory blob;\n" + "CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n" + "UPDATE \"Config\" SET \"v\" = 4 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 4; + } + } + + if (schemaVersion < 5) { + // Upgrade old rough draft Rule table to new release format + if (sqlite3_exec(_db, + "DROP TABLE Relay;\n" + "DROP INDEX Rule_networkId_ruleNo;\n" + "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" + "CREATE TABLE Rule (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " capId integer,\n" + " ruleNo integer NOT NULL,\n" + " ruleType integer NOT NULL DEFAULT(0),\n" + " \"addr\" blob(16),\n" + " \"int1\" integer,\n" + " \"int2\" integer,\n" + " \"int3\" integer,\n" + " \"int4\" integer\n" + ");\n" + "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" + "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" + "DROP TABLE RuleOld;\n" + "CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n" + "CREATE TABLE MemberTC (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" + " tagId integer,\n" + " tagValue integer,\n" + " capId integer,\n" + " capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n" + ");\n" + "CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n" + "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 5; + } + } + + if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { + sqlite3_close(_db); + throw std::runtime_error("SqliteNetworkController database schema version mismatch"); + } + } else { + // Prepare statement will fail if Config table doesn't exist, which means our DB + // needs to be initialized. + if (sqlite3_exec(_db,ZT_NETCONF_SCHEMA_SQL"INSERT INTO Config (k,v) VALUES ('schemaVersion',"ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR");",0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot initialize database and/or insert schemaVersion into Config table: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } + } + + if ( + + (sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO Network (id,name,creationTime,revision) VALUES (?,?,?,1)",-1,&_sCreateNetwork,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Network WHERE id = ?",-1,&_sDeleteNetwork,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT nodeId FROM Member WHERE networkId = ? AND activeBridge > 0 AND authorized > 0",-1,&_sGetActiveBridges,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT m.nodeId,m.memberRevision FROM Member AS m WHERE m.networkId = ? ORDER BY m.nodeId ASC",-1,&_sListNetworkMembers,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Member SET authorized = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberAuthorized,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Member SET activeBridge = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberActiveBridge,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Member SET \"lastRequestTime\" = ?, \"recentHistory\" = ? WHERE rowid = ?",-1,&_sUpdateMemberHistory,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sDeleteMember,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(const char **)0) != SQLITE_OK) + + ) { + std::string err(std::string("SqliteNetworkController unable to initialize one or more prepared statements: ") + sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } + + sqlite3_reset(_sGetConfig); + sqlite3_bind_text(_sGetConfig,1,"instanceId",10,SQLITE_STATIC); + if (sqlite3_step(_sGetConfig) != SQLITE_ROW) { + unsigned char sr[32]; + Utils::getSecureRandom(sr,32); + for(unsigned int i=0;i<32;++i) + _instanceId.push_back("0123456789abcdef"[(unsigned int)sr[i] & 0xf]); + + sqlite3_reset(_sSetConfig); + sqlite3_bind_text(_sSetConfig,1,"instanceId",10,SQLITE_STATIC); + sqlite3_bind_text(_sSetConfig,2,_instanceId.c_str(),-1,SQLITE_STATIC); + if (sqlite3_step(_sSetConfig) != SQLITE_DONE) + throw std::runtime_error("SqliteNetworkController unable to read or initialize instanceId"); + } else { + const char *iid = reinterpret_cast(sqlite3_column_text(_sGetConfig,0)); + if (!iid) + throw std::runtime_error("SqliteNetworkController unable to read instanceId (it's NULL)"); + _instanceId = iid; + } + +#ifdef ZT_NETCONF_SQLITE_TRACE + sqlite3_trace(_db,sqliteTraceFunc,(void *)0); +#endif + + _backupThread = Thread::start(this); + */ +} + +EmbeddedNetworkController::~EmbeddedNetworkController() +{ +} + +NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) +{ + if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { + return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; + } + + const uint64_t now = OSUtils::now(); + + // Check rate limit circuit breaker to prevent flooding + { + Mutex::Lock _l(_lastRequestTime_m); + uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; + if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + return NetworkController::NETCONF_QUERY_IGNORE; + lrt = now; + } + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; + const std::string memberJP(_memberJP(nwid,identity.address(),false)); + json member(_readJson(memberJP)); + + { + std::string haveIdStr = member.value("identity",""); + if (haveIdStr.length() > 0) { + try { + if (Identity(haveIdStr.c_str()) != identity) + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } catch ( ... ) { + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } + } else { + member["identity"] = identity.toString(false); + } + } + + // Make sure these are always present no matter what, and increment member revision since we will always at least log something + member["id"] = identity.address().toString(); + member["address"] = member["id"]; + member["nwid"] = network["id"]; + member["memberRevision"] = member.value("memberRevision",0ULL) + 1; + + // Update member log + { + json rlEntry = json::object(); + rlEntry["ts"] = now; + rlEntry["authorized"] = member["authorized"]; + rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); + if (fromAddr) + rlEntry["fromAddr"] = fromAddr.toString(); + json recentLog = json::array(); + recentLog.push_back(rlEntry); + auto oldLog = member["recentLog"]; + if (oldLog.is_array()) { + for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) + break; + } + } + member["recentLog"] = recentLog; + } + + // Stop if network is private and member is not authorized + if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } + // Else compose and send network config + + nc.networkId = nwid; + nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc.timestamp = now; + nc.revision = network.value("revision",0ULL); + nc.issuedTo = identity.address(); + if (network.value("enableBroadcast",true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (network.value("allowPassiveBridging",false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc.name,sizeof(nc.name),network.value("name","").c_str()); + nc.multicastLimit = (unsigned int)network.value("multicastLimit",32ULL); + + bool amActiveBridge = false; + { + json ab = network["activeBridges"]; + if (ab.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + break; + auto rule = rules[i]; + if (_parseRule(rule,nc.rules[nc.ruleCount])) + ++nc.ruleCount; + } + } + + if (routes.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) + break; + auto route = routes[i]; + InetAddress t(route.value("target","")); + InetAddress v(route.value("via","")); + if ((t)&&(v)&&(t.ss_family == v.ss_family)) { + ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + *(reinterpret_cast(&(r->target))) = t; + *(reinterpret_cast(&(r->via))) = v; + ++nc.routeCount; + } + } + } + + bool haveManagedIpv4AutoAssignment = false; + bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count + json ipAssignments = member["ipAssignments"]; + if (ipAssignments.is_array()) { + for(unsigned long i=0;i(&(nc.routes[rk].target))->containsAddress(ip)) ) + routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + } + + if (routedNetmaskBits > 0) { + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + ip.setPort(routedNetmaskBits); + nc.staticIps[nc.staticIpCount++] = ip; + } + if (ip.ss_family == AF_INET) + haveManagedIpv4AutoAssignment = true; + else if (ip.ss_family == AF_INET6) + haveManagedIpv6AutoAssignment = true; + } + } + } else { + ipAssignments = json::array(); + } + + std::set allocatedIps; + bool allocatedIpsLoaded = false; + + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(v6AssignMode.value("zt",false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { + if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + for(unsigned long p=0;((p s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { + // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. + xx[0] = Utils::hton(x[0]); + xx[1] = Utils::hton(x[1] + identity.address().toInt()); + } else { + // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway + Utils::getSecureRandom((void *)xx,16); + if ((e[0] > s[0])) + xx[0] %= (e[0] - s[0]); + else xx[0] = 0; + if ((e[1] > s[1])) + xx[1] %= (e[1] - s[1]); + else xx[1] = 0; + xx[0] = Utils::hton(x[0] + xx[0]); + xx[1] = Utils::hton(x[1] + xx[1]); + } + + InetAddress ip6((const void *)xx,16,0); + + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = 0; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) + routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + } + + // If it's routed, then try to claim and assign it and if successful end loop + if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip6))) { + ipAssignments.push_back(ip6.toIpString()); + member["ipAssignments"] = ipAssignments; + ip6.setPort((unsigned int)routedNetmaskBits); + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) + nc.staticIps[nc.staticIpCount++] = ip6; + haveManagedIpv4AutoAssignment = true; + break; + } + } + } + } + } + } + + if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(v4AssignMode.value("zt",false))) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { + if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + for(unsigned long p=0;((p(&ipRangeStart)->sin_addr.s_addr)); + uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); + if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) + continue; + uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; + + // Start with the LSB of the member's address + uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); + + for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { + uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; + ++ipTrialCounter; + if ((ip & 0x000000ff) == 0x000000ff) + continue; // don't allow addresses that end in .255 + + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = 0; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); + int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); + if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { + routedNetmaskBits = targetBits; + break; + } + } + } + + InetAddress ip4(Utils::hton(ip),0); + + // If it's routed, then try to claim and assign it and if successful end loop + if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip4))) { + ipAssignments.push_back(ip4.toIpString()); + member["ipAssignments"] = ipAssignments; + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); + v4ip->sin_family = AF_INET; + v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); + v4ip->sin_addr.s_addr = Utils::hton(ip); + } + haveManagedIpv4AutoAssignment = true; + break; + } + } + } + } + } + } + + if (network.value("private",true)) { + CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); + if (com.sign(signingId)) { + nc.com = com; + } else { + return NETCONF_QUERY_INTERNAL_SERVER_ERROR; + } + } + + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_OK; +} + +unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if ((path.size() > 0)&&(path[0] == "network")) { + + if ((path.size() >= 2)&&(path[1].length() == 16)) { + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; + + if (path.size() >= 3) { + + if (path[2] == "member") { + + if (path.size() >= 4) { + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + + char addrs[24]; + Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + + member["clock"] = OSUtils::now(); + responseBody = member.dump(2); + responseContentType = "application/json"; + + return 200; + } else { + + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + const std::string rc = member.value("memberRevision","0"); + responseBody.append(rc); + } + } + } + responseBody.push_back('}'); + responseContentType = "application/json"; + + return 200; + } + + } else if ((path[2] == "active")&&(path.size() == 3)) { + + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + auto recentLog = member["recentLog"]; + if ((recentLog.is_array())&&(recentLog.size() > 0)) { + auto mostRecentLog = recentLog[0]; + if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + responseBody.append(mostRecentLog.dump()); + } + } + } + } + } + responseBody.push_back('}'); + responseContentType = "application/json"; + return 200; + + } else if ((path[2] == "test")&&(path.size() >= 4)) { + + Mutex::Lock _l(_circuitTests_m); + std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); + if ((cte != _circuitTests.end())&&(cte->second.test)) { + + responseBody = "["; + responseBody.append(cte->second.jsonResults); + responseBody.push_back(']'); + responseContentType = "application/json"; + + return 200; + + } // else 404 + + } // else 404 + + } else { + + nlohmann::json o(network); + o["clock"] = OSUtils::now(); + responseBody = o.dump(2); + responseContentType = "application/json"; + return 200; + + } + } else if (path.size() == 1) { + + responseBody = "["; + std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); + for(auto i(networks.begin());i!=networks.end();++i) { + if (i->length() == 16) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\""); + } + } + responseBody.push_back(']'); + responseContentType = "application/json"; + return 200; + + } // else 404 + + } else { + + char tmp[4096]; + Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + responseBody = tmp; + responseContentType = "application/json"; + return 200; + + } + + return 404; +} + +unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if (path.empty()) + return 404; + + json b; + try { + b = json::parse(body); + if (!b.is_object()) + return 400; + } catch ( ... ) { + return 400; + } + + if (path[0] == "network") { + + if ((path.size() >= 2)&&(path[1].length() == 16)) { + uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + + if (path.size() >= 3) { + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; + + if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { + uint64_t address = Utils::hexStrToU64(path[3].c_str()); + char addrs[24]; + Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); + + json member(_readJson(_memberJP(nwid,Address(address),true))); + + try { + if (b.count("authorized")) member["authorized"] = b.value("authorized",false); + if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known + + if (b.count("ipAssignments")) { + auto ipa = b["ipAssignments"]; + if (ipa.is_array()) { + json mipa(json::array()); + for(unsigned long i=0;itestId),sizeof(test->testId)); + test->credentialNetworkId = nwid; + test->ptr = (void *)this; + json hops = b["hops"]; + if (hops.is_array()) { + for(unsigned long i=0;ihops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; + } + } else if (hops2.is_string()) { + std::string s = hops2; + test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; + } + } + } + test->reportAtEveryHop = (b.value("reportAtEveryHop",true) ? 1 : 0); + + if (!test->hopCount) { + ::free((void *)test); + return 400; + } + + test->timestamp = OSUtils::now(); + + _CircuitTestEntry &te = _circuitTests[test->testId]; + te.test = test; + te.jsonResults = ""; + + _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); + + char json[1024]; + Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); + responseBody = json; + responseContentType = "application/json"; + return 200; + + } // else 404 + + } else { + // POST to network ID + + // Magic ID ending with ______ picks a random unused network ID + if (path[1].substr(10) == "______") { + nwid = 0; + uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; + uint64_t nwidPostfix = 0; + for(unsigned long k=0;k<100000;++k) { // sanity limit on trials + Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); + uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); + if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); + if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) { + nwid = tryNwid; + break; + } + } + if (!nwid) + return 503; + } + + json network(_readJson(_networkJP(nwid,true))); + + try { + if (b.count("name")) network["name"] = b.value("name",""); + if (b.count("private")) network["private"] = b.value("private",true); + if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); + if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); + + if (b.count("activeBridges")) { + auto ab = b["activeBridges"]; + if (ab.is_array()) { + json ab2 = json::array(); + for(unsigned long i=0;i v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); + std::sort(v6m.begin(),v6m.end()); + v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); + for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { + if (*i == "rfc4193") + nv6m["rfc4193"] = true; + else if (*i == "zt") + nv6m["zt"] = true; + else if (*i == "6plane") + nv6m["6plane"] = true; + } + } else if (b["v6AssignMode"].is_object()) { + auto v6m = b["v6AssignMode"]; + if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); + if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); + if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); + } + if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; + if (!nv6m.count("zt")) nv6m["zt"] = false; + if (!nv6m.count("6plane")) nv6m["6plane"] = false; + } + + if (b.count("routes")) { + auto rts = b["routes"]; + if (rts.is_array()) { + for(unsigned long i=0;i &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if (path.empty()) + return 404; + + if (path[0] == "network") { + if ((path.size() >= 2)&&(path[1].length() == 16)) { + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; + + if (path.size() >= 3) { + if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + + OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str()); + + responseBody = member.dump(2); + responseContentType = "application/json"; + return 200; + } + } else { + OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); + responseBody = network.dump(2); + responseContentType = "application/json"; + return 200; + } + } // else 404 + + } // else 404 + + return 404; +} + +void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) +{ + char tmp[65535]; + EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); + + if (!test) + return; + if (!report) + return; + + Mutex::Lock _l(self->_circuitTests_m); + std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); + + if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? + self->_node->circuitTestEnd(test); + ::free((void *)test); + return; + } + + Utils::snprintf(tmp,sizeof(tmp), + "%s{\n" + "\t\"timestamp\": %llu," ZT_EOL_S + "\t\"testId\": \"%.16llx\"," ZT_EOL_S + "\t\"upstream\": \"%.10llx\"," ZT_EOL_S + "\t\"current\": \"%.10llx\"," ZT_EOL_S + "\t\"receivedTimestamp\": %llu," ZT_EOL_S + "\t\"remoteTimestamp\": %llu," ZT_EOL_S + "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S + "\t\"flags\": %llu," ZT_EOL_S + "\t\"sourcePacketHopCount\": %u," ZT_EOL_S + "\t\"errorCode\": %u," ZT_EOL_S + "\t\"vendor\": %d," ZT_EOL_S + "\t\"protocolVersion\": %u," ZT_EOL_S + "\t\"majorVersion\": %u," ZT_EOL_S + "\t\"minorVersion\": %u," ZT_EOL_S + "\t\"revision\": %u," ZT_EOL_S + "\t\"platform\": %d," ZT_EOL_S + "\t\"architecture\": %d," ZT_EOL_S + "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S + "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S + "}", + ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), + (unsigned long long)report->timestamp, + (unsigned long long)test->testId, + (unsigned long long)report->upstream, + (unsigned long long)report->current, + (unsigned long long)OSUtils::now(), + (unsigned long long)report->remoteTimestamp, + (unsigned long long)report->sourcePacketId, + (unsigned long long)report->flags, + report->sourcePacketHopCount, + report->errorCode, + (int)report->vendor, + report->protocolVersion, + report->majorVersion, + report->minorVersion, + report->revision, + (int)report->platform, + (int)report->architecture, + reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), + reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); + + cte->second.jsonResults.append(tmp); +} + +} // namespace ZeroTier diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp new file mode 100644 index 00000000..f6e6b098 --- /dev/null +++ b/controller/EmbeddedNetworkController.hpp @@ -0,0 +1,173 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_SQLITENETWORKCONTROLLER_HPP +#define ZT_SQLITENETWORKCONTROLLER_HPP + +#include + +#include +#include +#include +#include + +#include "../node/Constants.hpp" + +#include "../node/NetworkController.hpp" +#include "../node/Mutex.hpp" +#include "../node/Utils.hpp" +#include "../node/InetAddress.hpp" + +#include "../osdep/OSUtils.hpp" + +#include "../ext/json/json.hpp" + +namespace ZeroTier { + +class Node; + +class EmbeddedNetworkController : public NetworkController +{ +public: + EmbeddedNetworkController(Node *node,const char *dbPath); + virtual ~EmbeddedNetworkController(); + + virtual NetworkController::ResultCode doNetworkConfigRequest( + const InetAddress &fromAddr, + const Identity &signingId, + const Identity &identity, + uint64_t nwid, + const Dictionary &metaData, + NetworkConfig &nc); + + unsigned int handleControlPlaneHttpGET( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType); + unsigned int handleControlPlaneHttpPOST( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType); + unsigned int handleControlPlaneHttpDELETE( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType); + +private: + static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); + + inline nlohmann::json _readJson(const std::string &path) + { + std::string buf; + if (OSUtils::readFile(path.c_str(),buf)) { + try { + return nlohmann::json::parse(buf); + } catch ( ... ) {} + } + return nlohmann::json::object(); + } + + inline bool _writeJson(const std::string &path,const nlohmann::json &obj) + { + std::string buf(obj.dump(2)); + return OSUtils::writeFile(path.c_str(),buf); + } + + inline std::string _networkBP(const uint64_t nwid,bool create) + { + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nwid); + std::string p(_path + ZT_PATH_SEPARATOR_S + "network"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(tmp); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _networkJP(const uint64_t nwid,bool create) + { + return (_networkBP(nwid,create) + ZT_PATH_SEPARATOR + "config.json"); + } + inline std::string _memberBP(const uint64_t nwid,const Address &member,bool create) + { + std::string p(_networkBP(nwid,create)); + p.push_back(ZT_PATH_SEPARATOR); + p.append("member"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(member.toString()); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _memberJP(const uint64_t nwid,const Address &member,bool create) + { + return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); + } + + inline std::set _getAlreadyAllocatedIps(const uint64_t nwid) + { + std::set ips; + std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); + std::vector members(OSUtils::listSubdirectories(bp.c_str())); + for(std::vector::iterator m(members.begin());m!=members.end();++m) { + if (m->length() == ZT_ADDRESS_LENGTH_HEX) { + nlohmann::json mj = _readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"); + auto ipAssignments = mj["ipAssignments"]; + if (ipAssignments.is_array()) { + for(unsigned long i=0;i _circuitTests; + Mutex _circuitTests_m; + + // Last request time by address, for rate limitation + std::map< std::pair,uint64_t > _lastRequestTime; + Mutex _lastRequestTime_m; +}; + +} // namespace ZeroTier + +#endif diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp deleted file mode 100644 index 863f93f3..00000000 --- a/controller/SqliteNetworkController.cpp +++ /dev/null @@ -1,1462 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "../include/ZeroTierOne.h" -#include "../node/Constants.hpp" - -#include "SqliteNetworkController.hpp" - -#include "../node/Node.hpp" -#include "../node/Utils.hpp" -#include "../node/CertificateOfMembership.hpp" -#include "../node/NetworkConfig.hpp" -#include "../node/Dictionary.hpp" -#include "../node/InetAddress.hpp" -#include "../node/MAC.hpp" -#include "../node/Address.hpp" - -using json = nlohmann::json; - -// API version reported via JSON control plane -#define ZT_NETCONF_CONTROLLER_API_VERSION 3 - -// Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 8 - -// Min duration between requests for an address/nwid combo to prevent floods -#define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 - -// Nodes are considered active if they've queried in less than this long -#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) - -namespace ZeroTier { - -static json _renderRule(ZT_VirtualNetworkRule &rule) -{ - char tmp[128]; - json r = json::object(); - r["not"] = ((rule.t & 0x80) != 0); - switch((rule.t) & 0x7f) { - case ZT_NETWORK_RULE_ACTION_DROP: - r["type"] = "ACTION_DROP"; - break; - case ZT_NETWORK_RULE_ACTION_ACCEPT: - r["type"] = "ACTION_ACCEPT"; - break; - case ZT_NETWORK_RULE_ACTION_TEE: - r["type"] = "ACTION_TEE"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_ACTION_REDIRECT: - r["type"] = "ACTION_REDIRECT"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: - r["type"] = "MATCH_VLAN_ID"; - r["vlanId"] = (uint64_t)rule.v.vlanId; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - r["type"] = "MATCH_VLAN_PCP"; - r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - r["type"] = "MATCH_VLAN_DEI"; - r["vlanDei"] = (uint64_t)rule.v.vlanDei; - break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - r["type"] = "MATCH_ETHERTYPE"; - r["etherType"] = (uint64_t)rule.v.etherType; - break; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - r["type"] = "MATCH_MAC_SOURCE"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_MAC_DEST: - r["type"] = "MATCH_MAC_DEST"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - r["type"] = "MATCH_IPV4_SOURCE"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - r["type"] = "MATCH_IPV4_DEST"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - r["type"] = "MATCH_IPV6_SOURCE"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - r["type"] = "MATCH_IPV6_DEST"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IP_TOS: - r["type"] = "MATCH_IP_TOS"; - r["ipTos"] = (uint64_t)rule.v.ipTos; - break; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - r["type"] = "MATCH_IP_PROTOCOL"; - r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; - break; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: - r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; - r["start"] = (uint64_t)rule.v.port[0]; - r["end"] = (uint64_t)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - r["type"] = "MATCH_IP_DEST_PORT_RANGE"; - r["start"] = (uint64_t)rule.v.port[0]; - r["end"] = (uint64_t)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - r["type"] = "MATCH_CHARACTERISTICS"; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); - r["mask"] = tmp; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); - r["value"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - r["type"] = "MATCH_FRAME_SIZE_RANGE"; - r["start"] = (uint64_t)rule.v.frameSize[0]; - r["end"] = (uint64_t)rule.v.frameSize[1]; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: - r["type"] = "MATCH_TAGS_SAMENESS"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: - r["type"] = "MATCH_TAGS_BITWISE_AND"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: - r["type"] = "MATCH_TAGS_BITWISE_OR"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: - r["type"] = "MATCH_TAGS_BITWISE_XOR"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - } - return r; -} - -static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) -{ - if (r.is_object()) - return false; - std::string t = r["type"]; - memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); - if (r.value("not",false)) - rule.t = 0x80; - else rule.t = 0x00; - if (t == "ACTION_DROP") { - rule.t |= ZT_NETWORK_RULE_ACTION_DROP; - return true; - } else if (t == "ACTION_ACCEPT") { - rule.t |= ZT_NETWORK_RULE_ACTION_ACCEPT; - return true; - } else if (t == "ACTION_TEE") { - rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "ACTION_REDIRECT") { - rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "MATCH_VLAN_ID") { - rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; - rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_VLAN_PCP") { - rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; - rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_VLAN_DEI") { - rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; - rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_ETHERTYPE") { - rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; - rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_MAC_SOURCE") { - rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; - const std::string mac(r.value("mac","0")); - Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); - return true; - } else if (t == "MATCH_MAC_DEST") { - rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; - const std::string mac(r.value("mac","0")); - Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); - return true; - } else if (t == "MATCH_IPV4_SOURCE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; - InetAddress ip(r.value("ip","0.0.0.0")); - rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; - rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; - if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; - return true; - } else if (t == "MATCH_IPV4_DEST") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; - InetAddress ip(r.value("ip","0.0.0.0")); - rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; - rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; - if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; - return true; - } else if (t == "MATCH_IPV6_SOURCE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; - InetAddress ip(r.value("ip","::0")); - memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); - rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; - if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; - return true; - } else if (t == "MATCH_IPV6_DEST") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; - InetAddress ip(r.value("ip","::0")); - memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); - rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; - if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; - return true; - } else if (t == "MATCH_IP_TOS") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; - rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_IP_PROTOCOL") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; - rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_IP_DEST_PORT_RANGE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_CHARACTERISTICS") { - rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; - if (r.count("mask")) { - auto v = r["mask"]; - if (v.is_number()) { - rule.v.characteristics[0] = v; - } else { - std::string tmp = v; - rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); - } - } - if (r.count("value")) { - auto v = r["value"]; - if (v.is_number()) { - rule.v.characteristics[1] = v; - } else { - std::string tmp = v; - rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); - } - } - return true; - } else if (t == "MATCH_FRAME_SIZE_RANGE") { - rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0").c_str()) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0").c_str()) & 0xffffULL); - return true; - } else if (t == "MATCH_TAGS_SAMENESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } else if (t == "MATCH_TAGS_BITWISE_AND") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } else if (t == "MATCH_TAGS_BITWISE_OR") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } else if (t == "MATCH_TAGS_BITWISE_XOR") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } - return false; -} - -SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath) : - _node(node), - _path(dbPath) -{ - OSUtils::mkdir(dbPath); - /* - if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) - throw std::runtime_error("SqliteNetworkController cannot open database file"); - sqlite3_busy_timeout(_db,10000); - - sqlite3_exec(_db,"PRAGMA synchronous = OFF",0,0,0); - sqlite3_exec(_db,"PRAGMA journal_mode = MEMORY",0,0,0); - - sqlite3_stmt *s = (sqlite3_stmt *)0; - if ((sqlite3_prepare_v2(_db,"SELECT v FROM Config WHERE k = 'schemaVersion';",-1,&s,(const char **)0) == SQLITE_OK)&&(s)) { - int schemaVersion = -1234; - if (sqlite3_step(s) == SQLITE_ROW) { - schemaVersion = sqlite3_column_int(s,0); - } - - sqlite3_finalize(s); - - if (schemaVersion == -1234) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController schemaVersion not found in Config table (init failure?)"); - } - - if (schemaVersion < 2) { - // Create NodeHistory table to upgrade from version 1 to version 2 - if (sqlite3_exec(_db, - "CREATE TABLE NodeHistory (\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " networkVisitCounter INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestAuthorized INTEGER NOT NULL DEFAULT(0),\n" - " requestTime INTEGER NOT NULL DEFAULT(0),\n" - " clientMajorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientMinorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientRevision INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestMetaData VARCHAR(1024),\n" - " fromAddress VARCHAR(128)\n" - ");\n" - "CREATE INDEX NodeHistory_nodeId ON NodeHistory (nodeId);\n" - "CREATE INDEX NodeHistory_networkId ON NodeHistory (networkId);\n" - "CREATE INDEX NodeHistory_requestTime ON NodeHistory (requestTime);\n" - "UPDATE \"Config\" SET \"v\" = 2 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 2: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 2; - } - } - - if (schemaVersion < 3) { - // Create Route table to upgrade from version 2 to version 3 and migrate old - // data. Also delete obsolete Gateway table that was never actually used, and - // migrate Network flags to a bitwise flags field instead of ASCII cruft. - if (sqlite3_exec(_db, - "DROP TABLE Gateway;\n" - "CREATE TABLE Route (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " target blob(16) NOT NULL,\n" - " via blob(16),\n" - " targetNetmaskBits integer NOT NULL,\n" - " ipVersion integer NOT NULL,\n" - " flags integer NOT NULL,\n" - " metric integer NOT NULL\n" - ");\n" - "CREATE INDEX Route_networkId ON Route (networkId);\n" - "INSERT INTO Route SELECT DISTINCT networkId,\"ip\" AS \"target\",NULL AS \"via\",ipNetmaskBits AS targetNetmaskBits,ipVersion,0 AS \"flags\",0 AS \"metric\" FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "ALTER TABLE Network ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 1) WHERE v4AssignMode = 'zt';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 2) WHERE v6AssignMode = 'rfc4193';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 4) WHERE v6AssignMode = '6plane';\n" - "ALTER TABLE Member ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "DELETE FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "UPDATE \"Config\" SET \"v\" = 3 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 3; - } - } - - if (schemaVersion < 4) { - // Turns out this was overkill and a huge performance drag. Will be revisiting this - // more later but for now a brief snapshot of recent history stored in Member is fine. - // Also prepare for implementation of proof of work requests. - if (sqlite3_exec(_db, - "DROP TABLE NodeHistory;\n" - "ALTER TABLE Member ADD COLUMN lastRequestTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowDifficulty integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN recentHistory blob;\n" - "CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n" - "UPDATE \"Config\" SET \"v\" = 4 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 4; - } - } - - if (schemaVersion < 5) { - // Upgrade old rough draft Rule table to new release format - if (sqlite3_exec(_db, - "DROP TABLE Relay;\n" - "DROP INDEX Rule_networkId_ruleNo;\n" - "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" - "CREATE TABLE Rule (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " capId integer,\n" - " ruleNo integer NOT NULL,\n" - " ruleType integer NOT NULL DEFAULT(0),\n" - " \"addr\" blob(16),\n" - " \"int1\" integer,\n" - " \"int2\" integer,\n" - " \"int3\" integer,\n" - " \"int4\" integer\n" - ");\n" - "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" - "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" - "DROP TABLE RuleOld;\n" - "CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n" - "CREATE TABLE MemberTC (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " tagId integer,\n" - " tagValue integer,\n" - " capId integer,\n" - " capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n" - ");\n" - "CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n" - "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 5; - } - } - - if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController database schema version mismatch"); - } - } else { - // Prepare statement will fail if Config table doesn't exist, which means our DB - // needs to be initialized. - if (sqlite3_exec(_db,ZT_NETCONF_SCHEMA_SQL"INSERT INTO Config (k,v) VALUES ('schemaVersion',"ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR");",0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot initialize database and/or insert schemaVersion into Config table: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - } - - if ( - - (sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Network (id,name,creationTime,revision) VALUES (?,?,?,1)",-1,&_sCreateNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Network WHERE id = ?",-1,&_sDeleteNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId FROM Member WHERE networkId = ? AND activeBridge > 0 AND authorized > 0",-1,&_sGetActiveBridges,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.nodeId,m.memberRevision FROM Member AS m WHERE m.networkId = ? ORDER BY m.nodeId ASC",-1,&_sListNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET authorized = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberAuthorized,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET activeBridge = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberActiveBridge,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET \"lastRequestTime\" = ?, \"recentHistory\" = ? WHERE rowid = ?",-1,&_sUpdateMemberHistory,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sDeleteMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(const char **)0) != SQLITE_OK) - - ) { - std::string err(std::string("SqliteNetworkController unable to initialize one or more prepared statements: ") + sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - - sqlite3_reset(_sGetConfig); - sqlite3_bind_text(_sGetConfig,1,"instanceId",10,SQLITE_STATIC); - if (sqlite3_step(_sGetConfig) != SQLITE_ROW) { - unsigned char sr[32]; - Utils::getSecureRandom(sr,32); - for(unsigned int i=0;i<32;++i) - _instanceId.push_back("0123456789abcdef"[(unsigned int)sr[i] & 0xf]); - - sqlite3_reset(_sSetConfig); - sqlite3_bind_text(_sSetConfig,1,"instanceId",10,SQLITE_STATIC); - sqlite3_bind_text(_sSetConfig,2,_instanceId.c_str(),-1,SQLITE_STATIC); - if (sqlite3_step(_sSetConfig) != SQLITE_DONE) - throw std::runtime_error("SqliteNetworkController unable to read or initialize instanceId"); - } else { - const char *iid = reinterpret_cast(sqlite3_column_text(_sGetConfig,0)); - if (!iid) - throw std::runtime_error("SqliteNetworkController unable to read instanceId (it's NULL)"); - _instanceId = iid; - } - -#ifdef ZT_NETCONF_SQLITE_TRACE - sqlite3_trace(_db,sqliteTraceFunc,(void *)0); -#endif - - _backupThread = Thread::start(this); - */ -} - -SqliteNetworkController::~SqliteNetworkController() -{ -} - -NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) -{ - if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { - return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } - - const uint64_t now = OSUtils::now(); - - // Check rate limit circuit breaker to prevent flooding - { - Mutex::Lock _l(_lastRequestTime_m); - uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; - if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) - return NetworkController::NETCONF_QUERY_IGNORE; - lrt = now; - } - - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; - const std::string memberJP(_memberJP(nwid,identity.address(),false)); - json member(_readJson(memberJP)); - - { - std::string haveIdStr = member.value("identity",""); - if (haveIdStr.length() > 0) { - try { - if (Identity(haveIdStr.c_str()) != identity) - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } catch ( ... ) { - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } - } else { - member["identity"] = identity.toString(false); - } - } - - // Make sure these are always present no matter what, and increment member revision since we will always at least log something - member["id"] = identity.address().toString(); - member["address"] = member["id"]; - member["nwid"] = network["id"]; - member["memberRevision"] = member.value("memberRevision",0ULL) + 1; - - // Update member log - { - json rlEntry = json::object(); - rlEntry["ts"] = now; - rlEntry["authorized"] = member["authorized"]; - rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - if (fromAddr) - rlEntry["fromAddr"] = fromAddr.toString(); - json recentLog = json::array(); - recentLog.push_back(rlEntry); - auto oldLog = member["recentLog"]; - if (oldLog.is_array()) { - for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) - break; - } - } - member["recentLog"] = recentLog; - } - - // Stop if network is private and member is not authorized - if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } - // Else compose and send network config - - nc.networkId = nwid; - nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; - nc.timestamp = now; - nc.revision = network.value("revision",0ULL); - nc.issuedTo = identity.address(); - if (network.value("enableBroadcast",true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (network.value("allowPassiveBridging",false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),network.value("name","").c_str()); - nc.multicastLimit = (unsigned int)network.value("multicastLimit",32ULL); - - bool amActiveBridge = false; - { - json ab = network["activeBridges"]; - if (ab.is_array()) { - for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) - break; - auto rule = rules[i]; - if (_parseRule(rule,nc.rules[nc.ruleCount])) - ++nc.ruleCount; - } - } - - if (routes.is_array()) { - for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) - break; - auto route = routes[i]; - InetAddress t(route.value("target","")); - InetAddress v(route.value("via","")); - if ((t)&&(v)&&(t.ss_family == v.ss_family)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); - *(reinterpret_cast(&(r->target))) = t; - *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; - } - } - } - - bool haveManagedIpv4AutoAssignment = false; - bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count - json ipAssignments = member["ipAssignments"]; - if (ipAssignments.is_array()) { - for(unsigned long i=0;i(&(nc.routes[rk].target))->containsAddress(ip)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); - } - - if (routedNetmaskBits > 0) { - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - ip.setPort(routedNetmaskBits); - nc.staticIps[nc.staticIpCount++] = ip; - } - if (ip.ss_family == AF_INET) - haveManagedIpv4AutoAssignment = true; - else if (ip.ss_family == AF_INET6) - haveManagedIpv6AutoAssignment = true; - } - } - } else { - ipAssignments = json::array(); - } - - std::set allocatedIps; - bool allocatedIpsLoaded = false; - - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(v6AssignMode.value("zt",false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); - for(unsigned long p=0;((p s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { - // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. - xx[0] = Utils::hton(x[0]); - xx[1] = Utils::hton(x[1] + identity.address().toInt()); - } else { - // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway - Utils::getSecureRandom((void *)xx,16); - if ((e[0] > s[0])) - xx[0] %= (e[0] - s[0]); - else xx[0] = 0; - if ((e[1] > s[1])) - xx[1] %= (e[1] - s[1]); - else xx[1] = 0; - xx[0] = Utils::hton(x[0] + xx[0]); - xx[1] = Utils::hton(x[1] + xx[1]); - } - - InetAddress ip6((const void *)xx,16,0); - - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); - } - - // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip6))) { - ipAssignments.push_back(ip6.toIpString()); - member["ipAssignments"] = ipAssignments; - ip6.setPort((unsigned int)routedNetmaskBits); - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) - nc.staticIps[nc.staticIpCount++] = ip6; - haveManagedIpv4AutoAssignment = true; - break; - } - } - } - } - } - } - - if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(v4AssignMode.value("zt",false))) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); - for(unsigned long p=0;((p(&ipRangeStart)->sin_addr.s_addr)); - uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); - if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) - continue; - uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; - - // Start with the LSB of the member's address - uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); - - for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { - uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; - ++ipTrialCounter; - if ((ip & 0x000000ff) == 0x000000ff) - continue; // don't allow addresses that end in .255 - - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); - if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { - routedNetmaskBits = targetBits; - break; - } - } - } - - InetAddress ip4(Utils::hton(ip),0); - - // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip4))) { - ipAssignments.push_back(ip4.toIpString()); - member["ipAssignments"] = ipAssignments; - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); - v4ip->sin_family = AF_INET; - v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); - v4ip->sin_addr.s_addr = Utils::hton(ip); - } - haveManagedIpv4AutoAssignment = true; - break; - } - } - } - } - } - } - - if (network.value("private",true)) { - CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); - if (com.sign(signingId)) { - nc.com = com; - } else { - return NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } - } - - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_OK; -} - -unsigned int SqliteNetworkController::handleControlPlaneHttpGET( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if ((path.size() > 0)&&(path[0] == "network")) { - - if ((path.size() >= 2)&&(path[1].length() == 16)) { - const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return 404; - - if (path.size() >= 3) { - - if (path[2] == "member") { - - if (path.size() >= 4) { - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - json member(_readJson(_memberJP(nwid,Address(address),false))); - if (!member.size()) - return 404; - - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - - member["clock"] = OSUtils::now(); - responseBody = member.dump(2); - responseContentType = "application/json"; - - return 200; - } else { - - responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - const std::string rc = member.value("memberRevision","0"); - responseBody.append(rc); - } - } - } - responseBody.push_back('}'); - responseContentType = "application/json"; - - return 200; - } - - } else if ((path[2] == "active")&&(path.size() == 3)) { - - responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - auto recentLog = member["recentLog"]; - if ((recentLog.is_array())&&(recentLog.size() > 0)) { - auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - responseBody.append(mostRecentLog.dump()); - } - } - } - } - } - responseBody.push_back('}'); - responseContentType = "application/json"; - return 200; - - } else if ((path[2] == "test")&&(path.size() >= 4)) { - - Mutex::Lock _l(_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); - if ((cte != _circuitTests.end())&&(cte->second.test)) { - - responseBody = "["; - responseBody.append(cte->second.jsonResults); - responseBody.push_back(']'); - responseContentType = "application/json"; - - return 200; - - } // else 404 - - } // else 404 - - } else { - - nlohmann::json o(network); - o["clock"] = OSUtils::now(); - responseBody = o.dump(2); - responseContentType = "application/json"; - return 200; - - } - } else if (path.size() == 1) { - - responseBody = "["; - std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); - for(auto i(networks.begin());i!=networks.end();++i) { - if (i->length() == 16) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\""); - } - } - responseBody.push_back(']'); - responseContentType = "application/json"; - return 200; - - } // else 404 - - } else { - - char tmp[4096]; - Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); - responseBody = tmp; - responseContentType = "application/json"; - return 200; - - } - - return 404; -} - -unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if (path.empty()) - return 404; - - json b; - try { - b = json::parse(body); - if (!b.is_object()) - return 400; - } catch ( ... ) { - return 400; - } - - if (path[0] == "network") { - - if ((path.size() >= 2)&&(path[1].length() == 16)) { - uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - if (path.size() >= 3) { - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return 404; - - if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - uint64_t address = Utils::hexStrToU64(path[3].c_str()); - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); - - json member(_readJson(_memberJP(nwid,Address(address),true))); - - try { - if (b.count("authorized")) member["authorized"] = b.value("authorized",false); - if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known - - if (b.count("ipAssignments")) { - auto ipa = b["ipAssignments"]; - if (ipa.is_array()) { - json mipa(json::array()); - for(unsigned long i=0;itestId),sizeof(test->testId)); - test->credentialNetworkId = nwid; - test->ptr = (void *)this; - json hops = b["hops"]; - if (hops.is_array()) { - for(unsigned long i=0;ihops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - } - } else if (hops2.is_string()) { - std::string s = hops2; - test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - } - } - } - test->reportAtEveryHop = (b.value("reportAtEveryHop",true) ? 1 : 0); - - if (!test->hopCount) { - ::free((void *)test); - return 400; - } - - test->timestamp = OSUtils::now(); - - _CircuitTestEntry &te = _circuitTests[test->testId]; - te.test = test; - te.jsonResults = ""; - - _node->circuitTestBegin(test,&(SqliteNetworkController::_circuitTestCallback)); - - char json[1024]; - Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); - responseBody = json; - responseContentType = "application/json"; - return 200; - - } // else 404 - - } else { - // POST to network ID - - // Magic ID ending with ______ picks a random unused network ID - if (path[1].substr(10) == "______") { - nwid = 0; - uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; - uint64_t nwidPostfix = 0; - for(unsigned long k=0;k<100000;++k) { // sanity limit on trials - Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); - uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); - if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) { - nwid = tryNwid; - break; - } - } - if (!nwid) - return 503; - } - - json network(_readJson(_networkJP(nwid,true))); - - try { - if (b.count("name")) network["name"] = b.value("name",""); - if (b.count("private")) network["private"] = b.value("private",true); - if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); - if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); - if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); - - if (b.count("activeBridges")) { - auto ab = b["activeBridges"]; - if (ab.is_array()) { - json ab2 = json::array(); - for(unsigned long i=0;i v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); - std::sort(v6m.begin(),v6m.end()); - v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); - for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { - if (*i == "rfc4193") - nv6m["rfc4193"] = true; - else if (*i == "zt") - nv6m["zt"] = true; - else if (*i == "6plane") - nv6m["6plane"] = true; - } - } else if (b["v6AssignMode"].is_object()) { - auto v6m = b["v6AssignMode"]; - if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); - if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); - if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); - } - if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; - if (!nv6m.count("zt")) nv6m["zt"] = false; - if (!nv6m.count("6plane")) nv6m["6plane"] = false; - } - - if (b.count("routes")) { - auto rts = b["routes"]; - if (rts.is_array()) { - for(unsigned long i=0;i &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if (path.empty()) - return 404; - - if (path[0] == "network") { - if ((path.size() >= 2)&&(path[1].length() == 16)) { - const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return 404; - - if (path.size() >= 3) { - if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - json member(_readJson(_memberJP(nwid,Address(address),false))); - if (!member.size()) - return 404; - - OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str()); - - responseBody = member.dump(2); - responseContentType = "application/json"; - return 200; - } - } else { - OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); - responseBody = network.dump(2); - responseContentType = "application/json"; - return 200; - } - } // else 404 - - } // else 404 - - return 404; -} - -void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) -{ - char tmp[65535]; - SqliteNetworkController *const self = reinterpret_cast(test->ptr); - - if (!test) - return; - if (!report) - return; - - Mutex::Lock _l(self->_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); - - if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? - self->_node->circuitTestEnd(test); - ::free((void *)test); - return; - } - - Utils::snprintf(tmp,sizeof(tmp), - "%s{\n" - "\t\"timestamp\": %llu," ZT_EOL_S - "\t\"testId\": \"%.16llx\"," ZT_EOL_S - "\t\"upstream\": \"%.10llx\"," ZT_EOL_S - "\t\"current\": \"%.10llx\"," ZT_EOL_S - "\t\"receivedTimestamp\": %llu," ZT_EOL_S - "\t\"remoteTimestamp\": %llu," ZT_EOL_S - "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S - "\t\"flags\": %llu," ZT_EOL_S - "\t\"sourcePacketHopCount\": %u," ZT_EOL_S - "\t\"errorCode\": %u," ZT_EOL_S - "\t\"vendor\": %d," ZT_EOL_S - "\t\"protocolVersion\": %u," ZT_EOL_S - "\t\"majorVersion\": %u," ZT_EOL_S - "\t\"minorVersion\": %u," ZT_EOL_S - "\t\"revision\": %u," ZT_EOL_S - "\t\"platform\": %d," ZT_EOL_S - "\t\"architecture\": %d," ZT_EOL_S - "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S - "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S - "}", - ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), - (unsigned long long)report->timestamp, - (unsigned long long)test->testId, - (unsigned long long)report->upstream, - (unsigned long long)report->current, - (unsigned long long)OSUtils::now(), - (unsigned long long)report->remoteTimestamp, - (unsigned long long)report->sourcePacketId, - (unsigned long long)report->flags, - report->sourcePacketHopCount, - report->errorCode, - (int)report->vendor, - report->protocolVersion, - report->majorVersion, - report->minorVersion, - report->revision, - (int)report->platform, - (int)report->architecture, - reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), - reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); - - cte->second.jsonResults.append(tmp); -} - -} // namespace ZeroTier diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp deleted file mode 100644 index 288ea23a..00000000 --- a/controller/SqliteNetworkController.hpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef ZT_SQLITENETWORKCONTROLLER_HPP -#define ZT_SQLITENETWORKCONTROLLER_HPP - -#include - -#include -#include -#include -#include - -#include "../node/Constants.hpp" - -#include "../node/NetworkController.hpp" -#include "../node/Mutex.hpp" -#include "../node/Utils.hpp" -#include "../node/InetAddress.hpp" - -#include "../osdep/OSUtils.hpp" - -#include "../ext/json/json.hpp" - -namespace ZeroTier { - -class Node; - -class SqliteNetworkController : public NetworkController -{ -public: - SqliteNetworkController(Node *node,const char *dbPath); - virtual ~SqliteNetworkController(); - - virtual NetworkController::ResultCode doNetworkConfigRequest( - const InetAddress &fromAddr, - const Identity &signingId, - const Identity &identity, - uint64_t nwid, - const Dictionary &metaData, - NetworkConfig &nc); - - unsigned int handleControlPlaneHttpGET( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType); - unsigned int handleControlPlaneHttpPOST( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType); - unsigned int handleControlPlaneHttpDELETE( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType); - -private: - static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); - - inline nlohmann::json _readJson(const std::string &path) - { - std::string buf; - if (OSUtils::readFile(path.c_str(),buf)) { - try { - return nlohmann::json::parse(buf); - } catch ( ... ) {} - } - return nlohmann::json::object(); - } - - inline bool _writeJson(const std::string &path,const nlohmann::json &obj) - { - std::string buf(obj.dump(2)); - return OSUtils::writeFile(path.c_str(),buf); - } - - inline std::string _networkBP(const uint64_t nwid,bool create) - { - char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nwid); - std::string p(_path + ZT_PATH_SEPARATOR_S + "network"); - if (create) OSUtils::mkdir(p.c_str()); - p.push_back(ZT_PATH_SEPARATOR); - p.append(tmp); - if (create) OSUtils::mkdir(p.c_str()); - return p; - } - inline std::string _networkJP(const uint64_t nwid,bool create) - { - return (_networkBP(nwid,create) + ZT_PATH_SEPARATOR + "config.json"); - } - inline std::string _memberBP(const uint64_t nwid,const Address &member,bool create) - { - std::string p(_networkBP(nwid,create)); - p.push_back(ZT_PATH_SEPARATOR); - p.append("member"); - if (create) OSUtils::mkdir(p.c_str()); - p.push_back(ZT_PATH_SEPARATOR); - p.append(member.toString()); - if (create) OSUtils::mkdir(p.c_str()); - return p; - } - inline std::string _memberJP(const uint64_t nwid,const Address &member,bool create) - { - return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); - } - - inline std::set _getAlreadyAllocatedIps(const uint64_t nwid) - { - std::set ips; - std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); - std::vector members(OSUtils::listSubdirectories(bp.c_str())); - for(std::vector::iterator m(members.begin());m!=members.end();++m) { - if (m->length() == ZT_ADDRESS_LENGTH_HEX) { - nlohmann::json mj = _readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"); - auto ipAssignments = mj["ipAssignments"]; - if (ipAssignments.is_array()) { - for(unsigned long i=0;i _circuitTests; - Mutex _circuitTests_m; - - // Last request time by address, for rate limitation - std::map< std::pair,uint64_t > _lastRequestTime; - Mutex _lastRequestTime_m; -}; - -} // namespace ZeroTier - -#endif diff --git a/make-linux.mk b/make-linux.mk index acc22a63..d0d206a1 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -82,12 +82,6 @@ ifeq ($(ZT_USE_MINIUPNPC),1) endif endif -ifeq ($(ZT_ENABLE_NETWORK_CONTROLLER),1) - DEFS+=-DZT_ENABLE_NETWORK_CONTROLLER - LDLIBS+=-L/usr/local/lib -lsqlite3 - OBJS+=controller/SqliteNetworkController.o -endif - ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif diff --git a/make-mac.mk b/make-mac.mk index 63ffc3c8..f00f8d6b 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -45,12 +45,6 @@ ifeq ($(ZT_USE_MINIUPNPC),1) OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o endif -ifeq ($(ZT_ENABLE_NETWORK_CONTROLLER),1) - DEFS+=-DZT_ENABLE_NETWORK_CONTROLLER - LIBS+=-L/usr/local/lib -lsqlite3 - OBJS+=controller/SqliteNetworkController.o -endif - # Debug mode -- dump trace output, build binary with -g ifeq ($(ZT_DEBUG),1) DEFS+=-DZT_TRACE diff --git a/objects.mk b/objects.mk index 11b03c81..f92a907e 100644 --- a/objects.mk +++ b/objects.mk @@ -1,4 +1,5 @@ OBJS=\ + controller/EmbeddedNetworkController.o \ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ diff --git a/selftest.cpp b/selftest.cpp index f4232851..ab05c9ef 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -53,10 +53,6 @@ #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" -#ifdef ZT_ENABLE_NETWORK_CONTROLLER -#include "controller/SqliteNetworkController.hpp" -#endif // ZT_ENABLE_NETWORK_CONTROLLER - #ifdef __WINDOWS__ #include #endif diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index a10697a9..7aa757a9 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -34,9 +34,7 @@ #include "../ext/json-parser/json.h" #endif -#ifdef ZT_ENABLE_NETWORK_CONTROLLER -#include "../controller/SqliteNetworkController.hpp" -#endif +#include "../controller/EmbeddedNetworkController.hpp" #include "../node/InetAddress.hpp" #include "../node/Node.hpp" @@ -254,9 +252,7 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) ControlPlane::ControlPlane(OneService *svc,Node *n,const char *uiStaticPath) : _svc(svc), _node(n), -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - _controller((SqliteNetworkController *)0), -#endif + _controller((EmbeddedNetworkController *)0), _uiStaticPath((uiStaticPath) ? uiStaticPath : "") { } @@ -499,13 +495,9 @@ unsigned int ControlPlane::handleRequest( responseContentType = "text/plain"; scode = 200; } else { -#ifdef ZT_ENABLE_NETWORK_CONTROLLER if (_controller) scode = _controller->handleControlPlaneHttpGET(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); else scode = 404; -#else - scode = 404; -#endif } } else scode = 401; // isAuth == false @@ -559,13 +551,9 @@ unsigned int ControlPlane::handleRequest( } else scode = 500; } } else { -#ifdef ZT_ENABLE_NETWORK_CONTROLLER if (_controller) scode = _controller->handleControlPlaneHttpPOST(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); else scode = 404; -#else - scode = 404; -#endif } } else scode = 401; // isAuth == false @@ -594,13 +582,9 @@ unsigned int ControlPlane::handleRequest( _node->freeQueryResult((void *)nws); } else scode = 500; } else { -#ifdef ZT_ENABLE_NETWORK_CONTROLLER if (_controller) scode = _controller->handleControlPlaneHttpDELETE(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); else scode = 404; -#else - scode = 404; -#endif } } else { diff --git a/service/ControlPlane.hpp b/service/ControlPlane.hpp index 08a9d6e0..ec9b94d7 100644 --- a/service/ControlPlane.hpp +++ b/service/ControlPlane.hpp @@ -31,7 +31,7 @@ namespace ZeroTier { class OneService; class Node; -class SqliteNetworkController; +class EmbeddedNetworkController; struct InetAddress; /** @@ -43,18 +43,16 @@ public: ControlPlane(OneService *svc,Node *n,const char *uiStaticPath); ~ControlPlane(); -#ifdef ZT_ENABLE_NETWORK_CONTROLLER /** * Set controller, which will be available under /controller * * @param c Network controller instance */ - inline void setController(SqliteNetworkController *c) + inline void setController(EmbeddedNetworkController *c) { Mutex::Lock _l(_lock); _controller = c; } -#endif /** * Add an authentication token for API access @@ -89,9 +87,7 @@ public: private: OneService *const _svc; Node *const _node; -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - SqliteNetworkController *_controller; -#endif + EmbeddedNetworkController *_controller; std::string _uiStaticPath; std::set _authTokens; Mutex _lock; diff --git a/service/OneService.cpp b/service/OneService.cpp index 5c65dcc2..74628168 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -69,11 +69,7 @@ */ //#define ZT_BREAK_UDP -#ifdef ZT_ENABLE_NETWORK_CONTROLLER -#include "../controller/SqliteNetworkController.hpp" -#else -class SqliteNetworkController; -#endif // ZT_ENABLE_NETWORK_CONTROLLER +#include "../controller/EmbeddedNetworkController.hpp" #ifdef __WINDOWS__ #include @@ -129,7 +125,7 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } #define ZT_TAP_CHECK_MULTICAST_INTERVAL 5000 // Path under ZT1 home for controller database if controller is enabled -#define ZT_CONTROLLER_DB_PATH "controller.db" +#define ZT_CONTROLLER_DB_PATH "controller.d" // TCP fallback relay host -- geo-distributed using Amazon Route53 geo-aware DNS #define ZT_TCP_FALLBACK_RELAY "tcp-fallback.zerotier.com" @@ -487,9 +483,7 @@ public: const std::string _homePath; BackgroundResolver _tcpFallbackResolver; -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - SqliteNetworkController *_controller; -#endif + EmbeddedNetworkController *_controller; Phy _phy; Node *_node; @@ -579,9 +573,7 @@ public: OneServiceImpl(const char *hp,unsigned int port) : _homePath((hp) ? hp : ".") ,_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY) -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - ,_controller((SqliteNetworkController *)0) -#endif + ,_controller((EmbeddedNetworkController *)0) ,_phy(this,false,true) ,_node((Node *)0) ,_controlPlane((ControlPlane *)0) @@ -673,9 +665,7 @@ public: #ifdef ZT_USE_MINIUPNPC delete _portMapper; #endif -#ifdef ZT_ENABLE_NETWORK_CONTROLLER delete _controller; -#endif #ifdef ZT_ENABLE_CLUSTER delete _clusterDefinition; #endif @@ -794,10 +784,8 @@ public: } } -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - _controller = new SqliteNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(_homePath + ZT_PATH_SEPARATOR_S + "circuitTestResults.d").c_str()); + _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str()); _node->setNetconfMaster((void *)_controller); -#endif #ifdef ZT_ENABLE_CLUSTER if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str())) { @@ -850,10 +838,7 @@ public: _controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S + "ui").c_str()); _controlPlane->addAuthToken(authToken.c_str()); - -#ifdef ZT_ENABLE_NETWORK_CONTROLLER _controlPlane->setController(_controller); -#endif { // Remember networks from previous session std::vector networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S + "networks.d").c_str())); diff --git a/service/OneService.hpp b/service/OneService.hpp index cead381d..72ff7d84 100644 --- a/service/OneService.hpp +++ b/service/OneService.hpp @@ -25,18 +25,6 @@ namespace ZeroTier { /** * Local service for ZeroTier One as system VPN/NFV provider - * - * If built with ZT_ENABLE_NETWORK_CONTROLLER defined, this includes and - * runs controller/SqliteNetworkController with a database called - * controller.db in the specified home directory. - * - * If built with ZT_AUTO_UPDATE, an official ZeroTier update URL is - * periodically checked and updates are automatically downloaded, verified - * against a built-in list of update signing keys, and installed. This is - * only supported for certain platforms. - * - * If built with ZT_ENABLE_CLUSTER, a 'cluster' file is checked and if - * present is read to determine the identity of other cluster members. */ class OneService { -- cgit v1.2.3 From 168b86fdcd8f2a590ea59710dfbfb67c8d8c5cef Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 17 Aug 2016 12:27:07 -0700 Subject: Controller docs and API fix. --- controller/EmbeddedNetworkController.cpp | 24 ++-- controller/README.md | 202 +++++++++++++------------------ 2 files changed, 99 insertions(+), 127 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 479a65f5..d3f44fb4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -280,12 +280,12 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_IP_DEST_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; @@ -310,28 +310,28 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_FRAME_SIZE_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0").c_str()) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0").c_str()) & 0xffffULL); + rule.v.frameSize[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; } else if (t == "MATCH_TAGS_SAMENESS") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); return true; } return false; diff --git a/controller/README.md b/controller/README.md index 8b789a3e..2c7541ae 100644 --- a/controller/README.md +++ b/controller/README.md @@ -1,66 +1,33 @@ Network Controller Microservice ====== -ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit ZeroTier address of a network controller followed by a 6-digit (24-bit) network number on that controller. Fans of software defined networking will recognize this as a variation of the familiar [separation of data plane and control plane](http://sdntutorials.com/difference-between-control-plane-and-data-plane/) SDN design pattern. +ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit ZeroTier address of a network controller followed by a 6-digit (24-bit) network number on that controller. Fans of software defined networking will recognize this as a spin on the familiar [separation of data plane and control plane](http://sdntutorials.com/difference-between-control-plane-and-data-plane/) SDN design pattern. -This code implements the *node/NetworkController.hpp* interface and provides a SQLite3-backed network controller microservice. Including it in the build allows ZeroTier One to act as a controller and create/manage networks. +This code implements the *node/NetworkController.hpp* interface to provide an embedded microservice configurable via the same local HTTP control plane as ZeroTier One iteself. It is built by default in ZeroTier One in desktop and server builds. This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. -This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. +Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, or edited in place, though we do not recommend doing the latter while the controller is running. Also take care if editing in place that you do not save corrupted JSON since the controller may then lose data when it attempts to load, modify, and save. -### Building +### Scalability and Reliability -On Linux, Mac, or BSD you can create a controller-enabled build with: +Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. - make ZT_ENABLE_NETWORK_CONTROLLER=1 +Since this implementation uses a JSON store in the filesystem we recommend running it on SSD-backed hosts. Slow disks will become a speed bottleneck under heavy load. For really huge and busy controllers you could consider linking `controller.d/` to a folder under `/dev/shm` (Linux RAM disk) and then setting up an out-of-band periodic snapshot cron job or background process to persist the data and a script to populate `/dev/shm` on boot before the controller starts. This is beyond the scope of this guide but is not particularly hard. -You will need the development headers and libraries for SQLite3 installed. - -### Running - -After building and installing (`make install`) a controller-enabled build of ZeroTier One, start it and try: - - sudo zerotier-cli /controller - -You should see something like: - - { - "controller": true, - "apiVersion": 2, - "clock": 1468002975497, - "instanceId": "8ab354604debe1da27ee627c9ef94a48" - } - -When started, a controller-enabled build of ZeroTier One will automatically create and initialize a `controller.db` file in its home folder. This is where all the controller's data and persistent state lives. If you're upgrading an old controller it will upgrade its database schema automatically on first launch. Make a backup of the old controller's database first since you can't go backward. - -Controllers periodically make backups of their database as `controller.db.backup`. This is done so that this file can be more easily copied/rsync'ed to other systems without worrying about corruption. SQLite3 supports multiple processes accessing the same database file, so `sqlite3 /path/to/controller.db .dump` also works but can be slow on a busy controller. - -Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend running multiple controllers for a lot of networks to spread load and be more fault tolerant. +Since ZeroTier nodes are mobile and do not need static IPs, implementing high availability fail-over for controllers is easy. Just replicate their working directories from master to backup and have something automatically fire up the backup if the master goes down. Many modern orchestration tools have built-in support for this. It would also be possible in theory to run controllers on a replicated or distributed filesystem, but we haven't tested this yet. ### Dockerizing Controllers ZeroTier network controllers can easily be run in Docker or other container systems. Since containers do not need to actually join networks, extra privilege options like "--device=/dev/net/tun --privileged" are not needed. You'll just need to map the local JSON API port of the running controller and allow it to access the Internet (over UDP/9993 at a minimum) so things can reach and query it. -### Implementing High Availability Fail-Over - -ZeroTier network controllers are not single points of failure for networks-- in the sense that if a controller goes down *existing* members of a network can continue to communicate. But new members (or those that have been offline for a while) can't join, existing members can't be de-authorized, and other changes to the network's configuration can't be made. This means that short "glitches" in controller availability are not a major problem but long periods of unavailability can be. - -Because controllers are just regular ZeroTier nodes and controller queries are in-band, controllers can trivially be moved without worrying about changes to underlying physical IPs. This makes high-availability fail-over very easy to implement. - -Just set up two cloud hosts, preferably in different data centers (e.g. two different AWS regions or Digital Ocean SF and NYC). Now set up the hot spare controller to constantly mirror `controller.db.backup` from its active sibling. - -If the active controller goes down, rename `controller.db.backup` to `controller.db` on the hot spare and start the ZeroTier One service there. The spare will take over and has now become the active controller. If the original active node comes back, it should take on the role of spare and should not start its service. Instead it should start mirroring the active controller's backup and wait until it is needed. - -The details of actually implementing this kind of HA fail-over on Linux or other OSes are beyond the scope of these docs and there are many ways to do it. Docker orchestration tools like Kubernetes can also be used to accomplish this if you've dockerized your controller. - ### Network Controller API The controller API is hosted via the same JSON API endpoint that ZeroTier One uses for local control (usually at 127.0.0.1 port 9993). All controller options are routed under the `/controller` base path. The controller microservice does not implement any fine-grained access control (authentication is via authtoken.secret just like the regular JSON API) or other complex mangement features. It just takes network and network member configurations and reponds to controller queries. We have an enterprise product called [ZeroTier Central](https://my.zerotier.com/) that we host as a service (and that companies can license to self-host) that does this. -All working network IDs on a controller must begin with the controller's ZeroTier address. The API will *allow* "foreign" networks to be added but the controller will have no way of doing anything with them. +All working network IDs on a controller must begin with the controller's ZeroTier address. The API will *allow* "foreign" networks to be added but the controller will have no way of doing anything with them since nobody will know to query it. (In the future we might support secondaries, which would make this relevant.) -Also note that the API is *very* sensitive about types. Integers must be integers and strings strings, etc. Incorrectly typed and unrecognized fields are just ignored. +The JSON API is *very* sensitive about types. Integers must be integers and strings strings, etc. Incorrectly typed and unrecognized fields may result in ignored fields or a 400 (bad request) error. #### `/controller` @@ -71,11 +38,8 @@ Also note that the API is *very* sensitive about types. Integers must be integer | Field | Type | Description | Writable | | ------------------ | ----------- | ------------------------------------------------- | -------- | | controller | boolean | Always 'true' | no | -| apiVersion | integer | Controller API version, currently 2 | no | +| apiVersion | integer | Controller API version, currently 3 | no | | clock | integer | Current clock on controller, ms since epoch | no | -| instanceId | string | A random ID generated on first controller DB init | no | - -The instance ID can be used to check whether a controller's database has been reset or otherwise switched. #### `/controller/network` @@ -97,52 +61,51 @@ When POSTing new networks take care that their IDs are not in use, otherwise you | Field | Type | Description | Writable | | --------------------- | ------------- | ------------------------------------------------- | -------- | -| nwid | string | 16-digit network ID | no | -| controllerInstanceId | string | Controller database instance ID | no | +| id | string | 16-digit network ID | no | +| nwid | string | 16-digit network ID (old, but still around) | no | | clock | integer | Current clock, ms since epoch | no | | name | string | A short name for this network | YES | | private | boolean | Is access control enabled? | YES | | enableBroadcast | boolean | Ethernet ff:ff:ff:ff:ff:ff allowed? | YES | +| activeBridges | array[string] | Array of ZeroTier addresses of active bridges | YES | | allowPassiveBridging | boolean | Allow any member to bridge (very experimental) | YES | -| v4AssignMode | string | If 'zt', auto-assign IPv4 from pool(s) | YES | -| v6AssignMode | string | IPv6 address auto-assign modes; see below | YES | +| v4AssignMode | object | IPv4 management and assign options (see below) | YES | +| v6AssignMode | object | IPv6 management and assign options (see below) | YES | | multicastLimit | integer | Maximum recipients for a multicast packet | YES | | creationTime | integer | Time network was first created | no | | revision | integer | Network config revision counter | no | | memberRevisionCounter | integer | Network member revision counter | no | | authorizedMemberCount | integer | Number of authorized members (for private nets) | no | -| relays | array[object] | Alternative relays; see below | YES | | routes | array[object] | Managed IPv4 and IPv6 routes; see below | YES | | ipAssignmentPools | array[object] | IP auto-assign ranges; see below | YES | | rules | array[object] | Traffic rules; see below | YES | -(The `ipLocalRoutes` field appeared in older versions but is no longer present. Routes will now show up in `routes`.) +Recent changes: -Two important things to know about networks: + * The `ipLocalRoutes` field appeared in older versions but is no longer present. Routes will now show up in `routes`. + * The `relays` field is gone since network preferred relays are gone. This capability is replaced by VL1 level federation ("federated roots"). + * Active bridges are now set at the network level, not in individual member configs. - - Networks without rules won't carry any traffic. See below for an example with rules to permit IPv4 and IPv6. - - Managed IP address assignments and IP assignment pools that do not fall within a route configured in `routes` are ignored and won't be used or sent to members. - - The default for `private` is `true` and this is probably what you want. Turning `private` off means *anyone* can join your network with only its 16-digit network ID. It's also impossible to de-authorize a member as these networks don't issue or enforce certificates. Such "party line" networks are used for decentralized app backplanes, gaming, and testing but are not common in ordinary use. +Other important points: -**IPv6 Auto-Assign Modes:** + * Networks without rules won't carry any traffic. If you don't specify any on network creation an "accept anything" rule set will automatically be added. + * Managed IP address assignments and IP assignment pools that do not fall within a route configured in `routes` are ignored and won't be used or sent to members. + * The default for `private` is `true` and this is probably what you want. Turning `private` off means *anyone* can join your network with only its 16-digit network ID. It's also impossible to de-authorize a member as these networks don't issue or enforce certificates. Such "party line" networks are used for decentralized app backplanes, gaming, and testing but are otherwise not common. -This field is (for legacy reasons) a comma-delimited list of strings. These can be `rfc4193`, `6plane`, and `zt`. RFC4193 and 6PLANE are special addressing modes that deterministically assign IPv6 addresses based on the network ID and the ZeroTier address of each member. The `zt` mode enables IPv6 auto-assignment from arbitrary IPv6 IP ranges configured in `ipAssignmentPools`. +**Auto-Assign Modes:** -**Relay object format:** +Auto assign modes (`v4AssignMode` and `v6AssignMode`) contain objects that map assignment modes to booleans. -Relay objects define network-specific preferred relay nodes. Traffic to peers on this network will preferentially use these relays if they are available, and otherwise will fall back to the global rootserver infrastructure. +For IPv4 the only valid setting is `zt` which, if true, causes IPv4 addresses to be auto-assigned from `ipAssignmentPools` to members that do not have an IPv4 assignment. Note that active bridges are exempt and will not get auto-assigned IPs since this can interfere with bridging. (You can still manually assign one if you want.) -| Field | Type | Description | Writable | -| --------------------- | ------------- | ------------------------------------------------- | -------- | -| address | string | 10-digit ZeroTier address of relay | YES | -| phyAddress | string | Optional IP/port suggestion for finding relay | YES | +IPv6 includes this option and two others: `6plane` and `rfc4193`. These assign private IPv6 addresses to each member based on a deterministic assignment scheme that allows members to emulate IPv6 NDP to skip multicast for better performance and scalability. The `rfc4193` mode gives every member a /128 on a /88 network, while `6plane` gives every member a /80 within a /40 network but uses NDP emulation to route *all* IPs under that /80 to its owner. The `6plane` mode is great for use cases like Docker since it allows every member to assign IPv6 addresses within its /80 that just work instantly and globally across the network. **IP assignment pool object format:** -| Field | Type | Description | Writable | -| --------------------- | ------------- | ------------------------------------------------- | -------- | -| ipRangeStart | string | Starting IP address in range | YES | -| ipRangeEnd | string | Ending IP address in range (inclusive) | YES | +| Field | Type | Description | +| --------------------- | ------------- | ------------------------------------------------- | +| ipRangeStart | string | Starting IP address in range | +| ipRangeEnd | string | Ending IP address in range (inclusive) | Pools are only used if auto-assignment is on for the given address type (IPv4 or IPv6) and if the entire range falls within a managed route. @@ -159,57 +122,65 @@ That defines a range within network `fd00:feed:feed:beef::/64` that contains up **Rule object format:** -Rules are matched in order of ruleNo. If no rules match, the default action is `drop`. To allow all traffic, create a single rule with all *null* fields and an action of `accept`. - -In the future there will be many, many more types of rules. As of today only filtering by Ethernet packet type is supported. +Each rule is actually a sequence of zero or more `MATCH_` entries in the rule array followed by an `ACTION_` entry that describes what to do if all the preceding entries match. An `ACTION_` without any preceding `MATCH_` entries is always taken, so setting a single `ACTION_ACCEPT` rule yields a network that allows all traffic. If no rules are present the default action is `ACTION_DROP`. -| Field | Type | Description | Writable | -| --------------------- | ------------- | ------------------------------------------------- | -------- | -| ruleNo | integer | Rule sorting key | YES | -| etherType | integer | Ethernet frame type (e.g. 34525 for IPv6) | YES | -| action | string | Currently either `allow` or `drop` | YES | +Rules are evaluated in the order in which they appear in the array. There is currently a limit of 256 entries per network. Capabilities should be used if a larger and more complex rule set is needed since they allow rules to be grouped by purpose and only shipped to members that need them. -**An Example: The Configuration for Earth** +Each rule table entry has two common fields. -Here is an example of a correctly configured ZeroTier network with IPv4 auto-assigned addresses from 28.0.0.0/7 (a "de-facto private" space) and RFC4193 IPv6 addressing. Users might recognize this as *Earth*, our public "global LAN party" that's used for demos and testing and occasionally gaming. +| Field | Type | Description | +| --------------------- | ------------- | ------------------------------------------------- | +| type | string | Entry type (all caps, case sensitive) | +| not | boolean | If true, MATCHes match if they don't match | -For your own networks you'll probably want to change `private` to `true` unless you like company. These rules on the other hand probably are what you want. These allow IPv4, IPv4 ARP, and IPv6 Ethernet frames. To allow only IPv4 omit the one for Ethernet type 34525 (IPv6). +The following fields may or may not be present depending on rule type: - { - "nwid": "8056c2e21c000001", - "controllerInstanceId": "8ab354604debe1da27ee627c9ef94a48", - "clock": 1468004857100, - "name": "earth.zerotier.net", - "private": false, - "enableBroadcast": false, - "allowPassiveBridging": false, - "v4AssignMode": "zt", - "v6AssignMode": "rfc4193", - "multicastLimit": 64, - "creationTime": 1442292573165, - "revision": 234, - "memberRevisionCounter": 3326, - "authorizedMemberCount": 2873, - "relays": [], - "routes": [ - {"target":"28.0.0.0/7","via":null,"flags":0,"metric":0}], - "ipAssignmentPools": [ - {"ipRangeStart":"28.0.0.1","ipRangeEnd":"29.255.255.254"}], - "rules": [ - { - "ruleNo": 20, - "etherType": 2048, - "action": "accept" - },{ - "ruleNo": 21, - "etherType": 2054, - "action": "accept" - },{ - "ruleNo": 30, - "etherType": 34525, - "action": "accept" - }] - } +| Field | Type | Description | +| --------------------- | ------------- | ------------------------------------------------- | +| zt | string | 10-digit hex ZeroTier address | +| etherType | integer | Ethernet frame type | +| mac | string | Hex MAC address (with or without :'s) | +| ip | string | IPv4 or IPv6 address | +| ipTos | integer | IP type of service | +| ipProtocol | integer | IP protocol (e.g. TCP) | +| start | integer | Start of an integer range (e.g. port range) | +| end | integer | End of an integer range (inclusive) | +| id | integer | Tag ID | +| value | integer | Tag value or comparison value | +| mask | integer | Bit mask (for characteristics flags) | + +The entry types and their additional fields are: + +| Entry type | Description | Fields | +| ------------------------------- | ----------------------------------------------------------------- | -------------- | +| `ACTION_DROP` | Drop any packets matching this rule | (none) | +| `ACTION_ACCEPT` | Accept any packets matching this rule | (none) | +| `ACTION_TEE` | Send a copy of this packet to a node (rule parsing continues) | `zt` | +| `ACTION_REDIRECT` | Redirect this packet to another node | `zt` | +| `MATCH_SOURCE_ZEROTIER_ADDRESS` | Match VL1 ZeroTier address of packet sender. | `zt` | +| `MATCH_DEST_ZEROTIER_ADDRESS` | Match VL1 ZeroTier address of recipient | `zt` | +| `MATCH_ETHERTYPE` | Match Ethernet frame type | `etherType` | +| `MATCH_MAC_SOURCE` | Match source Ethernet MAC address | `mac` | +| `MATCH_MAC_DEST` | Match destination Ethernet MAC address | `mac` | +| `MATCH_IPV4_SOURCE` | Match source IPv4 address | `ip` | +| `MATCH_IPV4_DEST` | Match destination IPv4 address | `ip` | +| `MATCH_IPV6_SOURCE` | Match source IPv6 address | `ip` | +| `MATCH_IPV6_DEST` | Match destination IPv6 address | `ip` | +| `MATCH_IP_TOS` | Match IP TOS field | `ipTos` | +| `MATCH_IP_PROTOCOL` | Match IP protocol field | `ipProtocol` | +| `MATCH_IP_SOURCE_PORT_RANGE` | Match a source IP port range | `start`,`end` | +| `MATCH_IP_DEST_PORT_RANGE` | Match a destination IP port range | `start`,`end` | +| `MATCH_CHARACTERISTICS` | Match on characteristics flags | `mask`,`value` | +| `MATCH_FRAME_SIZE_RANGE` | Match a range of Ethernet frame sizes | `start`,`end` | +| `MATCH_TAGS_SAMENESS` | Match if both sides' tags differ by no more than value | `id`,`value` | +| `MATCH_TAGS_BITWISE_AND` | Match if both sides' tags AND to value | `id`,`value` | +| `MATCH_TAGS_BITWISE_OR` | Match if both sides' tags OR to value | `id`,`value` | +| `MATCH_TAGS_BITWISE_XOR` | Match if both sides` tags XOR to value | `id`,`value` | + +Notes: + + * `MATCH_DEST_ZEROTIER_ADDRESS` does not work for multicast packets since these have many and varying destinations and can take circuitous routes. Use MAC rules instead. + * IPv4 and IPv6 rules do not match for frames that are not IPv4 or IPv6 respectively. #### `/controller/network//member` @@ -235,9 +206,10 @@ This returns an object containing all currently online members and the most rece | Field | Type | Description | Writable | | --------------------- | ------------- | ------------------------------------------------- | -------- | +| id | string | Member's 10-digit ZeroTier address | no | +| address | string | Member's 10-digit ZeroTier address | no | | nwid | string | 16-digit network ID | no | | clock | integer | Current clock, ms since epoch | no | -| address | string | Member's 10-digit ZeroTier address | no | | authorized | boolean | Is member authorized? (for private networks) | YES | | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | -- cgit v1.2.3 From b72847d50404f4d751184d9977e7bba23050a797 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 17 Aug 2016 13:41:45 -0700 Subject: Finally implement network join auth tokens, at least at the protocol level. --- controller/EmbeddedNetworkController.cpp | 53 ++++++++++++++++++++++++++++++-- node/NetworkConfig.hpp | 3 ++ 2 files changed, 54 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d3f44fb4..dfb93b01 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -658,11 +658,39 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // Stop if network is private and member is not authorized if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + bool authenticatedViaToken = false; + char atok[256]; + if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { + atok[255] = (char)0; // not necessary but YDIFLO + if (strlen(atok) > 0) { // extra sanity check + auto authTokens = network["authTokens"]; + if (authTokens.is_array()) { + for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { + authenticatedViaToken = true; + break; + } + } + } + } + } + } + + if (!authenticatedViaToken) { + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } } // Else compose and send network config + // If we made it here for some reason other than authorized being true, such as this + // being a public network or via a bearer token, then we set this in the member config. + member["authorized"] = true; + nc.networkId = nwid; nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; @@ -1308,6 +1336,26 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["rules"] = nrules; } } + + if (b.count("authTokens")) { + auto authTokens = b["authTokens"]; + if (authTokens.is_array()) { + json nat = json::array(); + for(unsigned long i=0;i 0) { + json t = json::object(); + t["token"] = tstr; + t["expires"] = token.value("expires",0ULL); + nat.push_back(t); + } + } + } + network["authTokens"] = nat; + } + } } catch ( ... ) { return 400; } @@ -1319,6 +1367,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!network.count("v4AssignMode")) network["v4AssignMode"] = "{\"zt\":false}"_json; if (!network.count("v6AssignMode")) network["v6AssignMode"] = "{\"rfc4193\":false,\"zt\":false,\"6plane\":false}"_json; if (!network.count("activeBridges")) network["activeBridges"] = json::array(); + if (!network.count("authTokens")) network["authTokens"] = json::array(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 18244ec9..25e0ccaf 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -101,6 +101,9 @@ namespace ZeroTier { // Maximum number of tags this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS "mt" +// Network join authorization token (if any) +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN "atok" + // These dictionary keys are short so they don't take up much room. // By convention we use upper case for binary blobs, but it doesn't really matter. -- cgit v1.2.3 From b7ebf6edbfc9ade141f54e5510fed65e77db33fb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 17 Aug 2016 13:54:32 -0700 Subject: Cleanup and log how member was authorized. --- controller/EmbeddedNetworkController.cpp | 78 ++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 35 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index dfb93b01..30072f95 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -50,7 +50,7 @@ using json = nlohmann::json; #define ZT_NETCONF_CONTROLLER_API_VERSION 3 // Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 8 +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 64 // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 @@ -632,17 +632,52 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["nwid"] = network["id"]; member["memberRevision"] = member.value("memberRevision",0ULL) + 1; - // Update member log + // Determine whether and how member is authorized + const char *authorizedBy = (const char *)0; + if (!network.value("private",true)) { + authorizedBy = "networkIsPublic"; + } else if (member.value("authorized",false)) { + authorizedBy = "memberIsAuthorized"; + } else { + char atok[256]; + if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { + atok[255] = (char)0; // not necessary but YDIFLO + if (strlen(atok) > 0) { // extra sanity check + auto authTokens = network["authTokens"]; + if (authTokens.is_array()) { + for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { + authorizedBy = "token"; + break; + } + } + } + } + } + } + } + + // Remember authorization state for public networks and token auth + if (authorizedBy) + member["authorized"] = true; + + // Log this request { json rlEntry = json::object(); rlEntry["ts"] = now; - rlEntry["authorized"] = member["authorized"]; + rlEntry["authorized"] = (authorizedBy) ? true : false; + rlEntry["authorizedBy"] = (authorizedBy) ? authorizedBy : ""; rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); if (fromAddr) rlEntry["fromAddr"] = fromAddr.toString(); + json recentLog = json::array(); recentLog.push_back(rlEntry); auto oldLog = member["recentLog"]; @@ -656,40 +691,13 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["recentLog"] = recentLog; } - // Stop if network is private and member is not authorized - if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { - bool authenticatedViaToken = false; - char atok[256]; - if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { - atok[255] = (char)0; // not necessary but YDIFLO - if (strlen(atok) > 0) { // extra sanity check - auto authTokens = network["authTokens"]; - if (authTokens.is_array()) { - for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { - authenticatedViaToken = true; - break; - } - } - } - } - } - } - - if (!authenticatedViaToken) { - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } + // If they are not authorized, STOP! + if (!authorizedBy) { + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } - // Else compose and send network config - // If we made it here for some reason other than authorized being true, such as this - // being a public network or via a bearer token, then we set this in the member config. - member["authorized"] = true; + // If we made it this far, they are authorized. nc.networkId = nwid; nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; -- cgit v1.2.3 From faa9a06bf5302b246805ead12690b38c3036d802 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 17 Aug 2016 17:37:37 -0700 Subject: Controller fixes... --- controller/EmbeddedNetworkController.cpp | 219 +++++++++++++++++++------------ controller/README.md | 2 +- node/IncomingPacket.cpp | 6 +- service/OneService.cpp | 4 + 4 files changed, 148 insertions(+), 83 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 30072f95..649ff094 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -60,6 +60,50 @@ using json = nlohmann::json; namespace ZeroTier { +static uint64_t _jI(const json &jv,const uint64_t dfl) +{ + if (jv.is_number()) { + return (uint64_t)jv; + } else if (jv.is_string()) { + std::string s = jv; + return Utils::strToU64(s.c_str()); + } else if (jv.is_boolean()) { + return ((bool)jv ? 1ULL : 0ULL); + } + return dfl; +} +static bool _jB(const json &jv,const bool dfl) +{ + if (jv.is_boolean()) { + return (bool)jv; + } else if (jv.is_number()) { + return ((uint64_t)jv > 0ULL); + } else if (jv.is_string()) { + std::string s = jv; + if (s.length() > 0) { + switch(s[0]) { + case 't': + case 'T': + case '1': + return true; + } + } + return false; + } + return dfl; +} +static std::string _jS(const json &jv,const char *dfl) +{ + if (jv.is_string()) { + return jv; + } else if (jv.is_number()) { + return jv; + } else if (jv.is_boolean()) { + return ((bool)jv ? std::string("1") : std::string("0")); + } + return std::string((dfl) ? dfl : ""); +} + static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; @@ -190,7 +234,7 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return false; std::string t = r["type"]; memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); - if (r.value("not",false)) + if (_jB(r["not"],false)) rule.t = 0x80; else rule.t = 0x00; if (t == "ACTION_DROP") { @@ -201,91 +245,91 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_VLAN_ID") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; - rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); + rule.v.vlanId = (uint16_t)(_jI(r["vlanId"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_VLAN_PCP") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; - rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); + rule.v.vlanPcp = (uint8_t)(_jI(r["vlanPcp"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_VLAN_DEI") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; - rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); + rule.v.vlanDei = (uint8_t)(_jI(r["vlanDei"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_ETHERTYPE") { rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; - rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); + rule.v.etherType = (uint16_t)(_jI(r["etherType"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_MAC_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; - const std::string mac(r.value("mac","0")); + const std::string mac(_jS(r["mac"],"0")); Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); return true; } else if (t == "MATCH_MAC_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; - const std::string mac(r.value("mac","0")); + const std::string mac(_jS(r["mac"],"0")); Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); return true; } else if (t == "MATCH_IPV4_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; - InetAddress ip(r.value("ip","0.0.0.0")); + InetAddress ip(_jS(r["ip"],"0.0.0.0")); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV4_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; - InetAddress ip(r.value("ip","0.0.0.0")); + InetAddress ip(_jS(r["ip"],"0.0.0.0")); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV6_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; - InetAddress ip(r.value("ip","::0")); + InetAddress ip(_jS(r["ip"],"::0")); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IPV6_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; - InetAddress ip(r.value("ip","::0")); + InetAddress ip(_jS(r["ip"],"::0")); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IP_TOS") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; - rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); + rule.v.ipTos = (uint8_t)(_jI(r["ipTos"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_IP_PROTOCOL") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; - rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); + rule.v.ipProtocol = (uint8_t)(_jI(r["ipProtocol"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.port[0]) & 0xffffULL); + rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_IP_DEST_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.port[0]) & 0xffffULL); + rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; @@ -310,28 +354,28 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_FRAME_SIZE_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.frameSize[0]) & 0xffffULL); + rule.v.frameSize[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; } else if (t == "MATCH_TAGS_SAMENESS") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } return false; @@ -613,7 +657,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( json member(_readJson(memberJP)); { - std::string haveIdStr = member.value("identity",""); + std::string haveIdStr(_jS(member["identity"],"")); if (haveIdStr.length() > 0) { try { if (Identity(haveIdStr.c_str()) != identity) @@ -630,13 +674,18 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["id"] = identity.address().toString(); member["address"] = member["id"]; member["nwid"] = network["id"]; - member["memberRevision"] = member.value("memberRevision",0ULL) + 1; + member["lastModified"] = now; + { + auto revj = member["revision"]; + const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + member["revision"] = rev; + } // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; - if (!network.value("private",true)) { + if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; - } else if (member.value("authorized",false)) { + } else if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; } else { char atok[256]; @@ -648,8 +697,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { authorizedBy = "token"; break; @@ -700,14 +749,14 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // If we made it this far, they are authorized. nc.networkId = nwid; - nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; - nc.revision = network.value("revision",0ULL); + nc.revision = _jI(network["revision"],0ULL); nc.issuedTo = identity.address(); - if (network.value("enableBroadcast",true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (network.value("allowPassiveBridging",false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),network.value("name","").c_str()); - nc.multicastLimit = (unsigned int)network.value("multicastLimit",32ULL); + if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (_jB(network["allowPassiveBridging"],false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); + nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); bool amActiveBridge = false; { @@ -732,11 +781,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( auto rules = network["rules"]; if (v6AssignMode.is_object()) { - if ((v6AssignMode.value("rfc4193",false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } - if ((v6AssignMode.value("6plane",false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } @@ -757,8 +806,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if (nc.routeCount >= ZT_MAX_NETWORK_ROUTES) break; auto route = routes[i]; - InetAddress t(route.value("target","")); - InetAddress v(route.value("via","")); + InetAddress t(_jS(route["target"],"")); + InetAddress v(_jS(route["via"],"")); if ((t)&&(v)&&(t.ss_family == v.ss_family)) { ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); *(reinterpret_cast(&(r->target))) = t; @@ -803,13 +852,13 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( std::set allocatedIps; bool allocatedIpsLoaded = false; - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(v6AssignMode.value("zt",false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); for(unsigned long p=0;((p(&ipRangeStart)->sin_addr.s_addr)); uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); @@ -921,7 +970,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - if (network.value("private",true)) { + if (_jB(network["private"],true)) { CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); if (com.sign(signingId)) { nc.com = com; @@ -983,8 +1032,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); responseBody.append("\":"); - const std::string rc = member.value("memberRevision","0"); - responseBody.append(rc); + auto rev = member["revision"]; + if (rev.is_number()) + responseBody.append(rev); + else responseBody.push_back('0'); } } } @@ -1006,7 +1057,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( auto recentLog = member["recentLog"]; if ((recentLog.is_array())&&(recentLog.size() > 0)) { auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + if ((mostRecentLog.is_object())&&(_jI(mostRecentLog["ts"],0ULL) >= threshold)) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); responseBody.append("\":"); @@ -1116,8 +1167,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json member(_readJson(_memberJP(nwid,Address(address),true))); try { - if (b.count("authorized")) member["authorized"] = b.value("authorized",false); - if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known + if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); + if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = _jS(b["identity"],""); // allow identity to be populated only if not already known if (b.count("ipAssignments")) { auto ipa = b["ipAssignments"]; @@ -1144,12 +1195,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["id"] = addrs; member["address"] = addrs; // legacy member["nwid"] = nwids; - member["memberRevision"] = member.value("memberRevision",0ULL) + 1; member["objtype"] = "member"; + member["lastModified"] = OSUtils::now(); + { + auto revj = member["revision"]; + const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + member["revision"] = rev; + } _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); - member["clock"] = OSUtils::now(); + member["clock"] = member["lastModified"]; responseBody = member.dump(2); responseContentType = "application/json"; return 200; @@ -1178,7 +1234,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } } } - test->reportAtEveryHop = (b.value("reportAtEveryHop",true) ? 1 : 0); + test->reportAtEveryHop = (_jB(b["reportAtEveryHop"],true) ? 1 : 0); if (!test->hopCount) { ::free((void *)test); @@ -1226,11 +1282,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json network(_readJson(_networkJP(nwid,true))); try { - if (b.count("name")) network["name"] = b.value("name",""); - if (b.count("private")) network["private"] = b.value("private",true); - if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); - if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); - if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); + if (b.count("name")) network["name"] = _jS(b["name"],""); + if (b.count("private")) network["private"] = _jB(b["private"],true); + if (b.count("enableBroadcast")) network["enableBroadcast"] = _jB(b["enableBroadcast"],false); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = _jB(b["allowPassiveBridging"],false); + if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); if (b.count("activeBridges")) { auto ab = b["activeBridges"]; @@ -1249,10 +1305,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( auto nv4m = network["v4AssignMode"]; if (!nv4m.is_object()) nv4m = json::object(); if (b["v4AssignMode"].is_string()) { // backward compatibility - nv4m["zt"] = (b.value("v4AssignMode","") == "zt"); + nv4m["zt"] = (_jS(b["v4AssignMode"],"") == "zt"); } else if (b["v4AssignMode"].is_object()) { auto v4m = b["v4AssignMode"]; - if (v4m.count("zt")) nv4m["zt"] = v4m.value("zt",false); + if (v4m.count("zt")) nv4m["zt"] = _jB(v4m["zt"],false); } if (!nv4m.count("zt")) nv4m["zt"] = false; } @@ -1261,7 +1317,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( auto nv6m = network["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (b["v6AssignMode"].is_string()) { // backward compatibility - std::vector v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); + std::vector v6m(Utils::split(_jS(b["v6AssignMode"],"").c_str(),",","","")); std::sort(v6m.begin(),v6m.end()); v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { @@ -1274,9 +1330,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } } else if (b["v6AssignMode"].is_object()) { auto v6m = b["v6AssignMode"]; - if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); - if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); - if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); + if (v6m.count("rfc4193")) nv6m["rfc4193"] = _jB(v6m["rfc4193"],false); + if (v6m.count("zt")) nv6m["rfc4193"] = _jB(v6m["zt"],false); + if (v6m.count("6plane")) nv6m["rfc4193"] = _jB(v6m["6plane"],false); } if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; if (!nv6m.count("zt")) nv6m["zt"] = false; @@ -1289,8 +1345,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i 0) { json t = json::object(); t["token"] = tstr; - t["expires"] = token.value("expires",0ULL); + t["expires"] = _jI(token["expires"],0ULL); nat.push_back(t); } } @@ -1372,8 +1428,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!network.count("creationTime")) network["creationTime"] = OSUtils::now(); if (!network.count("name")) network["name"] = ""; if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; - if (!network.count("v4AssignMode")) network["v4AssignMode"] = "{\"zt\":false}"_json; - if (!network.count("v6AssignMode")) network["v6AssignMode"] = "{\"rfc4193\":false,\"zt\":false,\"6plane\":false}"_json; + if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; + if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; if (!network.count("activeBridges")) network["activeBridges"] = json::array(); if (!network.count("authTokens")) network["authTokens"] = json::array(); @@ -1387,7 +1443,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy - network["revision"] = network.value("revision",0ULL) + 1ULL; + auto rev = network["revision"]; + network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL); network["objtype"] = "network"; _writeJson(_networkJP(nwid,true),network); diff --git a/controller/README.md b/controller/README.md index 2c7541ae..0b57dd25 100644 --- a/controller/README.md +++ b/controller/README.md @@ -5,7 +5,7 @@ ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit This code implements the *node/NetworkController.hpp* interface to provide an embedded microservice configurable via the same local HTTP control plane as ZeroTier One iteself. It is built by default in ZeroTier One in desktop and server builds. This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. -Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, or edited in place, though we do not recommend doing the latter while the controller is running. Also take care if editing in place that you do not save corrupted JSON since the controller may then lose data when it attempts to load, modify, and save. +Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. Technically the JSON files under `controller.d` can also be edited in place, but we do not recommend doing this under a running controller since data loss or corruption might result. Also take care to keep JSON values of the correct types or data loss may also result. ### Scalability and Reliability diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 5c9e80f8..e4f09106 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -804,8 +804,12 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.armor(peer->key(),true); RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } + } catch (std::exception &exc) { + fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); + TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what()); } catch ( ... ) { - TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + fprintf(stderr,"WARNING: network config request failed with exception: unknown exception" ZT_EOL_S); + TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } return true; } diff --git a/service/OneService.cpp b/service/OneService.cpp index 74628168..7ce45beb 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1705,7 +1705,11 @@ public: if (_controlPlane) scode = _controlPlane->handleRequest(tc->from,tc->parser.method,tc->url,tc->headers,tc->body,data,contentType); else scode = 500; + } catch (std::exception &exc) { + fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: %s" ZT_EOL_S,exc.what()); + scode = 500; } catch ( ... ) { + fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: unknown exceptino" ZT_EOL_S); scode = 500; } -- cgit v1.2.3 From f119c4a45610bf5774a3fc88d0883b2fe8db6e8d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 18 Aug 2016 12:59:48 -0700 Subject: Cache network members for performance, add network non-persisted fields. --- controller/EmbeddedNetworkController.cpp | 404 ++++++++----------------------- controller/EmbeddedNetworkController.hpp | 54 +++-- controller/README.md | 7 +- 3 files changed, 137 insertions(+), 328 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 649ff094..c861e4d8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -386,247 +386,7 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa _path(dbPath) { OSUtils::mkdir(dbPath); - /* - if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) - throw std::runtime_error("SqliteNetworkController cannot open database file"); - sqlite3_busy_timeout(_db,10000); - - sqlite3_exec(_db,"PRAGMA synchronous = OFF",0,0,0); - sqlite3_exec(_db,"PRAGMA journal_mode = MEMORY",0,0,0); - - sqlite3_stmt *s = (sqlite3_stmt *)0; - if ((sqlite3_prepare_v2(_db,"SELECT v FROM Config WHERE k = 'schemaVersion';",-1,&s,(const char **)0) == SQLITE_OK)&&(s)) { - int schemaVersion = -1234; - if (sqlite3_step(s) == SQLITE_ROW) { - schemaVersion = sqlite3_column_int(s,0); - } - - sqlite3_finalize(s); - - if (schemaVersion == -1234) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController schemaVersion not found in Config table (init failure?)"); - } - - if (schemaVersion < 2) { - // Create NodeHistory table to upgrade from version 1 to version 2 - if (sqlite3_exec(_db, - "CREATE TABLE NodeHistory (\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " networkVisitCounter INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestAuthorized INTEGER NOT NULL DEFAULT(0),\n" - " requestTime INTEGER NOT NULL DEFAULT(0),\n" - " clientMajorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientMinorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientRevision INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestMetaData VARCHAR(1024),\n" - " fromAddress VARCHAR(128)\n" - ");\n" - "CREATE INDEX NodeHistory_nodeId ON NodeHistory (nodeId);\n" - "CREATE INDEX NodeHistory_networkId ON NodeHistory (networkId);\n" - "CREATE INDEX NodeHistory_requestTime ON NodeHistory (requestTime);\n" - "UPDATE \"Config\" SET \"v\" = 2 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 2: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 2; - } - } - - if (schemaVersion < 3) { - // Create Route table to upgrade from version 2 to version 3 and migrate old - // data. Also delete obsolete Gateway table that was never actually used, and - // migrate Network flags to a bitwise flags field instead of ASCII cruft. - if (sqlite3_exec(_db, - "DROP TABLE Gateway;\n" - "CREATE TABLE Route (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " target blob(16) NOT NULL,\n" - " via blob(16),\n" - " targetNetmaskBits integer NOT NULL,\n" - " ipVersion integer NOT NULL,\n" - " flags integer NOT NULL,\n" - " metric integer NOT NULL\n" - ");\n" - "CREATE INDEX Route_networkId ON Route (networkId);\n" - "INSERT INTO Route SELECT DISTINCT networkId,\"ip\" AS \"target\",NULL AS \"via\",ipNetmaskBits AS targetNetmaskBits,ipVersion,0 AS \"flags\",0 AS \"metric\" FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "ALTER TABLE Network ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 1) WHERE v4AssignMode = 'zt';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 2) WHERE v6AssignMode = 'rfc4193';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 4) WHERE v6AssignMode = '6plane';\n" - "ALTER TABLE Member ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "DELETE FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "UPDATE \"Config\" SET \"v\" = 3 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 3; - } - } - - if (schemaVersion < 4) { - // Turns out this was overkill and a huge performance drag. Will be revisiting this - // more later but for now a brief snapshot of recent history stored in Member is fine. - // Also prepare for implementation of proof of work requests. - if (sqlite3_exec(_db, - "DROP TABLE NodeHistory;\n" - "ALTER TABLE Member ADD COLUMN lastRequestTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowDifficulty integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN recentHistory blob;\n" - "CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n" - "UPDATE \"Config\" SET \"v\" = 4 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 4; - } - } - - if (schemaVersion < 5) { - // Upgrade old rough draft Rule table to new release format - if (sqlite3_exec(_db, - "DROP TABLE Relay;\n" - "DROP INDEX Rule_networkId_ruleNo;\n" - "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" - "CREATE TABLE Rule (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " capId integer,\n" - " ruleNo integer NOT NULL,\n" - " ruleType integer NOT NULL DEFAULT(0),\n" - " \"addr\" blob(16),\n" - " \"int1\" integer,\n" - " \"int2\" integer,\n" - " \"int3\" integer,\n" - " \"int4\" integer\n" - ");\n" - "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" - "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" - "DROP TABLE RuleOld;\n" - "CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n" - "CREATE TABLE MemberTC (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " tagId integer,\n" - " tagValue integer,\n" - " capId integer,\n" - " capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n" - ");\n" - "CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n" - "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 5; - } - } - - if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController database schema version mismatch"); - } - } else { - // Prepare statement will fail if Config table doesn't exist, which means our DB - // needs to be initialized. - if (sqlite3_exec(_db,ZT_NETCONF_SCHEMA_SQL"INSERT INTO Config (k,v) VALUES ('schemaVersion',"ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR");",0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot initialize database and/or insert schemaVersion into Config table: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - } - - if ( - - (sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Network (id,name,creationTime,revision) VALUES (?,?,?,1)",-1,&_sCreateNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Network WHERE id = ?",-1,&_sDeleteNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId FROM Member WHERE networkId = ? AND activeBridge > 0 AND authorized > 0",-1,&_sGetActiveBridges,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.nodeId,m.memberRevision FROM Member AS m WHERE m.networkId = ? ORDER BY m.nodeId ASC",-1,&_sListNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET authorized = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberAuthorized,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET activeBridge = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberActiveBridge,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET \"lastRequestTime\" = ?, \"recentHistory\" = ? WHERE rowid = ?",-1,&_sUpdateMemberHistory,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sDeleteMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(const char **)0) != SQLITE_OK) - - ) { - std::string err(std::string("SqliteNetworkController unable to initialize one or more prepared statements: ") + sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - - sqlite3_reset(_sGetConfig); - sqlite3_bind_text(_sGetConfig,1,"instanceId",10,SQLITE_STATIC); - if (sqlite3_step(_sGetConfig) != SQLITE_ROW) { - unsigned char sr[32]; - Utils::getSecureRandom(sr,32); - for(unsigned int i=0;i<32;++i) - _instanceId.push_back("0123456789abcdef"[(unsigned int)sr[i] & 0xf]); - - sqlite3_reset(_sSetConfig); - sqlite3_bind_text(_sSetConfig,1,"instanceId",10,SQLITE_STATIC); - sqlite3_bind_text(_sSetConfig,2,_instanceId.c_str(),-1,SQLITE_STATIC); - if (sqlite3_step(_sSetConfig) != SQLITE_DONE) - throw std::runtime_error("SqliteNetworkController unable to read or initialize instanceId"); - } else { - const char *iid = reinterpret_cast(sqlite3_column_text(_sGetConfig,0)); - if (!iid) - throw std::runtime_error("SqliteNetworkController unable to read instanceId (it's NULL)"); - _instanceId = iid; - } - -#ifdef ZT_NETCONF_SQLITE_TRACE - sqlite3_trace(_db,sqliteTraceFunc,(void *)0); -#endif - - _backupThread = Thread::start(this); - */ + OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions } EmbeddedNetworkController::~EmbeddedNetworkController() @@ -653,12 +413,16 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( json network(_readJson(_networkJP(nwid,false))); if (!network.size()) return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; - const std::string memberJP(_memberJP(nwid,identity.address(),false)); + + const std::string memberJP(_memberJP(nwid,identity.address(),true)); json member(_readJson(memberJP)); { std::string haveIdStr(_jS(member["identity"],"")); if (haveIdStr.length() > 0) { + // If we already know this member's identity perform a full compare. This prevents + // a "collision" from being able to auth onto our network in place of an already + // known member. try { if (Identity(haveIdStr.c_str()) != identity) return NetworkController::NETCONF_QUERY_ACCESS_DENIED; @@ -666,6 +430,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } } else { + // If we do not yet know this member's identity, learn it. member["identity"] = identity.toString(false); } } @@ -685,13 +450,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( const char *authorizedBy = (const char *)0; if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; + // If member already has an authorized field, leave it alone. That way its state is + // preserved if the user toggles the network back to private. Otherwise set it to + // true by default for new members of public nets. + if (!member.count("authorized")) member["authorized"] = true; } else if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; } else { char atok[256]; if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { atok[255] = (char)0; // not necessary but YDIFLO - if (strlen(atok) > 0) { // extra sanity check + if (strlen(atok) > 0) { // extra sanity check since we never want to compare a null token on either side auto authTokens = network["authTokens"]; if (authTokens.is_array()) { for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { authorizedBy = "token"; + member["authorized"] = true; // tokens actually change member authorization state break; } } @@ -710,10 +480,6 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - // Remember authorization state for public networks and token auth - if (authorizedBy) - member["authorized"] = true; - // Log this request { json rlEntry = json::object(); @@ -748,6 +514,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // If we made it this far, they are authorized. + _NetworkMemberInfo nmi; + _getNetworkMemberInfo(now,nwid,nmi); + nc.networkId = nwid; nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; @@ -758,21 +527,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); - bool amActiveBridge = false; - { - json ab = network["activeBridges"]; - if (ab.is_array()) { - for(unsigned long i=0;i::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) + nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); auto v4AssignMode = network["v4AssignMode"]; auto v6AssignMode = network["v6AssignMode"]; @@ -780,17 +536,6 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( auto routes = network["routes"]; auto rules = network["rules"]; - if (v6AssignMode.is_object()) { - if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; - } - if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; - } - } - if (rules.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) @@ -817,6 +562,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } + if (v6AssignMode.is_object()) { + if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); + nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + } + if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); + nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + } + } + bool haveManagedIpv4AutoAssignment = false; bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count json ipAssignments = member["ipAssignments"]; @@ -849,11 +605,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( ipAssignments = json::array(); } - std::set allocatedIps; - bool allocatedIpsLoaded = false; - - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { for(unsigned long p=0;((p 0)&&(!allocatedIps.count(ip6))) { + if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip6))) { ipAssignments.push_back(ip6.toIpString()); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); @@ -913,8 +665,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(_jB(v4AssignMode["zt"],false))) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(_jB(v4AssignMode["zt"],false))) && (!haveManagedIpv4AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { for(unsigned long p=0;((p 0)&&(!allocatedIps.count(ip4))) { + if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { @@ -1016,7 +767,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + // Add non-persisted fields member["clock"] = OSUtils::now(); + responseBody = member.dump(2); responseContentType = "application/json"; @@ -1090,9 +843,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } else { - nlohmann::json o(network); - o["clock"] = OSUtils::now(); - responseBody = o.dump(2); + const uint64_t now = OSUtils::now(); + _NetworkMemberInfo nmi; + _getNetworkMemberInfo(now,nwid,nmi); + _addNetworkNonPersistedFields(network,now,nmi); + responseBody = network.dump(2); responseContentType = "application/json"; return 200; @@ -1146,6 +901,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } catch ( ... ) { return 400; } + const uint64_t now = OSUtils::now(); if (path[0] == "network") { @@ -1168,6 +924,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( try { if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); + if (b.count("activeBridge")) member["activeBridge"] = _jB(b["activeBridge"],false); if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = _jS(b["identity"],""); // allow identity to be populated only if not already known if (b.count("ipAssignments")) { @@ -1191,12 +948,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!member.count("authorized")) member["authorized"] = false; if (!member.count("ipAssignments")) member["ipAssignments"] = json::array(); if (!member.count("recentLog")) member["recentLog"] = json::array(); + if (!member.count("activeBridge")) member["activeBridge"] = false; member["id"] = addrs; member["address"] = addrs; // legacy member["nwid"] = nwids; member["objtype"] = "member"; - member["lastModified"] = OSUtils::now(); + member["lastModified"] = now; { auto revj = member["revision"]; const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); @@ -1205,7 +963,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); + // Add non-persisted fields member["clock"] = member["lastModified"]; + responseBody = member.dump(2); responseContentType = "application/json"; return 200; @@ -1288,19 +1048,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = _jB(b["allowPassiveBridging"],false); if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); - if (b.count("activeBridges")) { - auto ab = b["activeBridges"]; - if (ab.is_array()) { - json ab2 = json::array(); - for(unsigned long i=0;isecond.jsonResults.append(tmp); } +void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) +{ + Mutex::Lock _mcl(_networkMemberCache_m); + + auto memberCacheEntry = _networkMemberCache[nwid]; + if ((now - memberCacheEntry.second) >= ZT_NETCONF_NETWORK_MEMBER_CACHE_EXPIRE) { + const std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); + std::vector members(OSUtils::listSubdirectories(bp.c_str())); + for(std::vector::iterator m(members.begin());m!=members.end();++m) { + if (m->length() == ZT_ADDRESS_LENGTH_HEX) { + nlohmann::json mj(_readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json")); + if ((mj.is_object())&&(mj.size() > 0)) { + memberCacheEntry.first[Address(*m)] = mj; + } + } + } + memberCacheEntry.second = now; + } + + nmi.totalMemberCount = memberCacheEntry.first.size(); + for(std::map< Address,nlohmann::json >::const_iterator nm(memberCacheEntry.first.begin());nm!=memberCacheEntry.first.end();++nm) { + if (_jB(nm->second["authorized"],false)) { + ++nmi.authorizedMemberCount; + + auto mlog = nm->second["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + auto mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) + ++nmi.activeMemberCount; + } + } + + if (_jB(nm->second["activeBridge"],false)) { + nmi.activeBridges.insert(nm->first); + } + + auto mips = nm->second["ipAssignments"]; + if (mips.is_array()) { + for(unsigned long i=0;i _getAlreadyAllocatedIps(const uint64_t nwid) + // We cache the members of networks in memory to avoid having to scan the filesystem so much + std::map< uint64_t,std::pair< std::map< Address,nlohmann::json >,uint64_t > > _networkMemberCache; + Mutex _networkMemberCache_m; + + // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places + // This does lock _networkMemberCache_m + struct _NetworkMemberInfo { - std::set ips; - std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); - std::vector members(OSUtils::listSubdirectories(bp.c_str())); - for(std::vector::iterator m(members.begin());m!=members.end();++m) { - if (m->length() == ZT_ADDRESS_LENGTH_HEX) { - nlohmann::json mj = _readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"); - auto ipAssignments = mj["ipAssignments"]; - if (ipAssignments.is_array()) { - for(unsigned long i=0;i activeBridges; + std::set allocatedIps; + unsigned long authorizedMemberCount; + unsigned long activeMemberCount; + unsigned long totalMemberCount; + }; + void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); + + inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const _NetworkMemberInfo &nmi) + { + network["clock"] = now; + network["authorizedMemberCount"] = nmi.authorizedMemberCount; + network["activeMemberCount"] = nmi.activeMemberCount; + network["totalMemberCount"] = nmi.totalMemberCount; } // These are const after construction diff --git a/controller/README.md b/controller/README.md index 0b57dd25..339adb31 100644 --- a/controller/README.md +++ b/controller/README.md @@ -11,8 +11,6 @@ Data is stored in JSON format under `controller.d` in the ZeroTier working direc Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. -Since this implementation uses a JSON store in the filesystem we recommend running it on SSD-backed hosts. Slow disks will become a speed bottleneck under heavy load. For really huge and busy controllers you could consider linking `controller.d/` to a folder under `/dev/shm` (Linux RAM disk) and then setting up an out-of-band periodic snapshot cron job or background process to persist the data and a script to populate `/dev/shm` on boot before the controller starts. This is beyond the scope of this guide but is not particularly hard. - Since ZeroTier nodes are mobile and do not need static IPs, implementing high availability fail-over for controllers is easy. Just replicate their working directories from master to backup and have something automatically fire up the backup if the master goes down. Many modern orchestration tools have built-in support for this. It would also be possible in theory to run controllers on a replicated or distributed filesystem, but we haven't tested this yet. ### Dockerizing Controllers @@ -67,15 +65,15 @@ When POSTing new networks take care that their IDs are not in use, otherwise you | name | string | A short name for this network | YES | | private | boolean | Is access control enabled? | YES | | enableBroadcast | boolean | Ethernet ff:ff:ff:ff:ff:ff allowed? | YES | -| activeBridges | array[string] | Array of ZeroTier addresses of active bridges | YES | | allowPassiveBridging | boolean | Allow any member to bridge (very experimental) | YES | | v4AssignMode | object | IPv4 management and assign options (see below) | YES | | v6AssignMode | object | IPv6 management and assign options (see below) | YES | | multicastLimit | integer | Maximum recipients for a multicast packet | YES | | creationTime | integer | Time network was first created | no | | revision | integer | Network config revision counter | no | -| memberRevisionCounter | integer | Network member revision counter | no | | authorizedMemberCount | integer | Number of authorized members (for private nets) | no | +| activeMemberCount | integer | Number of members that appear to be online | no | +| totalMemberCount | integer | Total known members of this network | no | | routes | array[object] | Managed IPv4 and IPv6 routes; see below | YES | | ipAssignmentPools | array[object] | IP auto-assign ranges; see below | YES | | rules | array[object] | Traffic rules; see below | YES | @@ -84,7 +82,6 @@ Recent changes: * The `ipLocalRoutes` field appeared in older versions but is no longer present. Routes will now show up in `routes`. * The `relays` field is gone since network preferred relays are gone. This capability is replaced by VL1 level federation ("federated roots"). - * Active bridges are now set at the network level, not in individual member configs. Other important points: -- cgit v1.2.3 From 1cadbfb4d1c26f5765253f052b218f7c74a42021 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 18 Aug 2016 13:47:02 -0700 Subject: Little fixes. --- controller/EmbeddedNetworkController.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c861e4d8..82a24284 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -60,6 +60,7 @@ using json = nlohmann::json; namespace ZeroTier { +// Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible static uint64_t _jI(const json &jv,const uint64_t dfl) { if (jv.is_number()) { @@ -97,7 +98,9 @@ static std::string _jS(const json &jv,const char *dfl) if (jv.is_string()) { return jv; } else if (jv.is_number()) { - return jv; + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); + return tmp; } else if (jv.is_boolean()) { return ((bool)jv ? std::string("1") : std::string("0")); } @@ -603,6 +606,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } else { ipAssignments = json::array(); + member["ipAssignments"] = ipAssignments; } if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { @@ -785,10 +789,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); responseBody.append("\":"); - auto rev = member["revision"]; - if (rev.is_number()) - responseBody.append(rev); - else responseBody.push_back('0'); + responseBody.append(_jS(member["revision"],"0")); } } } -- cgit v1.2.3 From 212a5af9a549270abc1230b93a80caf4c0cd4b3c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 18 Aug 2016 14:37:56 -0700 Subject: Capabilities and tags in POST JSON. --- controller/EmbeddedNetworkController.cpp | 102 +++++++++++++++++++++++-------- controller/EmbeddedNetworkController.hpp | 32 ++++++++++ 2 files changed, 109 insertions(+), 25 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 82a24284..d7d5c92c 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -438,6 +438,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } + _initMember(member); + // Make sure these are always present no matter what, and increment member revision since we will always at least log something member["id"] = identity.address().toString(); member["address"] = member["id"]; @@ -606,7 +608,6 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } else { ipAssignments = json::array(); - member["ipAssignments"] = ipAssignments; } if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { @@ -922,6 +923,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); json member(_readJson(_memberJP(nwid,Address(address),true))); + _initMember(member); try { if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); @@ -942,19 +944,46 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["ipAssignments"] = mipa; } } + + if (b.count("tags")) { + auto tags = b["tags"]; + if (tags.is_array()) { + std::map mtags; + for(unsigned long i=0;i::iterator t(mtags.begin());t!=mtags.end();++t) { + json ta = json::array(); + ta.push_back(t->first); + ta.push_back(t->second); + mtagsa.push_back(ta); + } + member["tags"] = mtagsa; + } + } + + if (b.count("capabilities")) { + auto capabilities = b["capabilities"]; + if (capabilities.is_array()) { + json mcaps = json::array(); + for(unsigned long i=0;i ncaps; + for(unsigned long i=0;i::iterator c(ncaps.begin());c!=ncaps.end();++c) + ncapsa.push_back(c->second); + network["capabilities"] = ncapsa; + } + } } catch ( ... ) { return 400; } - if (!network.count("private")) network["private"] = true; - if (!network.count("creationTime")) network["creationTime"] = OSUtils::now(); - if (!network.count("name")) network["name"] = ""; - if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; - if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; - if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; - if (!network.count("activeBridges")) network["activeBridges"] = json::array(); - if (!network.count("authTokens")) network["authTokens"] = json::array(); - - if (!network.count("rules")) { - // If unspecified, rules are set to allow anything and behave like a flat L2 segment - network["rules"] = { - { "not",false }, - { "type","ACTION_ACCEPT" } - }; - } - network["id"] = nwids; network["nwid"] = nwids; // legacy auto rev = network["revision"]; network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL); - network["objtype"] = "network"; network["lastModified"] = now; _writeJson(_networkJP(nwid,true),network); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 16a1adbe..e6b4bb59 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -150,6 +150,38 @@ private: }; void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); + // These init objects with default and static/informational fields + inline void _initMember(nlohmann::json &member) + { + if (!member.count("authorized")) member["authorized"] = false; + if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); + if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array(); + if (!member.count("activeBridge")) member["activeBridge"] = false; + if (!member.count("tags")) member["tags"] = nlohmann::json::array(); + if (!member.count("capabilities")) member["capabilities"] = nlohmann::json::array(); + if (!member.count("creationTime")) member["creationTime"] = OSUtils::now(); + member["objtype"] = "member"; + } + inline void _initNetwork(nlohmann::json &network) + { + if (!network.count("private")) network["private"] = true; + if (!network.count("creationTime")) network["creationTime"] = OSUtils::now(); + if (!network.count("name")) network["name"] = ""; + if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; + if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; + if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; + if (!network.count("activeBridges")) network["activeBridges"] = nlohmann::json::array(); + if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); + if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); + if (!network.count("rules")) { + // If unspecified, rules are set to allow anything and behave like a flat L2 segment + network["rules"] = { + { "not",false }, + { "type","ACTION_ACCEPT" } + }; + } + network["objtype"] = "network"; + } inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const _NetworkMemberInfo &nmi) { network["clock"] = now; -- cgit v1.2.3 From 4dce71879f5d15af6754c15c606554b1feccdeac Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 18 Aug 2016 18:18:50 -0700 Subject: . --- controller/EmbeddedNetworkController.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d7d5c92c..8d7e90c7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -50,7 +50,7 @@ using json = nlohmann::json; #define ZT_NETCONF_CONTROLLER_API_VERSION 3 // Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 64 +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 16 // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 @@ -540,6 +540,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( auto ipAssignmentPools = network["ipAssignmentPools"]; auto routes = network["routes"]; auto rules = network["rules"]; + auto capabilities = network["capabilities"]; + auto memberCapabilities = member["capabilities"]; + auto memberTags = member["tags"]; if (rules.is_array()) { for(unsigned long i=0;i 0)&&(capabilities.is_array())) { + std::map< uint64_t,json > capsById; + for(unsigned long i=0;i 0)) { + } + } + } + + if (memberTags.is_array()) { + } + if (routes.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) -- cgit v1.2.3 From b0d888d235ad8f830fb38090e35f80d49a84ebbf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 22 Aug 2016 14:25:59 -0700 Subject: Signing of Capability and Tag objects. --- controller/EmbeddedNetworkController.cpp | 36 ++++++++++++++++++++++++++++---- node/Capability.hpp | 3 +-- 2 files changed, 33 insertions(+), 6 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8d7e90c7..1088b852 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -548,8 +548,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) break; - auto rule = rules[i]; - if (_parseRule(rule,nc.rules[nc.ruleCount])) + if (_parseRule(rules[i],nc.rules[nc.ruleCount])) ++nc.ruleCount; } } @@ -559,18 +558,47 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(unsigned long i=0;i 0)) { + ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; + unsigned int caprc = 0; + auto caprj = cap["rules"]; + if ((caprj.is_array())&&(caprj.size() > 0)) { + for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) + break; + if (_parseRule(caprj[j],capr[caprc])) + ++caprc; + } + } + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,1,capr,caprc); + if (nc.capabilities[nc.capabilityCount].sign(signingId,identity.address())) + ++nc.capabilityCount; + if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + break; } } } if (memberTags.is_array()) { + std::map< uint32_t,uint32_t > tagsById; + for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { + if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + break; + nc.tags[nc.tagCount] = Tag(nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,identity.address(),t->first,t->second); + if (nc.tags[nc.tagCount].sign(signingId)) + ++nc.tagCount; + } } if (routes.is_array()) { diff --git a/node/Capability.hpp b/node/Capability.hpp index c129485d..689a2c6a 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -73,12 +73,11 @@ public: * @param nwid Network ID * @param ts Timestamp (at controller) * @param expiration Expiration relative to network config timestamp - * @param name Capability short name (max strlen == ZT_MAX_CAPABILITY_NAME_LENGTH, overflow ignored) * @param mccl Maximum custody chain length (1 to create non-transferrable capability) * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ - Capability(uint32_t id,uint64_t nwid,uint64_t ts,uint64_t expiration,const char *name,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + Capability(uint32_t id,uint64_t nwid,uint64_t ts,uint64_t expiration,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { memset(this,0,sizeof(Capability)); _nwid = nwid; -- cgit v1.2.3 From 9a3c652a518c40050a0190b489af9ab11647b0b0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 22 Aug 2016 18:06:46 -0700 Subject: Get rid of expiration in Capability and Tag and move this to NetworkConfig so it can be set network-wide and reset if needed. Also add NetworkConfig field for this and centralize checking of credential time validity. --- controller/EmbeddedNetworkController.cpp | 4 +-- node/Capability.hpp | 12 +------- node/CertificateOfMembership.hpp | 49 ++++++++------------------------ node/Membership.cpp | 2 +- node/Membership.hpp | 10 +++---- node/Network.cpp | 14 ++++----- node/NetworkConfig.cpp | 2 ++ node/NetworkConfig.hpp | 20 +++++++++++++ node/Tag.hpp | 8 +----- 9 files changed, 51 insertions(+), 70 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 1088b852..738fea9a 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -576,7 +576,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( ++caprc; } } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,1,capr,caprc); + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); if (nc.capabilities[nc.capabilityCount].sign(signingId,identity.address())) ++nc.capabilityCount; if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) @@ -595,7 +595,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(std::map< uint32_t,uint32_t >::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) break; - nc.tags[nc.tagCount] = Tag(nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,identity.address(),t->first,t->second); + nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); if (nc.tags[nc.tagCount].sign(signingId)) ++nc.tagCount; } diff --git a/node/Capability.hpp b/node/Capability.hpp index 689a2c6a..b0620891 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -72,17 +72,15 @@ public: * @param id Capability ID * @param nwid Network ID * @param ts Timestamp (at controller) - * @param expiration Expiration relative to network config timestamp * @param mccl Maximum custody chain length (1 to create non-transferrable capability) * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ - Capability(uint32_t id,uint64_t nwid,uint64_t ts,uint64_t expiration,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + Capability(uint32_t id,uint64_t nwid,uint64_t ts,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { memset(this,0,sizeof(Capability)); _nwid = nwid; _ts = ts; - _expiration = expiration; _id = id; _maxCustodyChainLength = (mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1; _ruleCount = (ruleCount < ZT_MAX_CAPABILITY_RULES) ? ruleCount : ZT_MAX_CAPABILITY_RULES; @@ -110,11 +108,6 @@ public: */ inline uint64_t networkId() const { return _nwid; } - /** - * @return Expiration time relative to network config timestamp - */ - inline uint64_t expiration() const { return _expiration; } - /** * @return Timestamp */ @@ -343,7 +336,6 @@ public: // These are the same between Tag and Capability b.append(_nwid); b.append(_ts); - b.append(_expiration); b.append(_id); b.append((uint16_t)_ruleCount); @@ -381,7 +373,6 @@ public: // These are the same between Tag and Capability _nwid = b.template at(p); p += 8; _ts = b.template at(p); p += 8; - _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; const unsigned int rc = b.template at(p); p += 2; @@ -420,7 +411,6 @@ public: private: uint64_t _nwid; uint64_t _ts; - uint64_t _expiration; uint32_t _id; unsigned int _maxCustodyChainLength; diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index a04f8255..304111d6 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -35,11 +35,6 @@ /** * Default window of time for certificate agreement - * - * Right now we use time for 'revision' so this is the maximum time divergence - * between two certs for them to agree. It comes out to five minutes, which - * gives a lot of margin for error if the controller hiccups or its clock - * drifts but causes de-authorized peers to fall off fast enough. */ #define ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA (ZT_NETWORK_AUTOCONF_DELAY * 5) @@ -93,25 +88,17 @@ public: enum ReservedId { /** - * Revision number of certificate - * - * Certificates may differ in revision number by a designated max - * delta. Differences wider than this cause certificates not to agree. + * Timestamp of certificate */ - COM_RESERVED_ID_REVISION = 0, + COM_RESERVED_ID_TIMESTAMP = 0, /** * Network ID for which certificate was issued - * - * maxDelta here is zero, since this must match. */ COM_RESERVED_ID_NETWORK_ID = 1, /** * ZeroTier address to whom certificate was issued - * - * maxDelta will be 0xffffffffffffffff here since it's permitted to differ - * from peers obviously. */ COM_RESERVED_ID_ISSUED_TO = 2 }; @@ -132,16 +119,16 @@ public: /** * Create from required fields common to all networks * - * @param revision Revision number of certificate + * @param timestamp Timestamp of certificate * @param timestampMaxDelta Maximum variation between timestamps on this net * @param nwid Network ID * @param issuedTo Certificate recipient */ - CertificateOfMembership(uint64_t revision,uint64_t revisionMaxDelta,uint64_t nwid,const Address &issuedTo) + CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Address &issuedTo) { - _qualifiers[0].id = COM_RESERVED_ID_REVISION; - _qualifiers[0].value = revision; - _qualifiers[0].maxDelta = revisionMaxDelta; + _qualifiers[0].id = COM_RESERVED_ID_TIMESTAMP; + _qualifiers[0].value = timestamp; + _qualifiers[0].maxDelta = timestampMaxDelta; _qualifiers[1].id = COM_RESERVED_ID_NETWORK_ID; _qualifiers[1].value = nwid; _qualifiers[1].maxDelta = 0; @@ -176,27 +163,15 @@ public: inline operator bool() const throw() { return (_qualifierCount != 0); } /** - * @return Maximum delta for mandatory revision field or 0 if field missing + * @return Timestamp for this cert and maximum delta for timestamp */ - inline uint64_t revisionMaxDelta() const + inline std::pair timestamp() const { for(unsigned int i=0;i<_qualifierCount;++i) { - if (_qualifiers[i].id == COM_RESERVED_ID_REVISION) - return _qualifiers[i].maxDelta; + if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP) + return std::pair(_qualifiers[i].value,_qualifiers[i].maxDelta); } - return 0ULL; - } - - /** - * @return Revision number for this cert - */ - inline uint64_t revision() const - { - for(unsigned int i=0;i<_qualifierCount;++i) { - if (_qualifiers[i].id == COM_RESERVED_ID_REVISION) - return _qualifiers[i].value; - } - return 0ULL; + return std::pair(0ULL,0ULL); } /** diff --git a/node/Membership.cpp b/node/Membership.cpp index dbba7f0d..969032ff 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -86,7 +86,7 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe if (_com == com) return 0; const int vr = com.verify(RR); - if ((vr == 0)&&(com.revision() > _com.revision())) + if ((vr == 0)&&(com.timestamp().first > _com.timestamp().first)) _com = com; return vr; } diff --git a/node/Membership.hpp b/node/Membership.hpp index e9f9d488..92bd7ebf 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -84,10 +84,10 @@ public: { } - inline const Capability *next() + inline const Capability *next(const NetworkConfig &nconf) { while (_i != _e) { - if (_i->second.lastReceived) + if ((_i->second.lastReceived)&&(nconf.isCredentialTimestampValid(_i->second.cap))) return &((_i++)->second.cap); else ++_i; } @@ -137,7 +137,7 @@ public: inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const { const TState *t = _tags.get(id); - return ((t) ? (((t->lastReceived != 0)&&(t->tag.expiration() < nconf.timestamp)) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); + return ((t) ? (((t->lastReceived != 0)&&(nconf.isCredentialTimestampValid(t->tag))) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); } /** @@ -154,7 +154,7 @@ public: TState *ts = (TState *)0; Hashtable::Iterator i(const_cast(this)->_tags); while (i.next(id,ts)) { - if ((ts->lastReceived)&&(ts->tag.expiration() < nconf.timestamp)) { + if ((ts->lastReceived)&&(nconf.isCredentialTimestampValid(ts->tag))) { if (n >= maxTags) return n; ids[n] = *id; @@ -172,7 +172,7 @@ public: inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const { std::map::const_iterator c(_caps.find(id)); - return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(c->second.cap.expiration() < nconf.timestamp)) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); + return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(nconf.isCredentialTimestampValid(c->second.cap))) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); } /** diff --git a/node/Network.cpp b/node/Network.cpp index 0bd4ea55..7adb6aeb 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -102,7 +102,7 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig // 0 == no match, -1 == match/drop, 1 == match/accept static int _doZtFilter( const RuntimeEnvironment *RR, - const uint64_t nwid, + const NetworkConfig &nconf, const bool inbound, const Address &ztSource, const Address &ztDest, @@ -155,7 +155,7 @@ static int _doZtFilter( case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(nwid); + outp.append(nconf.networkId); outp.append((uint8_t)((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02)); macDest.appendTo(outp); macSource.appendTo(outp); @@ -481,7 +481,7 @@ bool Network::filterOutgoingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch(_doZtFilter(RR,_id,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -491,7 +491,7 @@ bool Network::filterOutgoingPacket( for(unsigned int c=0;c<_config.capabilityCount;++c) { relevantLocalTagCount = 0; - switch (_doZtFilter(RR,_id,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -523,7 +523,7 @@ bool Network::filterIncomingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,_id,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -532,9 +532,9 @@ bool Network::filterIncomingPacket( Membership::CapabilityIterator mci(m); const Capability *c; - while ((c = mci.next())) { + while ((c = mci.next(_config))) { relevantLocalTagCount = 0; - switch(_doZtFilter(RR,_id,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index a4fddf40..14ebb209 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -37,6 +37,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,this->networkId)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,this->credentialTimeToLive)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; @@ -202,6 +203,7 @@ bool NetworkConfig::fromDictionary(const Identity &controllerId,Dictionarytimestamp = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,0); + this->credentialTimeToLive = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,0); this->revision = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REVISION,0); this->issuedTo = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,0); if (!this->issuedTo) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 25e0ccaf..97e9287a 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -125,6 +125,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_TYPE "t" // text #define ZT_NETWORKCONFIG_DICT_KEY_NAME "n" +// credential time to live in ms +#define ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL "cttl" // binary serialized certificate of membership #define ZT_NETWORKCONFIG_DICT_KEY_COM "C" // specialists (binary array of uint64_t) @@ -367,11 +369,24 @@ public: return (Tag *)0; } + /** + * Check whether a capability or tag is expired + * + * @param cred Credential to check -- must have timestamp() accessor method + * @return True if credential is NOT expired + */ + template + inline bool isCredentialTimestampValid(const C &cred) const + { + return ( (cred.timestamp() >= timestamp) || ((timestamp - cred.timestamp()) <= credentialTimeToLive) ); + } + /* inline void dump() const { printf("networkId==%.16llx\n",networkId); printf("timestamp==%llu\n",timestamp); + printf("credentialTimeToLive==%llu\n",credentialTimeToLive); printf("revision==%llu\n",revision); printf("issuedTo==%.10llx\n",issuedTo.toInt()); printf("multicastLimit==%u\n",multicastLimit); @@ -405,6 +420,11 @@ public: */ uint64_t timestamp; + /** + * TTL for capabilities and tags + */ + uint64_t credentialTimeToLive; + /** * Controller-side revision counter for this configuration */ diff --git a/node/Tag.hpp b/node/Tag.hpp index bb019474..14cc3a5d 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -62,15 +62,13 @@ public: /** * @param nwid Network ID * @param ts Timestamp - * @param expiration Tag expiration relative to network config timestamp * @param issuedTo Address to which this tag was issued * @param id Tag ID * @param value Tag value */ - Tag(const uint64_t nwid,const uint64_t ts,const uint64_t expiration,const Address &issuedTo,const uint32_t id,const uint32_t value) : + Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : _nwid(nwid), _ts(ts), - _expiration(expiration), _id(id), _value(value), _issuedTo(issuedTo), @@ -79,7 +77,6 @@ public: } inline uint64_t networkId() const { return _nwid; } - inline uint64_t expiration() const { return _expiration; } inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } @@ -120,7 +117,6 @@ public: // These are the same between Tag and Capability b.append(_nwid); b.append(_ts); - b.append(_expiration); b.append(_id); b.append(_value); @@ -146,7 +142,6 @@ public: // These are the same between Tag and Capability _nwid = b.template at(p); p += 8; _ts = b.template at(p); p += 8; - _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; _value = b.template at(p); p += 4; @@ -176,7 +171,6 @@ public: private: uint64_t _nwid; uint64_t _ts; - uint64_t _expiration; uint32_t _id; uint32_t _value; Address _issuedTo; -- cgit v1.2.3 From 32fa0617004e80c99b341eb1b4753705b515b53a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 13:02:59 -0700 Subject: Compute credential TTL et al. --- controller/EmbeddedNetworkController.cpp | 44 +++++++++++++++++++++++++++++--- controller/EmbeddedNetworkController.hpp | 8 ++++-- controller/README.md | 3 +++ node/CertificateOfMembership.hpp | 5 ---- node/Membership.hpp | 4 +-- node/NetworkConfig.hpp | 16 ++++++++++++ 6 files changed, 67 insertions(+), 13 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 738fea9a..4db219c9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -413,7 +413,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( lrt = now; } - json network(_readJson(_networkJP(nwid,false))); + const json network(_readJson(_networkJP(nwid,false))); if (!network.size()) return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; @@ -458,7 +458,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // If member already has an authorized field, leave it alone. That way its state is // preserved if the user toggles the network back to private. Otherwise set it to // true by default for new members of public nets. - if (!member.count("authorized")) member["authorized"] = true; + if (!member.count("authorized")) { + member["authorized"] = true; + member["lastAuthorizedTime"] = now; + member["lastAuthorizedBy"] = authorizedBy; + } } else if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; } else { @@ -476,6 +480,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if ( ((expires == 0ULL)||(expires > now)) && (tok.length() > 0) && (tok == atok) ) { authorizedBy = "token"; member["authorized"] = true; // tokens actually change member authorization state + member["lastAuthorizedTime"] = now; + member["lastAuthorizedBy"] = authorizedBy; break; } } @@ -517,14 +523,28 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } + // ------------------------------------------------------------------------- // If we made it this far, they are authorized. + // ------------------------------------------------------------------------- _NetworkMemberInfo nmi; _getNetworkMemberInfo(now,nwid,nmi); + // Compute credential TTL. This is the "moving window" for COM agreement and + // the global TTL for Capability and Tag objects. (The same value is used + // for both.) This is computed by reference to the last time we deauthorized + // a member, since within the time period since this event any temporal + // differences are not particularly relevant. + uint64_t credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL; + if (now > nmi.mostRecentDeauthTime) + credentialTtl += (now - nmi.mostRecentDeauthTime); + if (credentialTtl > ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL) + credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL; + nc.networkId = nwid; nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; + nc.credentialTimeToLive = credentialTtl; nc.revision = _jI(network["revision"],0ULL); nc.issuedTo = identity.address(); if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; @@ -777,7 +797,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } if (_jB(network["private"],true)) { - CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); + CertificateOfMembership com(now,credentialTtl,nwid,identity.address()); if (com.sign(signingId)) { nc.com = com; } else { @@ -976,10 +996,24 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _initMember(member); try { - if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); if (b.count("activeBridge")) member["activeBridge"] = _jB(b["activeBridge"],false); if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = _jS(b["identity"],""); // allow identity to be populated only if not already known + if (b.count("authorized")) { + const bool newAuth = _jB(b["authorized"],false); + const bool oldAuth = _jB(member["authorized"],false); + if (newAuth != oldAuth) { + if (newAuth) { + member["authorized"] = true; + member["lastAuthorizedTime"] = now; + member["lastAuthorizedBy"] = "user"; + } else { + member["authorized"] = false; + member["lastDeauthorizedTime"] = now; + } + } + } + if (b.count("ipAssignments")) { auto ipa = b["ipAssignments"]; if (ipa.is_array()) { @@ -1476,6 +1510,8 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid nmi.allocatedIps.insert(mip); } } + } else { + nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(nm->second["lastDeauthorizedTime"],0ULL)); } } } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index e6b4bb59..3613e0ef 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -141,12 +141,13 @@ private: // This does lock _networkMemberCache_m struct _NetworkMemberInfo { - _NetworkMemberInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0) {} + _NetworkMemberInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} std::set
activeBridges; std::set allocatedIps; unsigned long authorizedMemberCount; unsigned long activeMemberCount; unsigned long totalMemberCount; + uint64_t mostRecentDeauthTime; }; void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); @@ -154,7 +155,10 @@ private: inline void _initMember(nlohmann::json &member) { if (!member.count("authorized")) member["authorized"] = false; - if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); + if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL; + if (!member.count("lastAuthorizedBy")) member["lastAuthorizedBy"] = ""; + if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL; + if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array(); if (!member.count("activeBridge")) member["activeBridge"] = false; if (!member.count("tags")) member["tags"] = nlohmann::json::array(); diff --git a/controller/README.md b/controller/README.md index 339adb31..189fcdbd 100644 --- a/controller/README.md +++ b/controller/README.md @@ -208,6 +208,9 @@ This returns an object containing all currently online members and the most rece | nwid | string | 16-digit network ID | no | | clock | integer | Current clock, ms since epoch | no | | authorized | boolean | Is member authorized? (for private networks) | YES | +| lastAuthorizedTime | integer | Time 'authorized' was last set to 'true' | no | +| lastAuthorizedBy | string | What last set 'authorized' to 'true'? | no | +| lastDeauthorizedTime | integer | Time 'authorized' was last set to 'false' | no | | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | | ipAssignments | array[string] | Managed IP address assignments | YES | diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 304111d6..2d7c2cb3 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -33,11 +33,6 @@ #include "Identity.hpp" #include "Utils.hpp" -/** - * Default window of time for certificate agreement - */ -#define ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA (ZT_NETWORK_AUTOCONF_DELAY * 5) - /** * Maximum number of qualifiers allowed in a COM (absolute max: 65535) */ diff --git a/node/Membership.hpp b/node/Membership.hpp index 92bd7ebf..a845b992 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -32,10 +32,10 @@ #include "NetworkConfig.hpp" // Expiration time for capability and tag cache -#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 4) +#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME 600000 // Expiration time for Memberships (used in Peer::clean()) -#define ZT_MEMBERSHIP_EXPIRATION_TIME (ZT_MEMBERSHIP_STATE_EXPIRATION_TIME * 4) +#define ZT_MEMBERSHIP_EXPIRATION_TIME (ZT_MEMBERSHIP_STATE_EXPIRATION_TIME * 2) namespace ZeroTier { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index a853d020..e1a4e302 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -40,6 +40,22 @@ #include "Dictionary.hpp" #include "Identity.hpp" +/** + * Default maximum credential TTL and maxDelta for COM timestamps + * + * The current value is two hours, providing ample time for a controller to + * experience fail-over, etc. + */ +#define ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL 7200000ULL + +/** + * Default minimum credential TTL and maxDelta for COM timestamps + * + * This is just slightly over three minutes and provides three retries for + * all currently online members to refresh. + */ +#define ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL 185000ULL + /** * Flag: allow passive bridging (experimental) */ -- cgit v1.2.3 From 5f4df0c6a99fa64011f09e218dd2249de10d0151 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 15:30:36 -0700 Subject: Controller cleanup and perf improvements. --- controller/EmbeddedNetworkController.cpp | 45 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 23 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 4db219c9..7b4ff6bf 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -419,6 +419,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( const std::string memberJP(_memberJP(nwid,identity.address(),true)); json member(_readJson(memberJP)); + _initMember(member); { std::string haveIdStr(_jS(member["identity"],"")); @@ -438,18 +439,10 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - _initMember(member); - - // Make sure these are always present no matter what, and increment member revision since we will always at least log something + // These are always the same, but make sure they are set member["id"] = identity.address().toString(); member["address"] = member["id"]; member["nwid"] = network["id"]; - member["lastModified"] = now; - { - auto revj = member["revision"]; - const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - member["revision"] = rev; - } // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; @@ -462,6 +455,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["authorized"] = true; member["lastAuthorizedTime"] = now; member["lastAuthorizedBy"] = authorizedBy; + member["lastModified"] = now; + auto revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } } else if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; @@ -482,6 +478,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["authorized"] = true; // tokens actually change member authorization state member["lastAuthorizedTime"] = now; member["lastAuthorizedBy"] = authorizedBy; + member["lastModified"] = now; + auto revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); break; } } @@ -555,14 +554,14 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); - auto v4AssignMode = network["v4AssignMode"]; - auto v6AssignMode = network["v6AssignMode"]; - auto ipAssignmentPools = network["ipAssignmentPools"]; - auto routes = network["routes"]; - auto rules = network["rules"]; - auto capabilities = network["capabilities"]; - auto memberCapabilities = member["capabilities"]; - auto memberTags = member["tags"]; + const json &v4AssignMode = network["v4AssignMode"]; + const json &v6AssignMode = network["v6AssignMode"]; + const json &ipAssignmentPools = network["ipAssignmentPools"]; + const json &routes = network["routes"]; + const json &rules = network["rules"]; + const json &capabilities = network["capabilities"]; + const json &memberCapabilities = member["capabilities"]; + const json &memberTags = member["tags"]; if (rules.is_array()) { for(unsigned long i=0;i 0)&&(capabilities.is_array())) { - std::map< uint64_t,json > capsById; + std::map< uint64_t,const json * > capsById; for(unsigned long i=0;i 0)) { + const json *cap = capsById[capId]; + if ((cap->is_object())&&(cap->size() > 0)) { ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; unsigned int caprc = 0; - auto caprj = cap["rules"]; + auto caprj = (*cap)["rules"]; if ((caprj.is_array())&&(caprj.size() > 0)) { for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) -- cgit v1.2.3 From 8d594f8b536a1e4acfe62223eb4bd939a8e04fe9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 16:05:10 -0700 Subject: cleanup --- controller/EmbeddedNetworkController.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7b4ff6bf..4e8d1aad 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1068,16 +1068,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["address"] = addrs; // legacy member["nwid"] = nwids; member["lastModified"] = now; - { - auto revj = member["revision"]; - const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - member["revision"] = rev; - } + auto revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); // Add non-persisted fields - member["clock"] = member["lastModified"]; + member["clock"] = now; responseBody = member.dump(2); responseContentType = "application/json"; -- cgit v1.2.3 From 8e3463d47a8e7565784f349f359ebe7f4a4d0e57 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 13:37:57 -0700 Subject: Add length limit to TEE and REDIRECT, and completely factor out old C json-parser to eliminate a dependency. --- AUTHORS.md | 8 +- controller/EmbeddedNetworkController.cpp | 18 +- ext/json-parser/AUTHORS | 20 - ext/json-parser/LICENSE | 26 - ext/json-parser/README.md | 97 --- ext/json-parser/json.c | 1012 ------------------------------ ext/json-parser/json.h | 283 --------- include/ZeroTierOne.h | 14 + make-freebsd.mk | 2 +- make-linux.mk | 6 - make-mac.mk | 2 +- node/Capability.hpp | 9 + node/Network.cpp | 6 +- node/Packet.hpp | 1 + one.cpp | 288 +++------ service/ControlPlane.cpp | 33 +- 16 files changed, 158 insertions(+), 1667 deletions(-) delete mode 100644 ext/json-parser/AUTHORS delete mode 100644 ext/json-parser/LICENSE delete mode 100644 ext/json-parser/README.md delete mode 100644 ext/json-parser/json.c delete mode 100644 ext/json-parser/json.h (limited to 'controller') diff --git a/AUTHORS.md b/AUTHORS.md index aa9e9111..90a64ef6 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -39,11 +39,11 @@ These are included in ext/ for platforms that do not have them available in comm * Home page: https://github.com/joyent/http-parser/ * License grant: MIT/Expat - * json-parser by James McLaughlin + * C++11 json (nlohmann/json) by Niels Lohmann - * Files: ext/json-parser/* - * Home page: https://github.com/udp/json-parser/ - * License grant: BSD attribution + * Files: ext/json/* + * Home page: https://github.com/nlohmann/json + * License grant: MIT * TunTapOSX by Mattias Nissler diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 4e8d1aad..0a4a17f8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -121,11 +121,15 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_ACTION_TEE: r["type"] = "ACTION_TEE"; - r["zt"] = Address(rule.v.zt).toString(); + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (uint64_t)rule.v.fwd.flags; + r["length"] = (uint64_t)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; - r["zt"] = Address(rule.v.zt).toString(); + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (uint64_t)rule.v.fwd.flags; + r["length"] = (uint64_t)rule.v.fwd.length; break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; @@ -235,7 +239,7 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) { if (r.is_object()) return false; - std::string t = r["type"]; + const std::string t(_jS(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); if (_jB(r["not"],false)) rule.t = 0x80; @@ -248,11 +252,15 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; diff --git a/ext/json-parser/AUTHORS b/ext/json-parser/AUTHORS deleted file mode 100644 index 6a5c799f..00000000 --- a/ext/json-parser/AUTHORS +++ /dev/null @@ -1,20 +0,0 @@ -All contributors arranged by first commit: - -James McLaughlin -Alex Gartrell -Peter Scott -Mathias Kaerlev -Emiel Mols -Czarek Tomczak -Nicholas Braden -Ivan Kozub -Árpád Goretity -Igor Gnatenko -Haïkel Guémar -Tobias Waldekranz -Patrick Donnelly -Wilmer van der Gaast -Jin Wei -François Cartegnie -Matthijs Boelstra - diff --git a/ext/json-parser/LICENSE b/ext/json-parser/LICENSE deleted file mode 100644 index 1aee375e..00000000 --- a/ext/json-parser/LICENSE +++ /dev/null @@ -1,26 +0,0 @@ - - Copyright (C) 2012, 2013 James McLaughlin et al. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - diff --git a/ext/json-parser/README.md b/ext/json-parser/README.md deleted file mode 100644 index e0b70b6d..00000000 --- a/ext/json-parser/README.md +++ /dev/null @@ -1,97 +0,0 @@ -Very low footprint JSON parser written in portable ANSI C. - -* BSD licensed with no dependencies (i.e. just drop the C file into your project) -* Never recurses or allocates more memory than it needs -* Very simple API with operator sugar for C++ - -[![Build Status](https://secure.travis-ci.org/udp/json-parser.png)](http://travis-ci.org/udp/json-parser) - -_Want to serialize? Check out [json-builder](https://github.com/udp/json-builder)!_ - -Installing ----------- - -There is now a makefile which will produce a libjsonparser static and dynamic library. However, this -is _not_ required to build json-parser, and the source files (`json.c` and `json.h`) should be happy -in any build system you already have in place. - - -API ---- - - json_value * json_parse (const json_char * json, - size_t length); - - json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error); - - void json_value_free (json_value *); - -The `type` field of `json_value` is one of: - -* `json_object` (see `u.object.length`, `u.object.values[x].name`, `u.object.values[x].value`) -* `json_array` (see `u.array.length`, `u.array.values`) -* `json_integer` (see `u.integer`) -* `json_double` (see `u.dbl`) -* `json_string` (see `u.string.ptr`, `u.string.length`) -* `json_boolean` (see `u.boolean`) -* `json_null` - - -Compile-Time Options --------------------- - - -DJSON_TRACK_SOURCE - -Stores the source location (line and column number) inside each `json_value`. - -This is useful for application-level error reporting. - - -Runtime Options ---------------- - - settings |= json_enable_comments; - -Enables C-style `// line` and `/* block */` comments. - - size_t value_extra - -The amount of space (if any) to allocate at the end of each `json_value`, in -order to give the application space to add metadata. - - void * (* mem_alloc) (size_t, int zero, void * user_data); - void (* mem_free) (void *, void * user_data); - -Custom allocator routines. If NULL, the default `malloc` and `free` will be used. - -The `user_data` pointer will be forwarded from `json_settings` to allow application -context to be passed. - - -Changes in version 1.1.0 ------------------------- - -* UTF-8 byte order marks are now skipped if present - -* Allows cross-compilation by honoring --host if given (@wkz) - -* Maximum size for error buffer is now exposed in header (@LB--) - -* GCC warning for `static` after `const` fixed (@batrick) - -* Optional support for C-style line and block comments added (@Jin-W-FS) - -* `name_length` field added to object values - -* It is now possible to retrieve the source line/column number of a parsed `json_value` when `JSON_TRACK_SOURCE` is enabled - -* The application may now extend `json_value` using the `value_extra` setting - -* Un-ambiguate pow call in the case of C++ overloaded pow (@fcartegnie) - -* Fix null pointer de-reference when a non-existing array is closed and no root value is present - - diff --git a/ext/json-parser/json.c b/ext/json-parser/json.c deleted file mode 100644 index 166cdcb6..00000000 --- a/ext/json-parser/json.c +++ /dev/null @@ -1,1012 +0,0 @@ -/* vim: set et ts=3 sw=3 sts=3 ft=c: - * - * Copyright (C) 2012, 2013, 2014 James McLaughlin et al. All rights reserved. - * https://github.com/udp/json-parser - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "json.h" - -#ifdef _MSC_VER - #ifndef _CRT_SECURE_NO_WARNINGS - #define _CRT_SECURE_NO_WARNINGS - #endif - #pragma warning(disable:4996) -#endif - -const struct _json_value json_value_none; - -#include -#include -#include -#include - -typedef unsigned int json_uchar; - -static unsigned char hex_value (json_char c) -{ - if (isdigit(c)) - return c - '0'; - - switch (c) { - case 'a': case 'A': return 0x0A; - case 'b': case 'B': return 0x0B; - case 'c': case 'C': return 0x0C; - case 'd': case 'D': return 0x0D; - case 'e': case 'E': return 0x0E; - case 'f': case 'F': return 0x0F; - default: return 0xFF; - } -} - -typedef struct -{ - unsigned long used_memory; - - unsigned int uint_max; - unsigned long ulong_max; - - json_settings settings; - int first_pass; - - const json_char * ptr; - unsigned int cur_line, cur_col; - -} json_state; - -static void * default_alloc (size_t size, int zero, void * user_data) -{ - return zero ? calloc (1, size) : malloc (size); -} - -static void default_free (void * ptr, void * user_data) -{ - free (ptr); -} - -static void * json_alloc (json_state * state, unsigned long size, int zero) -{ - if ((state->ulong_max - state->used_memory) < size) - return 0; - - if (state->settings.max_memory - && (state->used_memory += size) > state->settings.max_memory) - { - return 0; - } - - return state->settings.mem_alloc (size, zero, state->settings.user_data); -} - -static int new_value (json_state * state, - json_value ** top, json_value ** root, json_value ** alloc, - json_type type) -{ - json_value * value; - int values_size; - - if (!state->first_pass) - { - value = *top = *alloc; - *alloc = (*alloc)->_reserved.next_alloc; - - if (!*root) - *root = value; - - switch (value->type) - { - case json_array: - - if (value->u.array.length == 0) - break; - - if (! (value->u.array.values = (json_value **) json_alloc - (state, value->u.array.length * sizeof (json_value *), 0)) ) - { - return 0; - } - - value->u.array.length = 0; - break; - - case json_object: - - if (value->u.object.length == 0) - break; - - values_size = sizeof (*value->u.object.values) * value->u.object.length; - - if (! (value->u.object.values = (json_object_entry *) json_alloc - (state, values_size + ((unsigned long) value->u.object.values), 0)) ) - { - return 0; - } - - value->_reserved.object_mem = (*(char **) &value->u.object.values) + values_size; - - value->u.object.length = 0; - break; - - case json_string: - - if (! (value->u.string.ptr = (json_char *) json_alloc - (state, (value->u.string.length + 1) * sizeof (json_char), 0)) ) - { - return 0; - } - - value->u.string.length = 0; - break; - - default: - break; - }; - - return 1; - } - - if (! (value = (json_value *) json_alloc - (state, sizeof (json_value) + state->settings.value_extra, 1))) - { - return 0; - } - - if (!*root) - *root = value; - - value->type = type; - value->parent = *top; - - #ifdef JSON_TRACK_SOURCE - value->line = state->cur_line; - value->col = state->cur_col; - #endif - - if (*alloc) - (*alloc)->_reserved.next_alloc = value; - - *alloc = *top = value; - - return 1; -} - -#define whitespace \ - case '\n': ++ state.cur_line; state.cur_col = 0; \ - case ' ': case '\t': case '\r' - -#define string_add(b) \ - do { if (!state.first_pass) string [string_length] = b; ++ string_length; } while (0); - -#define line_and_col \ - state.cur_line, state.cur_col - -static const long - flag_next = 1 << 0, - flag_reproc = 1 << 1, - flag_need_comma = 1 << 2, - flag_seek_value = 1 << 3, - flag_escaped = 1 << 4, - flag_string = 1 << 5, - flag_need_colon = 1 << 6, - flag_done = 1 << 7, - flag_num_negative = 1 << 8, - flag_num_zero = 1 << 9, - flag_num_e = 1 << 10, - flag_num_e_got_sign = 1 << 11, - flag_num_e_negative = 1 << 12, - flag_line_comment = 1 << 13, - flag_block_comment = 1 << 14; - -json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error_buf) -{ - json_char error [json_error_max]; - const json_char * end; - json_value * top, * root, * alloc = 0; - json_state state = { 0 }; - long flags; - long num_digits = 0, num_e = 0; - json_int_t num_fraction = 0; - - /* Skip UTF-8 BOM - */ - if (length >= 3 && ((unsigned char) json [0]) == 0xEF - && ((unsigned char) json [1]) == 0xBB - && ((unsigned char) json [2]) == 0xBF) - { - json += 3; - length -= 3; - } - - error[0] = '\0'; - end = (json + length); - - memcpy (&state.settings, settings, sizeof (json_settings)); - - if (!state.settings.mem_alloc) - state.settings.mem_alloc = default_alloc; - - if (!state.settings.mem_free) - state.settings.mem_free = default_free; - - memset (&state.uint_max, 0xFF, sizeof (state.uint_max)); - memset (&state.ulong_max, 0xFF, sizeof (state.ulong_max)); - - state.uint_max -= 8; /* limit of how much can be added before next check */ - state.ulong_max -= 8; - - for (state.first_pass = 1; state.first_pass >= 0; -- state.first_pass) - { - json_uchar uchar; - unsigned char uc_b1, uc_b2, uc_b3, uc_b4; - json_char * string = 0; - unsigned int string_length = 0; - - top = root = 0; - flags = flag_seek_value; - - state.cur_line = 1; - - for (state.ptr = json ;; ++ state.ptr) - { - json_char b = (state.ptr == end ? 0 : *state.ptr); - - if (flags & flag_string) - { - if (!b) - { sprintf (error, "Unexpected EOF in string (at %d:%d)", line_and_col); - goto e_failed; - } - - if (string_length > state.uint_max) - goto e_overflow; - - if (flags & flag_escaped) - { - flags &= ~ flag_escaped; - - switch (b) - { - case 'b': string_add ('\b'); break; - case 'f': string_add ('\f'); break; - case 'n': string_add ('\n'); break; - case 'r': string_add ('\r'); break; - case 't': string_add ('\t'); break; - case 'u': - - if (end - state.ptr < 4 || - (uc_b1 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b2 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b3 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b4 = hex_value (*++ state.ptr)) == 0xFF) - { - sprintf (error, "Invalid character value `%c` (at %d:%d)", b, line_and_col); - goto e_failed; - } - - uc_b1 = (uc_b1 << 4) | uc_b2; - uc_b2 = (uc_b3 << 4) | uc_b4; - uchar = (uc_b1 << 8) | uc_b2; - - if ((uchar & 0xF800) == 0xD800) { - json_uchar uchar2; - - if (end - state.ptr < 6 || (*++ state.ptr) != '\\' || (*++ state.ptr) != 'u' || - (uc_b1 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b2 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b3 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b4 = hex_value (*++ state.ptr)) == 0xFF) - { - sprintf (error, "Invalid character value `%c` (at %d:%d)", b, line_and_col); - goto e_failed; - } - - uc_b1 = (uc_b1 << 4) | uc_b2; - uc_b2 = (uc_b3 << 4) | uc_b4; - uchar2 = (uc_b1 << 8) | uc_b2; - - uchar = 0x010000 | ((uchar & 0x3FF) << 10) | (uchar2 & 0x3FF); - } - - if (sizeof (json_char) >= sizeof (json_uchar) || (uchar <= 0x7F)) - { - string_add ((json_char) uchar); - break; - } - - if (uchar <= 0x7FF) - { - if (state.first_pass) - string_length += 2; - else - { string [string_length ++] = 0xC0 | (uchar >> 6); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - } - - if (uchar <= 0xFFFF) { - if (state.first_pass) - string_length += 3; - else - { string [string_length ++] = 0xE0 | (uchar >> 12); - string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - } - - if (state.first_pass) - string_length += 4; - else - { string [string_length ++] = 0xF0 | (uchar >> 18); - string [string_length ++] = 0x80 | ((uchar >> 12) & 0x3F); - string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - - default: - string_add (b); - }; - - continue; - } - - if (b == '\\') - { - flags |= flag_escaped; - continue; - } - - if (b == '"') - { - if (!state.first_pass) - string [string_length] = 0; - - flags &= ~ flag_string; - string = 0; - - switch (top->type) - { - case json_string: - - top->u.string.length = string_length; - flags |= flag_next; - - break; - - case json_object: - - if (state.first_pass) - (*(json_char **) &top->u.object.values) += string_length + 1; - else - { - top->u.object.values [top->u.object.length].name - = (json_char *) top->_reserved.object_mem; - - top->u.object.values [top->u.object.length].name_length - = string_length; - - (*(json_char **) &top->_reserved.object_mem) += string_length + 1; - } - - flags |= flag_seek_value | flag_need_colon; - continue; - - default: - break; - }; - } - else - { - string_add (b); - continue; - } - } - - if (state.settings.settings & json_enable_comments) - { - if (flags & (flag_line_comment | flag_block_comment)) - { - if (flags & flag_line_comment) - { - if (b == '\r' || b == '\n' || !b) - { - flags &= ~ flag_line_comment; - -- state.ptr; /* so null can be reproc'd */ - } - - continue; - } - - if (flags & flag_block_comment) - { - if (!b) - { sprintf (error, "%d:%d: Unexpected EOF in block comment", line_and_col); - goto e_failed; - } - - if (b == '*' && state.ptr < (end - 1) && state.ptr [1] == '/') - { - flags &= ~ flag_block_comment; - ++ state.ptr; /* skip closing sequence */ - } - - continue; - } - } - else if (b == '/') - { - if (! (flags & (flag_seek_value | flag_done)) && top->type != json_object) - { sprintf (error, "%d:%d: Comment not allowed here", line_and_col); - goto e_failed; - } - - if (++ state.ptr == end) - { sprintf (error, "%d:%d: EOF unexpected", line_and_col); - goto e_failed; - } - - switch (b = *state.ptr) - { - case '/': - flags |= flag_line_comment; - continue; - - case '*': - flags |= flag_block_comment; - continue; - - default: - sprintf (error, "%d:%d: Unexpected `%c` in comment opening sequence", line_and_col, b); - goto e_failed; - }; - } - } - - if (flags & flag_done) - { - if (!b) - break; - - switch (b) - { - whitespace: - continue; - - default: - - sprintf (error, "%d:%d: Trailing garbage: `%c`", - state.cur_line, state.cur_col, b); - - goto e_failed; - }; - } - - if (flags & flag_seek_value) - { - switch (b) - { - whitespace: - continue; - - case ']': - - if (top && top->type == json_array) - flags = (flags & ~ (flag_need_comma | flag_seek_value)) | flag_next; - else - { sprintf (error, "%d:%d: Unexpected ]", line_and_col); - goto e_failed; - } - - break; - - default: - - if (flags & flag_need_comma) - { - if (b == ',') - { flags &= ~ flag_need_comma; - continue; - } - else - { - sprintf (error, "%d:%d: Expected , before %c", - state.cur_line, state.cur_col, b); - - goto e_failed; - } - } - - if (flags & flag_need_colon) - { - if (b == ':') - { flags &= ~ flag_need_colon; - continue; - } - else - { - sprintf (error, "%d:%d: Expected : before %c", - state.cur_line, state.cur_col, b); - - goto e_failed; - } - } - - flags &= ~ flag_seek_value; - - switch (b) - { - case '{': - - if (!new_value (&state, &top, &root, &alloc, json_object)) - goto e_alloc_failure; - - continue; - - case '[': - - if (!new_value (&state, &top, &root, &alloc, json_array)) - goto e_alloc_failure; - - flags |= flag_seek_value; - continue; - - case '"': - - if (!new_value (&state, &top, &root, &alloc, json_string)) - goto e_alloc_failure; - - flags |= flag_string; - - string = top->u.string.ptr; - string_length = 0; - - continue; - - case 't': - - if ((end - state.ptr) < 3 || *(++ state.ptr) != 'r' || - *(++ state.ptr) != 'u' || *(++ state.ptr) != 'e') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_boolean)) - goto e_alloc_failure; - - top->u.boolean = 1; - - flags |= flag_next; - break; - - case 'f': - - if ((end - state.ptr) < 4 || *(++ state.ptr) != 'a' || - *(++ state.ptr) != 'l' || *(++ state.ptr) != 's' || - *(++ state.ptr) != 'e') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_boolean)) - goto e_alloc_failure; - - flags |= flag_next; - break; - - case 'n': - - if ((end - state.ptr) < 3 || *(++ state.ptr) != 'u' || - *(++ state.ptr) != 'l' || *(++ state.ptr) != 'l') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_null)) - goto e_alloc_failure; - - flags |= flag_next; - break; - - default: - - if (isdigit (b) || b == '-') - { - if (!new_value (&state, &top, &root, &alloc, json_integer)) - goto e_alloc_failure; - - if (!state.first_pass) - { - while (isdigit (b) || b == '+' || b == '-' - || b == 'e' || b == 'E' || b == '.') - { - if ( (++ state.ptr) == end) - { - b = 0; - break; - } - - b = *state.ptr; - } - - flags |= flag_next | flag_reproc; - break; - } - - flags &= ~ (flag_num_negative | flag_num_e | - flag_num_e_got_sign | flag_num_e_negative | - flag_num_zero); - - num_digits = 0; - num_fraction = 0; - num_e = 0; - - if (b != '-') - { - flags |= flag_reproc; - break; - } - - flags |= flag_num_negative; - continue; - } - else - { sprintf (error, "%d:%d: Unexpected %c when seeking value", line_and_col, b); - goto e_failed; - } - }; - }; - } - else - { - switch (top->type) - { - case json_object: - - switch (b) - { - whitespace: - continue; - - case '"': - - if (flags & flag_need_comma) - { sprintf (error, "%d:%d: Expected , before \"", line_and_col); - goto e_failed; - } - - flags |= flag_string; - - string = (json_char *) top->_reserved.object_mem; - string_length = 0; - - break; - - case '}': - - flags = (flags & ~ flag_need_comma) | flag_next; - break; - - case ',': - - if (flags & flag_need_comma) - { - flags &= ~ flag_need_comma; - break; - } - - default: - sprintf (error, "%d:%d: Unexpected `%c` in object", line_and_col, b); - goto e_failed; - }; - - break; - - case json_integer: - case json_double: - - if (isdigit (b)) - { - ++ num_digits; - - if (top->type == json_integer || flags & flag_num_e) - { - if (! (flags & flag_num_e)) - { - if (flags & flag_num_zero) - { sprintf (error, "%d:%d: Unexpected `0` before `%c`", line_and_col, b); - goto e_failed; - } - - if (num_digits == 1 && b == '0') - flags |= flag_num_zero; - } - else - { - flags |= flag_num_e_got_sign; - num_e = (num_e * 10) + (b - '0'); - continue; - } - - top->u.integer = (top->u.integer * 10) + (b - '0'); - continue; - } - - num_fraction = (num_fraction * 10) + (b - '0'); - continue; - } - - if (b == '+' || b == '-') - { - if ( (flags & flag_num_e) && !(flags & flag_num_e_got_sign)) - { - flags |= flag_num_e_got_sign; - - if (b == '-') - flags |= flag_num_e_negative; - - continue; - } - } - else if (b == '.' && top->type == json_integer) - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit before `.`", line_and_col); - goto e_failed; - } - - top->type = json_double; - top->u.dbl = (double) top->u.integer; - - num_digits = 0; - continue; - } - - if (! (flags & flag_num_e)) - { - if (top->type == json_double) - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit after `.`", line_and_col); - goto e_failed; - } - - top->u.dbl += ((double) num_fraction) / (pow (10.0, (double) num_digits)); - } - - if (b == 'e' || b == 'E') - { - flags |= flag_num_e; - - if (top->type == json_integer) - { - top->type = json_double; - top->u.dbl = (double) top->u.integer; - } - - num_digits = 0; - flags &= ~ flag_num_zero; - - continue; - } - } - else - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit after `e`", line_and_col); - goto e_failed; - } - - top->u.dbl *= pow (10.0, (double) - (flags & flag_num_e_negative ? - num_e : num_e)); - } - - if (flags & flag_num_negative) - { - if (top->type == json_integer) - top->u.integer = - top->u.integer; - else - top->u.dbl = - top->u.dbl; - } - - flags |= flag_next | flag_reproc; - break; - - default: - break; - }; - } - - if (flags & flag_reproc) - { - flags &= ~ flag_reproc; - -- state.ptr; - } - - if (flags & flag_next) - { - flags = (flags & ~ flag_next) | flag_need_comma; - - if (!top->parent) - { - /* root value done */ - - flags |= flag_done; - continue; - } - - if (top->parent->type == json_array) - flags |= flag_seek_value; - - if (!state.first_pass) - { - json_value * parent = top->parent; - - switch (parent->type) - { - case json_object: - - parent->u.object.values - [parent->u.object.length].value = top; - - break; - - case json_array: - - parent->u.array.values - [parent->u.array.length] = top; - - break; - - default: - break; - }; - } - - if ( (++ top->parent->u.array.length) > state.uint_max) - goto e_overflow; - - top = top->parent; - - continue; - } - } - - alloc = root; - } - - return root; - -e_unknown_value: - - sprintf (error, "%d:%d: Unknown value", line_and_col); - goto e_failed; - -e_alloc_failure: - - strcpy (error, "Memory allocation failure"); - goto e_failed; - -e_overflow: - - sprintf (error, "%d:%d: Too long (caught overflow)", line_and_col); - goto e_failed; - -e_failed: - - if (error_buf) - { - if (*error) - strcpy (error_buf, error); - else - strcpy (error_buf, "Unknown error"); - } - - if (state.first_pass) - alloc = root; - - while (alloc) - { - top = alloc->_reserved.next_alloc; - state.settings.mem_free (alloc, state.settings.user_data); - alloc = top; - } - - if (!state.first_pass) - json_value_free_ex (&state.settings, root); - - return 0; -} - -json_value * json_parse (const json_char * json, size_t length) -{ - json_settings settings = { 0 }; - return json_parse_ex (&settings, json, length, 0); -} - -void json_value_free_ex (json_settings * settings, json_value * value) -{ - json_value * cur_value; - - if (!value) - return; - - value->parent = 0; - - while (value) - { - switch (value->type) - { - case json_array: - - if (!value->u.array.length) - { - settings->mem_free (value->u.array.values, settings->user_data); - break; - } - - value = value->u.array.values [-- value->u.array.length]; - continue; - - case json_object: - - if (!value->u.object.length) - { - settings->mem_free (value->u.object.values, settings->user_data); - break; - } - - value = value->u.object.values [-- value->u.object.length].value; - continue; - - case json_string: - - settings->mem_free (value->u.string.ptr, settings->user_data); - break; - - default: - break; - }; - - cur_value = value; - value = value->parent; - settings->mem_free (cur_value, settings->user_data); - } -} - -void json_value_free (json_value * value) -{ - json_settings settings = { 0 }; - settings.mem_free = default_free; - json_value_free_ex (&settings, value); -} - diff --git a/ext/json-parser/json.h b/ext/json-parser/json.h deleted file mode 100644 index f6549ec4..00000000 --- a/ext/json-parser/json.h +++ /dev/null @@ -1,283 +0,0 @@ - -/* vim: set et ts=3 sw=3 sts=3 ft=c: - * - * Copyright (C) 2012, 2013, 2014 James McLaughlin et al. All rights reserved. - * https://github.com/udp/json-parser - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef _JSON_H -#define _JSON_H - -#ifndef json_char - #define json_char char -#endif - -#ifndef json_int_t - #ifndef _MSC_VER - #include - #define json_int_t int64_t - #else - #define json_int_t __int64 - #endif -#endif - -#include - -#ifdef __cplusplus - - #include - - extern "C" - { - -#endif - -typedef struct -{ - unsigned long max_memory; - int settings; - - /* Custom allocator support (leave null to use malloc/free) - */ - - void * (* mem_alloc) (size_t, int zero, void * user_data); - void (* mem_free) (void *, void * user_data); - - void * user_data; /* will be passed to mem_alloc and mem_free */ - - size_t value_extra; /* how much extra space to allocate for values? */ - -} json_settings; - -#define json_enable_comments 0x01 - -typedef enum -{ - json_none, - json_object, - json_array, - json_integer, - json_double, - json_string, - json_boolean, - json_null - -} json_type; - -extern const struct _json_value json_value_none; - -typedef struct _json_object_entry -{ - json_char * name; - unsigned int name_length; - - struct _json_value * value; - -} json_object_entry; - -typedef struct _json_value -{ - struct _json_value * parent; - - json_type type; - - union - { - int boolean; - json_int_t integer; - double dbl; - - struct - { - unsigned int length; - json_char * ptr; /* null terminated */ - - } string; - - struct - { - unsigned int length; - - json_object_entry * values; - - #if defined(__cplusplus) && __cplusplus >= 201103L - decltype(values) begin () const - { return values; - } - decltype(values) end () const - { return values + length; - } - #endif - - } object; - - struct - { - unsigned int length; - struct _json_value ** values; - - #if defined(__cplusplus) && __cplusplus >= 201103L - decltype(values) begin () const - { return values; - } - decltype(values) end () const - { return values + length; - } - #endif - - } array; - - } u; - - union - { - struct _json_value * next_alloc; - void * object_mem; - - } _reserved; - - #ifdef JSON_TRACK_SOURCE - - /* Location of the value in the source JSON - */ - unsigned int line, col; - - #endif - - - /* Some C++ operator sugar */ - - #ifdef __cplusplus - - public: - - inline _json_value () - { memset (this, 0, sizeof (_json_value)); - } - - inline const struct _json_value &operator [] (int index) const - { - if (type != json_array || index < 0 - || ((unsigned int) index) >= u.array.length) - { - return json_value_none; - } - - return *u.array.values [index]; - } - - inline const struct _json_value &operator [] (const char * index) const - { - if (type != json_object) - return json_value_none; - - for (unsigned int i = 0; i < u.object.length; ++ i) - if (!strcmp (u.object.values [i].name, index)) - return *u.object.values [i].value; - - return json_value_none; - } - - inline operator const char * () const - { - switch (type) - { - case json_string: - return u.string.ptr; - - default: - return ""; - }; - } - - inline operator json_int_t () const - { - switch (type) - { - case json_integer: - return u.integer; - - case json_double: - return (json_int_t) u.dbl; - - default: - return 0; - }; - } - - inline operator bool () const - { - if (type != json_boolean) - return false; - - return u.boolean != 0; - } - - inline operator double () const - { - switch (type) - { - case json_integer: - return (double) u.integer; - - case json_double: - return u.dbl; - - default: - return 0; - }; - } - - #endif - -} json_value; - -json_value * json_parse (const json_char * json, - size_t length); - -#define json_error_max 128 -json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error); - -void json_value_free (json_value *); - - -/* Not usually necessary, unless you used a custom mem_alloc and now want to - * use a custom mem_free. - */ -void json_value_free_ex (json_settings * settings, - json_value *); - - -#ifdef __cplusplus - } /* extern "C" */ -#endif - -#endif - - diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index aa7ecc2c..5864e346 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -468,6 +468,11 @@ enum ZT_VirtualNetworkType ZT_NETWORK_TYPE_PUBLIC = 1 }; +/* + - TEE : should use a field to indicate how many bytes of each packet max are TEE'd + - Controller : web hooks for auth, optional required re-auth? or auth for a period of time? auto-expiring auth? +*/ + /** * The type of a virtual network rules table entry * @@ -721,6 +726,15 @@ typedef struct uint32_t id; uint32_t value; } tag; + + /** + * Destinations for TEE and REDIRECT + */ + struct { + uint64_t address; + uint32_t flags; + uint16_t length; + } fwd; } v; } ZT_VirtualNetworkRule; diff --git a/make-freebsd.mk b/make-freebsd.mk index e7bd9fd2..cb9a2e6d 100644 --- a/make-freebsd.mk +++ b/make-freebsd.mk @@ -6,7 +6,7 @@ DEFS= LIBS= include objects.mk -OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/json-parser/json.o ext/http-parser/http_parser.o +OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o # "make official" is a shortcut for this ifeq ($(ZT_OFFICIAL_RELEASE),1) diff --git a/make-linux.mk b/make-linux.mk index fe9ecad8..1fc27bde 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -51,12 +51,6 @@ else LDLIBS+=-lhttp_parser DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER endif -ifeq ($(wildcard /usr/include/json-parser/json.h),) - OBJS+=ext/json-parser/json.o -else - LDLIBS+=-ljsonparser - DEFS+=-DZT_USE_SYSTEM_JSON_PARSER -endif ifeq ($(ZT_USE_MINIUPNPC),1) OBJS+=osdep/PortMapper.o diff --git a/make-mac.mk b/make-mac.mk index f00f8d6b..ee90ae4c 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -11,7 +11,7 @@ LIBS= ARCH_FLAGS=-arch x86_64 include objects.mk -OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/json-parser/json.o ext/http-parser/http_parser.o +OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o # Disable codesign since open source users will not have ZeroTier's certs CODESIGN=echo diff --git a/node/Capability.hpp b/node/Capability.hpp index b0620891..0b352725 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -181,6 +181,11 @@ public: break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: + b.append((uint8_t)14); + b.append((uint64_t)rules[i].v.fwd.address); + b.append((uint32_t)rules[i].v.fwd.flags); + b.append((uint16_t)rules[i].v.fwd.length); + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: b.append((uint8_t)5); @@ -266,6 +271,10 @@ public: break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: + rules[ruleCount].v.fwd.address = b.template at(p); + rules[ruleCount].v.fwd.flags = b.template at(p + 8); + rules[ruleCount].v.fwd.length = b.template at(p + 12); + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: rules[ruleCount].v.zt = Address(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); diff --git a/node/Network.cpp b/node/Network.cpp index 1319df4e..e12dd027 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -154,13 +154,13 @@ static int _doZtFilter( break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { - Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME); + Packet outp(Address(rules[rn].v.fwd.address),RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(nconf.networkId); - outp.append((uint8_t)((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02)); + outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); - outp.append(frameData,frameLen); + outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); outp.compress(); RR->sw->send(outp,true); diff --git a/node/Packet.hpp b/node/Packet.hpp index 0a5d3fec..570bace9 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -657,6 +657,7 @@ public: * 0x01 - Certificate of network membership attached (DEPRECATED) * 0x02 - Packet is a TEE'd packet * 0x04 - Packet is a REDIRECT'ed packet + * 0x08 - TEE/REDIRECT'ed packet is on inbound side of connection * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we diff --git a/one.cpp b/one.cpp index 9f7a0a29..f3442933 100644 --- a/one.cpp +++ b/one.cpp @@ -48,16 +48,12 @@ #include #include +#include +#include #include "version.h" #include "include/ZeroTierOne.h" -#ifdef ZT_USE_SYSTEM_JSON_PARSER -#include -#else -#include "ext/json-parser/json.h" -#endif - #include "node/Identity.hpp" #include "node/CertificateOfMembership.hpp" #include "node/Utils.hpp" @@ -68,6 +64,8 @@ #include "service/OneService.hpp" +#include "ext/json/json.hpp" + #define ZT_PID_PATH "zerotier-one.pid" using namespace ZeroTier; @@ -283,221 +281,135 @@ static int cli(int argc,char **argv) return 1; } } else if ((command == "info")||(command == "status")) { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/status", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/status",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - bool good = false; - if (j) { - if (j->type == json_object) { - const char *address = (const char *)0; - bool online = false; - const char *version = (const char *)0; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(j->u.object.values[k].name,"address"))&&(j->u.object.values[k].value->type == json_string)) - address = j->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(j->u.object.values[k].name,"version"))&&(j->u.object.values[k].value->type == json_string)) - version = j->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(j->u.object.values[k].name,"online"))&&(j->u.object.values[k].value->type == json_boolean)) - online = (j->u.object.values[k].value->u.boolean != 0); - } - if ((address)&&(version)) { - printf("200 info %s %s %s" ZT_EOL_S,address,(online ? "ONLINE" : "OFFLINE"),version); - good = true; - } - } - json_value_free(j); - } - if (good) { - return 0; - } else { - printf("%u %s invalid JSON response" ZT_EOL_S,scode,command.c_str()); - return 1; - } + if (j.is_object()) + out << "200 info " << j["address"].get() << " " << j["version"].get() << " " << ((j["tcpFallbackActive"]) ? "TUNNELED" : ((j["online"]) ? "ONLINE" : "OFFLINE")) << ZT_EOL_S; } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; } } else if (command == "listpeers") { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/peer", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/peer",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - printf("200 listpeers " ZT_EOL_S); - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - if (j) { - if (j->type == json_array) { - for(unsigned int p=0;pu.array.length;++p) { - json_value *jp = j->u.array.values[p]; - if (jp->type == json_object) { - const char *address = (const char *)0; - std::string paths; - int64_t latency = 0; - int64_t versionMajor = -1,versionMinor = -1,versionRev = -1; - const char *role = (const char *)0; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(jp->u.object.values[k].name,"address"))&&(jp->u.object.values[k].value->type == json_string)) - address = jp->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jp->u.object.values[k].name,"versionMajor"))&&(jp->u.object.values[k].value->type == json_integer)) - versionMajor = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"versionMinor"))&&(jp->u.object.values[k].value->type == json_integer)) - versionMinor = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"versionRev"))&&(jp->u.object.values[k].value->type == json_integer)) - versionRev = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"role"))&&(jp->u.object.values[k].value->type == json_string)) - role = jp->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jp->u.object.values[k].name,"latency"))&&(jp->u.object.values[k].value->type == json_integer)) - latency = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"paths"))&&(jp->u.object.values[k].value->type == json_array)) { - for(unsigned int pp=0;ppu.object.values[k].value->u.array.length;++pp) { - json_value *jpath = jp->u.object.values[k].value->u.array.values[pp]; - if (jpath->type == json_object) { - const char *paddr = (const char *)0; - int64_t lastSend = 0; - int64_t lastReceive = 0; - bool preferred = false; - bool active = false; - for(unsigned int kk=0;kku.object.length;++kk) { - if ((!strcmp(jpath->u.object.values[kk].name,"address"))&&(jpath->u.object.values[kk].value->type == json_string)) - paddr = jpath->u.object.values[kk].value->u.string.ptr; - else if ((!strcmp(jpath->u.object.values[kk].name,"lastSend"))&&(jpath->u.object.values[kk].value->type == json_integer)) - lastSend = jpath->u.object.values[kk].value->u.integer; - else if ((!strcmp(jpath->u.object.values[kk].name,"lastReceive"))&&(jpath->u.object.values[kk].value->type == json_integer)) - lastReceive = jpath->u.object.values[kk].value->u.integer; - else if ((!strcmp(jpath->u.object.values[kk].name,"preferred"))&&(jpath->u.object.values[kk].value->type == json_boolean)) - preferred = (jpath->u.object.values[kk].value->u.boolean != 0); - else if ((!strcmp(jpath->u.object.values[kk].name,"active"))&&(jpath->u.object.values[kk].value->type == json_boolean)) - active = (jpath->u.object.values[kk].value->u.boolean != 0); - } - if ((paddr)&&(active)) { - int64_t now = (int64_t)OSUtils::now(); - if (lastSend > 0) - lastSend = now - lastSend; - if (lastReceive > 0) - lastReceive = now - lastReceive; - char pathtmp[256]; - Utils::snprintf(pathtmp,sizeof(pathtmp),"%s;%lld;%lld;%s", - paddr, - lastSend, - lastReceive, - (preferred ? "preferred" : "active")); - if (paths.length()) - paths.push_back(','); - paths.append(pathtmp); - } - } - } - } - } - if ((address)&&(role)) { - char verstr[64]; - if ((versionMajor >= 0)&&(versionMinor >= 0)&&(versionRev >= 0)) - Utils::snprintf(verstr,sizeof(verstr),"%lld.%lld.%lld",versionMajor,versionMinor,versionRev); - else { - verstr[0] = '-'; - verstr[1] = (char)0; - } - printf("200 listpeers %s %s %lld %s %s" ZT_EOL_S,address,(paths.length()) ? paths.c_str() : "-",(long long)latency,verstr,role); + out << "200 listpeers " << ZT_EOL_S; + if (j.is_array()) { + for(unsigned long k=0;k= 0) { + Utils::snprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + } else { + ver[0] = '-'; + ver[1] = (char)0; + } + out << "200 listpeers " << p["address"].get() << " " << bestPath << " " << p["latency"] << " " << ver << " " << p["role"].get() << ZT_EOL_S; } - json_value_free(j); } - return 0; } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; } } else if (command == "listnetworks") { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/network", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/network",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - printf("200 listnetworks " ZT_EOL_S); - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - if (j) { - if (j->type == json_array) { - for(unsigned int p=0;pu.array.length;++p) { - json_value *jn = j->u.array.values[p]; - if (jn->type == json_object) { - const char *nwid = (const char *)0; - const char *name = ""; - const char *mac = (const char *)0; - const char *status = (const char *)0; - const char *type = (const char *)0; - const char *portDeviceName = ""; - std::string ips; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(jn->u.object.values[k].name,"nwid"))&&(jn->u.object.values[k].value->type == json_string)) - nwid = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"name"))&&(jn->u.object.values[k].value->type == json_string)) - name = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"mac"))&&(jn->u.object.values[k].value->type == json_string)) - mac = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"status"))&&(jn->u.object.values[k].value->type == json_string)) - status = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"type"))&&(jn->u.object.values[k].value->type == json_string)) - type = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"portDeviceName"))&&(jn->u.object.values[k].value->type == json_string)) - portDeviceName = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"assignedAddresses"))&&(jn->u.object.values[k].value->type == json_array)) { - for(unsigned int a=0;au.object.values[k].value->u.array.length;++a) { - json_value *aa = jn->u.object.values[k].value->u.array.values[a]; - if (aa->type == json_string) { - if (ips.length()) - ips.push_back(','); - ips.append(aa->u.string.ptr); - } - } + out << "200 listnetworks " << ZT_EOL_S; + if (j.is_array()) { + for(unsigned long i=0;i 0) aa.push_back(','); + aa.append(addr); } } - if ((nwid)&&(mac)&&(status)&&(type)) { - printf("200 listnetworks %s %s %s %s %s %s %s" ZT_EOL_S, - nwid, - (((name)&&(name[0])) ? name : "-"), - mac, - status, - type, - (((portDeviceName)&&(portDeviceName[0])) ? portDeviceName : "-"), - ((ips.length() > 0) ? ips.c_str() : "-")); - } } + if (aa.length() == 0) aa = "-"; + out << "200 listnetworks " << n["nwid"].get() << " " << n["name"].get() << " " << n["mac"].get() << " " << n["status"].get() << " " << n["type"].get() << " " << n["portDeviceName"].get() << " " << aa << ZT_EOL_S; } } - json_value_free(j); } } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 7aa757a9..5ed6b8b7 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -28,11 +28,7 @@ #include "../ext/http-parser/http_parser.h" #endif -#ifdef ZT_USE_SYSTEM_JSON_PARSER -#include -#else -#include "../ext/json-parser/json.h" -#endif +#include "../ext/json/json.hpp" #include "../controller/EmbeddedNetworkController.hpp" @@ -519,23 +515,18 @@ unsigned int ControlPlane::handleRequest( OneService::NetworkSettings localSettings; _svc->getNetworkSettings(nws->networks[i].nwid,localSettings); - json_value *j = json_parse(body.c_str(),body.length()); - if (j) { - if (j->type == json_object) { - for(unsigned int k=0;ku.object.length;++k) { - if (!strcmp(j->u.object.values[k].name,"allowManaged")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowManaged = (j->u.object.values[k].value->u.boolean != 0); - } else if (!strcmp(j->u.object.values[k].name,"allowGlobal")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowGlobal = (j->u.object.values[k].value->u.boolean != 0); - } else if (!strcmp(j->u.object.values[k].name,"allowDefault")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowDefault = (j->u.object.values[k].value->u.boolean != 0); - } - } + try { + nlohmann::json j(nlohmann::json::parse(body)); + if (j.is_object()) { + auto allowManaged = j["allowManaged"]; + if (allowManaged.is_boolean()) localSettings.allowManaged = (bool)allowManaged; + auto allowGlobal = j["allowGlobal"]; + if (allowGlobal.is_boolean()) localSettings.allowGlobal = (bool)allowGlobal; + auto allowDefault = j["allowDefault"]; + if (allowDefault.is_boolean()) localSettings.allowDefault = (bool)allowDefault; } - json_value_free(j); + } catch ( ... ) { + // discard invalid JSON } _svc->setNetworkSettings(nws->networks[i].nwid,localSettings); -- cgit v1.2.3 From ccea3d04d63e39a020e140b348aa87e272747c7e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 14:28:16 -0700 Subject: Push NETWORK_CONFIG_REFRESH on POSTs to /member/... in controller. --- controller/EmbeddedNetworkController.cpp | 2 ++ include/ZeroTierOne.h | 21 +++++++++++++++++++++ node/Network.cpp | 7 +++++++ node/Network.hpp | 5 +++++ node/Node.cpp | 32 ++++++++++++++++++++++++++++++++ node/Node.hpp | 1 + 6 files changed, 68 insertions(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0a4a17f8..9cf43b79 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1081,6 +1081,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); + _node->pushNetworkRefresh(address,nwid,(const uint64_t *)0,(const uint64_t *)0,0); + // Add non-persisted fields member["clock"] = now; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 5864e346..906edef2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1853,6 +1853,27 @@ enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,v */ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test); +/** + * Push a network refresh + * + * This is used by network controller implementations to send a + * NETWORK_CONFIG_REFRESH message to tell a node to refresh its + * config and to optionally push one or more credential timestamp + * blacklist thresholds for members of the network. + * + * Code outside a controller implementation will have no use for + * this as these messages are ignored if they do not come from a + * controller. + * + * @param node Node instance + * @param dest ZeroTier address of destination to which to send NETWORK_CONFIG_REFRESH + * @param nwid Network ID + * @param blacklistAddresses Array of ZeroTier addresses of network members to set timestamp blacklists for + * @param blacklistBeforeTimestamps Timestamps before which to blacklist credentials for each corresponding address in blacklistAddresses[] + * @param blacklistCount Size of blacklistAddresses[] and blacklistBeforeTimestamps[] + */ +void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); + /** * Initialize cluster operation * diff --git a/node/Network.cpp b/node/Network.cpp index e12dd027..97341ee2 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -402,6 +402,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : _portInitialized(false), _inboundConfigPacketId(0), _lastConfigUpdate(0), + _lastRequestedConfiguration(0), _destroyed(false), _netconfFailure(NETCONF_FAILURE_NONE), _portError(0) @@ -691,6 +692,12 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d void Network::requestConfiguration() { + // Sanity limit: do not request more often than once per second + const uint64_t now = RR->node->now(); + if ((now - _lastRequestedConfiguration) < 1000ULL) + return; + _lastRequestedConfiguration = RR->node->now(); + Dictionary rmd; rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION); diff --git a/node/Network.hpp b/node/Network.hpp index 37154dc7..382b16aa 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -249,6 +249,10 @@ public: /** * Causes this network to request an updated configuration from its master node now + * + * There is a circuit breaker here to prevent this from being done more often + * than once per second. This is to prevent things like NETWORK_CONFIG_REFRESH + * from causing multiple requests. */ void requestConfiguration(); @@ -442,6 +446,7 @@ private: NetworkConfig _config; volatile uint64_t _lastConfigUpdate; + volatile uint64_t _lastRequestedConfiguration; volatile bool _destroyed; diff --git a/node/Node.cpp b/node/Node.cpp index 4da79347..ff564eee 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -548,6 +548,31 @@ void Node::circuitTestEnd(ZT_CircuitTest *test) } } +void Node::pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) +{ + Packet outp(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); + outp.append(nwid); + outp.addSize(2); + unsigned int c = 0; + for(unsigned int i=0;i= ZT_PROTO_MAX_PACKET_LENGTH) { + outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); + RR->sw->send(outp,true); + outp = Packet(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); + outp.append(nwid); + outp.addSize(2); + c = 0; + } + Address(blacklistAddresses[i]).appendTo(outp); + outp.append(blacklistBeforeTimestamps[i]); + ++c; + } + if (c > 0) { + outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); + RR->sw->send(outp,true); + } +} + ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -935,6 +960,13 @@ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test) } catch ( ... ) {} } +void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) +{ + try { + reinterpret_cast(node)->pushNetworkRefresh(dest,nwid,blacklistAddresses,blacklistBeforeTimestamps,blacklistCount); + } catch ( ... ) {} +} + enum ZT_ResultCode ZT_Node_clusterInit( ZT_Node *node, unsigned int myId, diff --git a/node/Node.hpp b/node/Node.hpp index 98c4fd7c..3c0a5e92 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -102,6 +102,7 @@ public: void setNetconfMaster(void *networkControllerInstance); ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); + void pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); ZT_ResultCode clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, -- cgit v1.2.3 From 60bc2914141ec4c453b40b962ac18f4186a1c50b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 17:05:43 -0700 Subject: Add noAutoAssignIps for member of networks. --- controller/EmbeddedNetworkController.cpp | 5 +++-- controller/EmbeddedNetworkController.hpp | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 9cf43b79..69585d00 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -687,7 +687,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( ipAssignments = json::array(); } - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["noAutoAssignIps"],false)) ) { for(unsigned long p=0;((p Date: Thu, 25 Aug 2016 10:46:03 -0700 Subject: docs and null check in controller code --- README.md | 15 +++++++++------ controller/EmbeddedNetworkController.cpp | 7 +++++-- 2 files changed, 14 insertions(+), 8 deletions(-) (limited to 'controller') diff --git a/README.md b/README.md index a34a1a53..f8e438fa 100644 --- a/README.md +++ b/README.md @@ -33,19 +33,22 @@ To create networks of your own you'll need a network controller. You can use [ou ### Building from Source -For Mac, Linux, and BSD, just type "make" (or "gmake" on BSD). You won't need much installed; here are the requirements for various platforms: +For Mac, Linux, and BSD, just type `make` (or `gmake` on BSD). Platform requirements are: - * **Mac**: Xcode command line tools. It should build on OSX 10.7 or newer. - * **Linux**: gcc/g++ (4.9 or newer recommended) or clang/clang++ (3.4 or newer recommended) Makefile will use clang by default if available. The Linux build will auto-detect the presence of development headers for *json-parser*, *http-parser*, *li8bnatpmp*, and *libminiupnpc* and will link against the system libraries for these if they are present and recent enough. Otherwise the bundled versions in [ext/](ext/) will be used. Type `make install` to install the binaries and other files on the system, though this will not create init.d or systemd links. - * **FreeBSD**: C++ compiler (G++ usually) and GNU make (gmake). + - **Mac**: Xcode command line tools for OSX 10.7 or newer. + - **Linux**: GCC/G++ 4.9 or newer or CLANG 3.4 or newer, and GNU make and standard shell tools of course. + - The Linux make file will auto-detect and prefer CLANG if present since in our experience it does a better job optimizing C++ code. You can specify CC= and CXX= on the make command line to override this behavior. + - Several distributions including CentOS 7 ship with G++ 4.8 which has broken C++11 support. If you are running CentOS 7 type `sudo yum install clang` to install CLANG 3.4, which will work fine. There is no C++11 in the ZeroTier core but we have started using it in the service and network controller code. + - If any of the following have development headers on the system they are auto-detected and the build will link against system versions: `http-parser`, `lz4`, `libnatpmp`, and `miniupnpc`. If these are missing (or too old) the versions in `ext/` are used and are statically linked into the binary. Please be aware of this since if you build with these present the resulting binary will require them on other systems too. + - **FreeBSD**: GCC/G++ 4.9 or newer and `gmake` since our make files use GNU extensions. Each supported platform has its own *make-XXX.mk* file that contains the actual make rules for the platform. The right .mk file is included by the main Makefile based on the GNU make *OSTYPE* variable. Take a look at the .mk file for your platform for other targets, debug build rules, etc. Typing `make selftest` will build a *zerotier-selftest* binary which unit tests various internals and reports on a few aspects of the build environment. It's a good idea to try this on novel platforms or architectures. -Windows, of course, is special. We build for Windows with Microsoft Visual Studio 2012 on Windows 7. A solution file is located in the *windows/* subfolder. Newer versions of Visual Studio (and Windows) may work but haven't been tested. Older versions almost certainly will not, since they lack things like *stdint.h* and certain STL features. MinGW or other ports of gcc/clang to Windows should also work but haven't been tested. +Windows, of course, is special. A Visual Studio solution file is located in the *windows/* subfolder. We have not tried MinGW or CLANG for Windows but these may work, though you will have issues building the Windows GUI with them. -32 and 64 bit X86 and ARM (e.g. Raspberry Pi, Android) are officially supported. Community members have built for MIPS and Sparc without issues. +Mac and Windows require tun/tap virtual network port drivers. We include pre-built signed binaries for these in `ext/bin`, so you should not need to build these yourself. Linux, FreeBSD, and others can use built-in kernel support for tun/tap network devices. ### Running diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 69585d00..e98e4d6d 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1082,7 +1082,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); - _node->pushNetworkRefresh(address,nwid,(const uint64_t *)0,(const uint64_t *)0,0); + if (_node) + _node->pushNetworkRefresh(address,nwid,(const uint64_t *)0,(const uint64_t *)0,0); // Add non-persisted fields member["clock"] = now; @@ -1128,7 +1129,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( te.test = test; te.jsonResults = ""; - _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); + if (_node) + _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); + else return 500; char json[1024]; Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); -- cgit v1.2.3 From 1814016eb75b87d6f28711f41ecdafab69e556ee Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 25 Aug 2016 11:26:45 -0700 Subject: Add daemon thread to controller and move network member cache refreshes there. --- controller/EmbeddedNetworkController.cpp | 103 +++++++++++++++++++++++++------ controller/EmbeddedNetworkController.hpp | 45 +++++++------- 2 files changed, 107 insertions(+), 41 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index e98e4d6d..713b7618 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -60,6 +60,22 @@ using json = nlohmann::json; namespace ZeroTier { +// JSON blob I/O +static json _readJson(const std::string &path) +{ + std::string buf; + if (OSUtils::readFile(path.c_str(),buf)) { + try { + return json::parse(buf); + } catch ( ... ) {} + } + return json::object(); +} +static bool _writeJson(const std::string &path,const json &obj) +{ + return OSUtils::writeFile(path.c_str(),obj.dump(2)); +} + // Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible static uint64_t _jI(const json &jv,const uint64_t dfl) { @@ -394,16 +410,67 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _node(node), - _path(dbPath) + _path(dbPath), + _daemonRun(true) { OSUtils::mkdir(dbPath); OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions + _daemon = Thread::start(this); } EmbeddedNetworkController::~EmbeddedNetworkController() { } +void EmbeddedNetworkController::threadMain() + throw() +{ + uint64_t lastUpdatedNetworkMemberCache = 0; + while (_daemonRun) { + // Every 60 seconds we rescan the filesystem for network members and rebuild our cache + if ((OSUtils::now() - lastUpdatedNetworkMemberCache) >= 60000) { + const std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); + for(auto n=networks.begin();n!=networks.end();++n) { + if (n->length() == 16) { + const std::vector members(OSUtils::listSubdirectories((*n + ZT_PATH_SEPARATOR_S + "member").c_str())); + std::map newCache; + for(auto m=members.begin();m!=members.end();++m) { + if (m->length() == ZT_ADDRESS_LENGTH_HEX) { + const Address maddr(*m); + try { + const json mj(_readJson((_path + ZT_PATH_SEPARATOR_S + "network" + ZT_PATH_SEPARATOR_S + *n + ZT_PATH_SEPARATOR_S + "member" + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"))); + if ((mj.is_object())&&(mj.size() > 0)) { + newCache[maddr] = mj; + } + } catch ( ... ) {} + } + } + { + Mutex::Lock _l(_networkMemberCache_m); + _networkMemberCache[Utils::hexStrToU64(n->c_str())] = newCache; + } + } + } + lastUpdatedNetworkMemberCache = OSUtils::now(); + } + + { // Every 25ms we push up to 50 network refreshes, which amounts to a max of about 300-500kb/sec + unsigned int count = 0; + Mutex::Lock _l(_refreshQueue_m); + while (_refreshQueue.size() > 0) { + _Refresh &r = _refreshQueue.front(); + if (_node) + _node->pushNetworkRefresh(r.dest,r.nwid,r.blacklistAddresses,r.blacklistThresholds,r.numBlacklistEntries); + _refreshQueue.pop_front(); + if (++count >= 50) + break; + } + } + + Thread::sleep(25); + } +} + NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) { if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { @@ -1082,8 +1149,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); - if (_node) - _node->pushNetworkRefresh(address,nwid,(const uint64_t *)0,(const uint64_t *)0,0); + { + Mutex::Lock _l(_networkMemberCache_m); + _networkMemberCache[nwid][Address(address)] = member; + } + + { + Mutex::Lock _l(_refreshQueue_m); + _refreshQueue.push_back(_Refresh()); + _Refresh &r = _refreshQueue.back(); + r.dest = Address(address); + r.nwid = nwid; + r.numBlacklistEntries = 0; + } // Add non-persisted fields member["clock"] = now; @@ -1478,24 +1556,9 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) { Mutex::Lock _mcl(_networkMemberCache_m); - auto memberCacheEntry = _networkMemberCache[nwid]; - if ((now - memberCacheEntry.second) >= ZT_NETCONF_NETWORK_MEMBER_CACHE_EXPIRE) { - const std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); - std::vector members(OSUtils::listSubdirectories(bp.c_str())); - for(std::vector::iterator m(members.begin());m!=members.end();++m) { - if (m->length() == ZT_ADDRESS_LENGTH_HEX) { - nlohmann::json mj(_readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json")); - if ((mj.is_object())&&(mj.size() > 0)) { - memberCacheEntry.first[Address(*m)] = mj; - } - } - } - memberCacheEntry.second = now; - } - - nmi.totalMemberCount = memberCacheEntry.first.size(); - for(std::map< Address,nlohmann::json >::const_iterator nm(memberCacheEntry.first.begin());nm!=memberCacheEntry.first.end();++nm) { + nmi.totalMemberCount = memberCacheEntry.size(); + for(std::map< Address,nlohmann::json >::const_iterator nm(memberCacheEntry.begin());nm!=memberCacheEntry.end();++nm) { if (_jB(nm->second["authorized"],false)) { ++nmi.authorizedMemberCount; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 47419f52..b7a40a4c 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -25,21 +25,21 @@ #include #include #include +#include #include "../node/Constants.hpp" #include "../node/NetworkController.hpp" #include "../node/Mutex.hpp" #include "../node/Utils.hpp" +#include "../node/Address.hpp" #include "../node/InetAddress.hpp" #include "../osdep/OSUtils.hpp" +#include "../osdep/Thread.hpp" #include "../ext/json/json.hpp" -// Expiration time for network member cache entries in ms -#define ZT_NETCONF_NETWORK_MEMBER_CACHE_EXPIRE 30000 - namespace ZeroTier { class Node; @@ -50,6 +50,10 @@ public: EmbeddedNetworkController(Node *node,const char *dbPath); virtual ~EmbeddedNetworkController(); + // Thread main method -- do not call directly + void threadMain() + throw(); + virtual NetworkController::ResultCode doNetworkConfigRequest( const InetAddress &fromAddr, const Identity &signingId, @@ -83,22 +87,6 @@ public: private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); - // JSON blob I/O - inline nlohmann::json _readJson(const std::string &path) - { - std::string buf; - if (OSUtils::readFile(path.c_str(),buf)) { - try { - return nlohmann::json::parse(buf); - } catch ( ... ) {} - } - return nlohmann::json::object(); - } - inline bool _writeJson(const std::string &path,const nlohmann::json &obj) - { - return OSUtils::writeFile(path.c_str(),obj.dump(2)); - } - // Network base path and network JSON path inline std::string _networkBP(const uint64_t nwid,bool create) { @@ -133,8 +121,8 @@ private: return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); } - // We cache the members of networks in memory to avoid having to scan the filesystem so much - std::map< uint64_t,std::pair< std::map< Address,nlohmann::json >,uint64_t > > _networkMemberCache; + // In-memory cache of network members + std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache; Mutex _networkMemberCache_m; // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places @@ -211,6 +199,21 @@ private: // Last request time by address, for rate limitation std::map< std::pair,uint64_t > _lastRequestTime; Mutex _lastRequestTime_m; + + // Queue of network member refreshes to be pushed + struct _Refresh + { + Address dest; + uint64_t nwid; + uint64_t blacklistAddresses[64]; + uint64_t blacklistThresholds[64]; + unsigned int numBlacklistEntries; + }; + std::list< _Refresh > _refreshQueue; + Mutex _refreshQueue_m; + + Thread _daemon; + volatile bool _daemonRun; }; } // namespace ZeroTier -- cgit v1.2.3 From 5eaf397a948923d3de68763c972be7ae8c83132d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 25 Aug 2016 13:31:23 -0700 Subject: Add a debug log feature in the filter, which only works if enabled in Network.cpp. --- controller/EmbeddedNetworkController.cpp | 6 ++ controller/README.md | 2 +- include/ZeroTierOne.h | 5 ++ node/Capability.hpp | 1 + node/Network.cpp | 94 +++++++++++++++++++++++--------- 5 files changed, 80 insertions(+), 28 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 713b7618..1899edb9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -147,6 +147,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["flags"] = (uint64_t)rule.v.fwd.flags; r["length"] = (uint64_t)rule.v.fwd.length; break; + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: + r["type"] = "ACTION_DEBUG_LOG"; + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; r["zt"] = Address(rule.v.zt).toString(); @@ -278,6 +281,9 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; + } else if (t == "ACTION_DEBUG_LOG") { + rule.t |= ZT_NETWORK_RULE_ACTION_DEBUG_LOG; + return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; diff --git a/controller/README.md b/controller/README.md index 189fcdbd..68b8cdb6 100644 --- a/controller/README.md +++ b/controller/README.md @@ -5,7 +5,7 @@ ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit This code implements the *node/NetworkController.hpp* interface to provide an embedded microservice configurable via the same local HTTP control plane as ZeroTier One iteself. It is built by default in ZeroTier One in desktop and server builds. This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. -Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. Technically the JSON files under `controller.d` can also be edited in place, but we do not recommend doing this under a running controller since data loss or corruption might result. Also take care to keep JSON values of the correct types or data loss may also result. +Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, but they can be safely backed up or copied. ### Scalability and Reliability diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 4c0da76d..1d7620d4 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -515,6 +515,11 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_ACTION_REDIRECT = 3, + /** + * Log if match and if rule debugging is enabled in the build, otherwise does nothing (for developers) + */ + ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 4, + // 32 to 127 reserved for match criteria /** diff --git a/node/Capability.hpp b/node/Capability.hpp index 0b352725..ce2eedc4 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -176,6 +176,7 @@ public: switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x7f)) { //case ZT_NETWORK_RULE_ACTION_DROP: //case ZT_NETWORK_RULE_ACTION_ACCEPT: + //case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: default: b.append((uint8_t)0); break; diff --git a/node/Network.cpp b/node/Network.cpp index ecbacd93..02f53b55 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -35,9 +35,13 @@ #include "Node.hpp" #include "Peer.hpp" +// Uncomment to enable ZT_NETWORK_RULE_ACTION_DEBUG_LOG rule output to STDOUT +#define ZT_RULES_ENGINE_DEBUGGING 1 + namespace ZeroTier { -#ifdef ZT_TRACE +#ifdef ZT_RULES_ENGINE_DEBUGGING +#define FILTER_TRACE(f,...) { Utils::snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } static const char *_rtn(const ZT_VirtualNetworkRuleType rt) { switch(rt) { @@ -45,6 +49,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_ACTION_ACCEPT: return "ACTION_ACCEPT"; case ZT_NETWORK_RULE_ACTION_TEE: return "ACTION_TEE"; case ZT_NETWORK_RULE_ACTION_REDIRECT: return "ACTION_REDIRECT"; + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: return "ACTION_DEBUG_LOG"; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: return "MATCH_SOURCE_ZEROTIER_ADDRESS"; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: return "MATCH_DEST_ZEROTIER_ADDRESS"; case ZT_NETWORK_RULE_MATCH_VLAN_ID: return "MATCH_VLAN_ID"; @@ -65,7 +70,9 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) default: return "BAD_RULE_TYPE"; } } -#endif // ZT_TRACE +#else +#define FILTER_TRACE(f,...) {} +#endif // ZT_RULES_ENGINE_DEBUGGING // Returns true if packet appears valid; pos and proto will be set static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) @@ -96,9 +103,6 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig return false; // overflow == invalid } -//#define FILTER_TRACE TRACE -#define FILTER_TRACE(f,...) {} - // 0 == no match, -1 == match/drop, 1 == match/accept static int _doZtFilter( const RuntimeEnvironment *RR, @@ -127,6 +131,11 @@ static int _doZtFilter( // yields a 'match all' rule). uint8_t thisSetMatches = 1; +#ifdef ZT_RULES_ENGINE_DEBUGGING + std::vector dlog; + char dpbuf[1024]; +#endif + for(unsigned int rn=0;rn%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + ztSource.toString().c_str(), + ztDest.toString().c_str(), + (unsigned int)macSource[0], + (unsigned int)macSource[1], + (unsigned int)macSource[2], + (unsigned int)macSource[3], + (unsigned int)macSource[4], + (unsigned int)macSource[5], + (unsigned int)macDest[0], + (unsigned int)macDest[1], + (unsigned int)macDest[2], + (unsigned int)macDest[3], + (unsigned int)macDest[4], + (unsigned int)macDest[5], + (int)inbound, + (int)noRedirect, + frameLen, + etherType + ); + for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) + printf(" %s" ZT_EOL_S,m->c_str()); + dlog.clear(); + } +#endif // ZT_RULES_ENGINE_DEBUGGING + thisRuleMatches = 1; + thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation + break; // Rules --------------------------------------------------------------- // thisSetMatches is the binary AND of the result of all rules in a set case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - FILTER_TRACE("FILTER[%u] %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); + FILTER_TRACE("%u %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - FILTER_TRACE("FILTER[%u] %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); + FILTER_TRACE("%u %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanId); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanId); thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: // NOT SUPPORTED YET - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanPcp); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanPcp); thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: // NOT SUPPORTED YET - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanDei); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanDei); thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - FILTER_TRACE("FILTER[%u] %s param0=%u etherType=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.etherType,etherType); + FILTER_TRACE("%u %s param0=%u etherType=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.etherType,etherType); thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - FILTER_TRACE("FILTER[%u] %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); + FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: - FILTER_TRACE("FILTER[%u] %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); + FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); } else { @@ -222,7 +262,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); } else { @@ -230,7 +270,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); } else { @@ -238,7 +278,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); } else { @@ -246,7 +286,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipTos); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipTos); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { @@ -257,7 +297,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipProtocol); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipProtocol); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); } else if (etherType == ZT_ETHERTYPE_IPV6) { @@ -289,7 +329,7 @@ static int _doZtFilter( } break; } - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port==%u proto==%u etherType=%u (IPv4)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,(unsigned int)frameData[9],etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto==%u etherType=%u (IPv4)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,(unsigned int)frameData[9],etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; @@ -308,14 +348,14 @@ static int _doZtFilter( } break; } - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port==%u proto=%u etherType=%u (IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,proto,etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto=%u etherType=%u (IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,proto,etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; } else { - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port=0 proto=0 etherType=%u (IPv6 parse failed)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (IPv6 parse failed)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; } } else { - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port=0 proto=0 etherType=%u (not IPv4 or IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (not IPv4 or IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; } break; @@ -336,18 +376,18 @@ static int _doZtFilter( } } } - FILTER_TRACE("FILTER[%u] %s param0=%.16llx param1=%.16llx actual=%.16llx",rn,_rtn(rt),rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],cf); + FILTER_TRACE("%u %s param0=%.16llx param1=%.16llx actual=%.16llx",rn,_rtn(rt),rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],cf); thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1]); + FILTER_TRACE("%u %s param0=%u param1=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1]); thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.tag.value); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.tag.value); const Tag *lt = (const Tag *)0; for(unsigned int i=0;i> 7)); - FILTER_TRACE("FILTER[%u] %s/%u thisRuleMatches==%u thisSetMatches==%u",rn,_rtn(rt),(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); + FILTER_TRACE("%u %s/%u thisRuleMatches==%u thisSetMatches==%u",rn,_rtn(rt),(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); } return 0; -- cgit v1.2.3 From b5e0d014ab7fc96f8ebc3444cc1d2f3f0648c66f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 25 Aug 2016 16:08:40 -0700 Subject: Controller bug fixes --- controller/EmbeddedNetworkController.cpp | 208 +++++++++++++++++++------------ controller/EmbeddedNetworkController.hpp | 2 + node/Network.cpp | 30 +++++ 3 files changed, 158 insertions(+), 82 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 1899edb9..23514448 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -127,7 +127,6 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; json r = json::object(); - r["not"] = ((rule.t & 0x80) != 0); switch((rule.t) & 0x7f) { case ZT_NETWORK_RULE_ACTION_DROP: r["type"] = "ACTION_DROP"; @@ -152,74 +151,91 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["not"] = ((rule.t & 0x80) != 0); r["zt"] = Address(rule.v.zt).toString(); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["not"] = ((rule.t & 0x80) != 0); r["zt"] = Address(rule.v.zt).toString(); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: r["type"] = "MATCH_VLAN_ID"; + r["not"] = ((rule.t & 0x80) != 0); r["vlanId"] = (uint64_t)rule.v.vlanId; break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: r["type"] = "MATCH_VLAN_PCP"; + r["not"] = ((rule.t & 0x80) != 0); r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: r["type"] = "MATCH_VLAN_DEI"; + r["not"] = ((rule.t & 0x80) != 0); r["vlanDei"] = (uint64_t)rule.v.vlanDei; break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: r["type"] = "MATCH_ETHERTYPE"; + r["not"] = ((rule.t & 0x80) != 0); r["etherType"] = (uint64_t)rule.v.etherType; break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; + r["not"] = ((rule.t & 0x80) != 0); Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: r["type"] = "MATCH_MAC_DEST"; + r["not"] = ((rule.t & 0x80) != 0); Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: r["type"] = "MATCH_IPV4_SOURCE"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: r["type"] = "MATCH_IPV4_DEST"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: r["type"] = "MATCH_IPV6_SOURCE"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: r["type"] = "MATCH_IPV6_DEST"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IP_TOS: r["type"] = "MATCH_IP_TOS"; + r["not"] = ((rule.t & 0x80) != 0); r["ipTos"] = (uint64_t)rule.v.ipTos; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: r["type"] = "MATCH_IP_PROTOCOL"; + r["not"] = ((rule.t & 0x80) != 0); r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["not"] = ((rule.t & 0x80) != 0); r["start"] = (uint64_t)rule.v.port[0]; r["end"] = (uint64_t)rule.v.port[1]; break; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["not"] = ((rule.t & 0x80) != 0); r["start"] = (uint64_t)rule.v.port[0]; r["end"] = (uint64_t)rule.v.port[1]; break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; + r["not"] = ((rule.t & 0x80) != 0); Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); r["mask"] = tmp; Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); @@ -227,26 +243,31 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["not"] = ((rule.t & 0x80) != 0); r["start"] = (uint64_t)rule.v.frameSize[0]; r["end"] = (uint64_t)rule.v.frameSize[1]; break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: r["type"] = "MATCH_TAGS_SAMENESS"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; @@ -254,9 +275,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) return r; } -static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) +static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) { - if (r.is_object()) + if (!r.is_object()) return false; const std::string t(_jS(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); @@ -494,7 +515,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( lrt = now; } - const json network(_readJson(_networkJP(nwid,false))); + json network(_readJson(_networkJP(nwid,false))); if (!network.size()) return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; @@ -635,14 +656,14 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); - const json &v4AssignMode = network["v4AssignMode"]; - const json &v6AssignMode = network["v6AssignMode"]; - const json &ipAssignmentPools = network["ipAssignmentPools"]; - const json &routes = network["routes"]; - const json &rules = network["rules"]; - const json &capabilities = network["capabilities"]; - const json &memberCapabilities = member["capabilities"]; - const json &memberTags = member["tags"]; + json &v4AssignMode = network["v4AssignMode"]; + json &v6AssignMode = network["v6AssignMode"]; + json &ipAssignmentPools = network["ipAssignmentPools"]; + json &routes = network["routes"]; + json &rules = network["rules"]; + json &capabilities = network["capabilities"]; + json &memberCapabilities = member["capabilities"]; + json &memberTags = member["tags"]; if (rules.is_array()) { for(unsigned long i=0;i 0)&&(capabilities.is_array())) { - std::map< uint64_t,const json * > capsById; + std::map< uint64_t,json * > capsById; for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; unsigned int caprc = 0; - auto caprj = (*cap)["rules"]; + json &caprj = (*cap)["rules"]; if ((caprj.is_array())&&(caprj.size() > 0)) { for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) @@ -688,7 +709,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if (memberTags.is_array()) { std::map< uint32_t,uint32_t > tagsById; for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) break; - auto route = routes[i]; - InetAddress t(_jS(route["target"],"")); - InetAddress v(_jS(route["via"],"")); - if ((t)&&(v)&&(t.ss_family == v.ss_family)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); - *(reinterpret_cast(&(r->target))) = t; - *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; + json &route = routes[i]; + json &target = route["target"]; + json &via = route["via"]; + if (target.is_string()) { + const InetAddress t(target.get()); + InetAddress v; + if (via.is_string()) v.fromString(via.get()); + if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { + ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + *(reinterpret_cast(&(r->target))) = t; + if (v.ss_family == t.ss_family) + *(reinterpret_cast(&(r->via))) = v; + ++nc.routeCount; + } } } } - if (v6AssignMode.is_object()) { + const bool noAutoAssignIps = _jB(member["noAutoAssignIps"],false); + + if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; @@ -730,9 +759,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( bool haveManagedIpv4AutoAssignment = false; bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count - json ipAssignments = member["ipAssignments"]; + json ipAssignments = member["ipAssignments"]; // we want to make a copy if (ipAssignments.is_array()) { for(unsigned long i=0;i(&ipRangeStart)->sin_addr.s_addr)); - uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); - if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) + InetAddress ipRangeStartIA(_jS(pool["ipRangeStart"],"")); + InetAddress ipRangeEndIA(_jS(pool["ipRangeEnd"],"")); + if ( (ipRangeStartIA.ss_family == AF_INET) && (ipRangeEndIA.ss_family == AF_INET) ) { + uint32_t ipRangeStart = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeStartIA)->sin_addr.s_addr)); + uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); + if ((ipRangeEnd < ipRangeStart)||(ipRangeStart == 0)) continue; uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; // Start with the LSB of the member's address uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); - for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { + for(uint32_t k=ipRangeStart,trialCount=0;((k<=ipRangeEnd)&&(trialCount < 1000));++k,++trialCount) { uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; ++ipTrialCounter; if ((ip & 0x000000ff) == 0x000000ff) continue; // don't allow addresses that end in .255 // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; + int routedNetmaskBits = -1; for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { @@ -855,9 +886,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - InetAddress ip4(Utils::hton(ip),0); - // If it's routed, then try to claim and assign it and if successful end loop + const InetAddress ip4(Utils::hton(ip),0); if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; @@ -1048,9 +1078,18 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json b; try { b = json::parse(body); - if (!b.is_object()) + if (!b.is_object()) { + responseBody = "{ \"message\": \"body is not a JSON object\" }"; + responseContentType = "application/json"; return 400; + } + } catch (std::exception &exc) { + responseBody = std::string("{ \"message\": \"body JSON is invalid: ") + exc.what() + "\" }"; + responseContentType = "application/json"; + return 400; } catch ( ... ) { + responseBody = "{ \"message\": \"body JSON is invalid\" }"; + responseContentType = "application/json"; return 400; } const uint64_t now = OSUtils::now(); @@ -1143,6 +1182,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } } } catch ( ... ) { + responseBody = "{ \"message\": \"exception while processing parameters in JSON body\" }"; + responseContentType = "application/json"; return 400; } @@ -1204,6 +1245,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!test->hopCount) { ::free((void *)test); + responseBody = "{ \"message\": \"a test must contain at least one hop\" }"; + responseContentType = "application/json"; return 400; } @@ -1258,19 +1301,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); if (b.count("v4AssignMode")) { - auto nv4m = network["v4AssignMode"]; + json &nv4m = network["v4AssignMode"]; if (!nv4m.is_object()) nv4m = json::object(); if (b["v4AssignMode"].is_string()) { // backward compatibility nv4m["zt"] = (_jS(b["v4AssignMode"],"") == "zt"); } else if (b["v4AssignMode"].is_object()) { - auto v4m = b["v4AssignMode"]; + json &v4m = b["v4AssignMode"]; if (v4m.count("zt")) nv4m["zt"] = _jB(v4m["zt"],false); } if (!nv4m.count("zt")) nv4m["zt"] = false; } if (b.count("v6AssignMode")) { - auto nv6m = network["v6AssignMode"]; + json &nv6m = network["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (b["v6AssignMode"].is_string()) { // backward compatibility std::vector v6m(Utils::split(_jS(b["v6AssignMode"],"").c_str(),",","","")); @@ -1285,7 +1328,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( nv6m["6plane"] = true; } } else if (b["v6AssignMode"].is_object()) { - auto v6m = b["v6AssignMode"]; + json &v6m = b["v6AssignMode"]; if (v6m.count("rfc4193")) nv6m["rfc4193"] = _jB(v6m["rfc4193"],false); if (v6m.count("zt")) nv6m["rfc4193"] = _jB(v6m["zt"],false); if (v6m.count("6plane")) nv6m["rfc4193"] = _jB(v6m["6plane"],false); @@ -1296,61 +1339,64 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } if (b.count("routes")) { - auto rts = b["routes"]; + json &rts = b["routes"]; if (rts.is_array()) { + json nrts = json::array(); for(unsigned long i=0;i()); + InetAddress v; + if (via.is_string()) v.fromString(via.get()); + if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.netmaskBitsValid()) ) { + json tmp; + tmp["target"] = t.toString(); + if (v.ss_family == t.ss_family) + tmp["via"] = v.toIpString(); + else tmp["via"] = json(); + nrts.push_back(tmp); + } } } } + network["routes"] = nrts; } } if (b.count("ipAssignmentPools")) { - auto ipp = b["ipAssignmentPools"]; + json &ipp = b["ipAssignmentPools"]; if (ipp.is_array()) { + json nipp = json::array(); for(unsigned long i=0;i ncaps; for(unsigned long i=0;i Date: Thu, 25 Aug 2016 16:25:28 -0700 Subject: A little bit more controller code cleanup. --- controller/EmbeddedNetworkController.cpp | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 23514448..7a599d44 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1606,18 +1606,20 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) { Mutex::Lock _mcl(_networkMemberCache_m); - auto memberCacheEntry = _networkMemberCache[nwid]; + std::map< Address,nlohmann::json > &memberCacheEntry = _networkMemberCache[nwid]; nmi.totalMemberCount = memberCacheEntry.size(); - for(std::map< Address,nlohmann::json >::const_iterator nm(memberCacheEntry.begin());nm!=memberCacheEntry.end();++nm) { + for(std::map< Address,nlohmann::json >::iterator nm(memberCacheEntry.begin());nm!=memberCacheEntry.end();++nm) { if (_jB(nm->second["authorized"],false)) { ++nmi.authorizedMemberCount; - auto mlog = nm->second["recentLog"]; - if ((mlog.is_array())&&(mlog.size() > 0)) { - auto mlog1 = mlog[0]; - if (mlog1.is_object()) { - if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) - ++nmi.activeMemberCount; + if (nm->second.count("recentLog")) { + json &mlog = nm->second["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + json &mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) + ++nmi.activeMemberCount; + } } } @@ -1625,12 +1627,14 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid nmi.activeBridges.insert(nm->first); } - auto mips = nm->second["ipAssignments"]; - if (mips.is_array()) { - for(unsigned long i=0;isecond.count("ipAssignments")) { + json &mips = nm->second["ipAssignments"]; + if (mips.is_array()) { + for(unsigned long i=0;i Date: Thu, 25 Aug 2016 16:28:54 -0700 Subject: one more... --- controller/EmbeddedNetworkController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7a599d44..8f013464 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -607,7 +607,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( json recentLog = json::array(); recentLog.push_back(rlEntry); - auto oldLog = member["recentLog"]; + json &oldLog = member["recentLog"]; if (oldLog.is_array()) { for(unsigned long i=0;i Date: Thu, 25 Aug 2016 18:21:20 -0700 Subject: Fix chicken or egg problem in tags, and better filter debug instrumentation. --- controller/EmbeddedNetworkController.cpp | 2 +- include/ZeroTierOne.h | 6 +- node/Membership.cpp | 25 ++++-- node/Membership.hpp | 10 ++- node/Network.cpp | 136 ++++++++++++++++--------------- 5 files changed, 96 insertions(+), 83 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8f013464..45af30be 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -50,7 +50,7 @@ using json = nlohmann::json; #define ZT_NETCONF_CONTROLLER_API_VERSION 3 // Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 16 +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 32 // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 1d7620d4..a3d82adb 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -107,14 +107,14 @@ extern "C" { #define ZT_MAX_NETWORK_RULES 1024 /** - * Maximum number of per-node capabilities per network + * Maximum number of per-member capabilities per network */ #define ZT_MAX_NETWORK_CAPABILITIES 128 /** - * Maximum number of per-node tags per network + * Maximum number of per-member tags per network */ -#define ZT_MAX_NETWORK_TAGS 128 +#define ZT_MAX_NETWORK_TAGS 16 /** * Maximum number of direct network paths to a given peer diff --git a/node/Membership.cpp b/node/Membership.cpp index 8bf5b784..7ced9dcf 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -28,8 +28,12 @@ namespace ZeroTier { -bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const CertificateOfMembership &com,const Capability *cap,const Tag **tags,const unsigned int tagCount) +bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap) { + if ((now - _lastPushAttempt) < 1000ULL) + return false; + _lastPushAttempt = now; + try { Buffer capsAndTags; @@ -50,27 +54,29 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint unsigned int appendedTags = 0; const unsigned int tagCountPos = capsAndTags.size(); capsAndTags.addSize(2); - for(unsigned int i=0;iid()); + for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { if ((capsAndTags.size() + sizeof(Tag)) > (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) break; - tags[i]->serialize(capsAndTags); + nconf.tags[i].serialize(capsAndTags); ts->lastPushed = now; ++appendedTags; } } capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - if ( ((com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)) || (appendedCaps) || (appendedTags) ) { + const bool needCom = ((nconf.isPrivate())&&(nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); + if ( (needCom) || (appendedCaps) || (appendedTags) ) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - if (com) - com.serialize(outp); + if (needCom) { + nconf.com.serialize(outp); + _lastPushedCom = now; + } outp.append((uint8_t)0x00); outp.append(capsAndTags.data(),capsAndTags.size()); outp.compress(); RR->sw->send(outp,true); - _lastPushedCom = now; return true; } } catch ( ... ) { @@ -88,8 +94,9 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe const int vr = com.verify(RR); if (vr == 0) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); - if (com.timestamp().first > _com.timestamp().first) + if (com.timestamp().first > _com.timestamp().first) { _com = com; + } } else { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); } diff --git a/node/Membership.hpp b/node/Membership.hpp index bb356902..580aa529 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -107,6 +107,7 @@ public: friend class CapabilityIterator; Membership() : + _lastPushAttempt(0), _lastPushedCom(0), _blacklistBefore(0), _com(), @@ -124,13 +125,11 @@ public: * @param RR Runtime environment * @param now Current time * @param peerAddress Address of member peer - * @param com My network certificate of membership (if any) (not the one here, but ours -- in NetworkConfig) + * @param nconf My network config * @param cap Capability to send or 0 if none - * @param tags Tags that this peer might need - * @param tagCount Number of tag IDs * @return True if we pushed something */ - bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const CertificateOfMembership &com,const Capability *cap,const Tag **tags,const unsigned int tagCount); + bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap); /** * @param nconf Our network config @@ -270,6 +269,9 @@ public: } private: + // Last time we checked if credential push was needed + uint64_t _lastPushAttempt; + // Last time we pushed our COM to this peer uint64_t _lastPushedCom; diff --git a/node/Network.cpp b/node/Network.cpp index 5293e2f0..1267f99c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -143,7 +143,7 @@ static int _doZtFilter( const Address &ztDest, const MAC &macSource, const MAC &macDest, - const uint8_t *frameData, + const uint8_t *const frameData, const unsigned int frameLen, const unsigned int etherType, const unsigned int vlanId, @@ -151,11 +151,9 @@ static int _doZtFilter( const unsigned int ruleCount, const Tag *localTags, const unsigned int localTagCount, - const uint32_t *remoteTagIds, - const uint32_t *remoteTagValues, - const unsigned int remoteTagCount, - const Tag **relevantLocalTags, // pointer array must be at least [localTagCount] in size - unsigned int &relevantLocalTagCount) + const uint32_t *const remoteTagIds, + const uint32_t *const remoteTagValues, + const unsigned int remoteTagCount) { // For each set of rules we start by assuming that they match (since no constraints // yields a 'match all' rule). @@ -168,30 +166,22 @@ static int _doZtFilter( for(unsigned int rn=0;rn %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.zt,ztSource.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - FILTER_TRACE("%u %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); + FILTER_TRACE("%u %s %c %.10llx==%.10llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.zt,ztDest.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanId); thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanId,(unsigned int)vlanId,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: // NOT SUPPORTED YET - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanPcp); thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanPcp,0,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: // NOT SUPPORTED YET - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanDei); thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanDei,0,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - FILTER_TRACE("%u %s param0=%u etherType=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.etherType,etherType); thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.etherType,etherType,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); + FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macSource.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: - FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); + FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macDest.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str(),InetAddress((const void *)(frameData + 12),4,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv4] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str(),InetAddress((const void *)(frameData + 16),4,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv4] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str(),InetAddress((const void *)(frameData + 8),16,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str(),InetAddress((const void *)(frameData + 24),16,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipTos); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((frameData[1] & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { const uint8_t trafficClass = ((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f); thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((trafficClass & 0xfc) >> 2)); + FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((trafficClass & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipProtocol); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); + FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipProtocol,(unsigned int)frameData[9],(unsigned int)thisRuleMatches); } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto); + FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipProtocol,proto,(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: @@ -359,8 +362,9 @@ static int _doZtFilter( } break; } - FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto==%u etherType=%u (IPv4)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,(unsigned int)frameData[9],etherType); - thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + + thisRuleMatches = (p >= 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + FILTER_TRACE("%u %s %c (IPv4) %d in %d-%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),p,(int)rules[rn].v.port[0],(int)rules[rn].v.port[1],(unsigned int)thisRuleMatches); } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { @@ -378,15 +382,15 @@ static int _doZtFilter( } break; } - FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto=%u etherType=%u (IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,proto,etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + FILTER_TRACE("%u %s %c (IPv6) %d in %d-%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),p,(int)rules[rn].v.port[0],(int)rules[rn].v.port[1],(unsigned int)thisRuleMatches); } else { - FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (IPv6 parse failed)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { - FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (not IPv4 or IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: { @@ -406,18 +410,17 @@ static int _doZtFilter( } } } - FILTER_TRACE("%u %s param0=%.16llx param1=%.16llx actual=%.16llx",rn,_rtn(rt),rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],cf); thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); + FILTER_TRACE("%u %s %c (%.16llx & %.16llx)==%.16llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],(unsigned int)thisRuleMatches); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - FILTER_TRACE("%u %s param0=%u param1=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1]); thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); + FILTER_TRACE("%u %s %c %u in %u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),frameLen,(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1],(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.tag.value); const Tag *lt = (const Tag *)0; for(unsigned int i=0;i 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { const uint32_t *rtv = (const uint32_t *)0; for(unsigned int i=0;i 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { if (rt == ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS) { const uint32_t sameness = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); thisRuleMatches = (uint8_t)(sameness <= rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u sameness:%u <= %u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,sameness,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { - thisRuleMatches = (uint8_t)((lt->value() & *rtv) <= rules[rn].v.tag.value); + thisRuleMatches = (uint8_t)((lt->value() & *rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) { - thisRuleMatches = (uint8_t)((lt->value() | *rtv) <= rules[rn].v.tag.value); + thisRuleMatches = (uint8_t)((lt->value() | *rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x | remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { - thisRuleMatches = (uint8_t)((lt->value() ^ *rtv) <= rules[rn].v.tag.value); + thisRuleMatches = (uint8_t)((lt->value() ^ *rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { // sanity check, can't really happen thisRuleMatches = 0; } - if (thisRuleMatches) { - relevantLocalTags[relevantLocalTagCount++] = lt; - } } } } break; + + default: continue; } // thisSetMatches remains true if the current rule matched (or did NOT match if NOT bit is set) - thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t & 0x80) >> 7)); - - FILTER_TRACE("%u %s/%u thisRuleMatches==%u thisSetMatches==%u",rn,_rtn(rt),(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); + thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); } return 0; @@ -543,31 +549,32 @@ bool Network::filterOutgoingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; - const Tag *relevantLocalTags[ZT_MAX_NETWORK_TAGS]; - unsigned int relevantLocalTagCount = 0; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch(_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: + if (ztDest) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return false; case 1: if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,(const Capability *)0,relevantLocalTags,relevantLocalTagCount); + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return true; } for(unsigned int c=0;c<_config.capabilityCount;++c) { - relevantLocalTagCount = 0; - switch (_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: + if (ztDest) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return false; case 1: if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,&(_config.capabilities[c]),relevantLocalTags,relevantLocalTagCount); + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,&(_config.capabilities[c])); return true; } } @@ -587,15 +594,13 @@ bool Network::filterIncomingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; - const Tag *relevantLocalTags[ZT_MAX_NETWORK_TAGS]; - unsigned int relevantLocalTagCount = 0; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: return false; case 1: @@ -605,8 +610,7 @@ bool Network::filterIncomingPacket( Membership::CapabilityIterator mci(m); const Capability *c; while ((c = mci.next(_config))) { - relevantLocalTagCount = 0; - switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: return false; case 1: @@ -1030,7 +1034,7 @@ void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std:: { Membership *m = _memberships.get(peer->address()); if (m) - m->sendCredentialsIfNeeded(RR,RR->node->now(),peer->address(),_config.com,(const Capability *)0,(const Tag **)0,0); + m->sendCredentialsIfNeeded(RR,RR->node->now(),peer->address(),_config,(const Capability *)0); } Packet outp(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); -- cgit v1.2.3 From ded5a53a6ce5f6de2f5ebfc76f5d1ca68edc605b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 10:38:43 -0700 Subject: Documentation updates, add rules engine revision to network config request meta-data. --- controller/README.md | 36 ++++++++++++++++++++++++++++------- include/ZeroTierOne.h | 7 ++++++- node/Network.cpp | 51 ++++++++++++++++---------------------------------- node/NetworkConfig.hpp | 2 ++ one.cpp | 6 ++++++ 5 files changed, 59 insertions(+), 43 deletions(-) (limited to 'controller') diff --git a/controller/README.md b/controller/README.md index 68b8cdb6..1eb0ca0d 100644 --- a/controller/README.md +++ b/controller/README.md @@ -1,15 +1,33 @@ Network Controller Microservice ====== -ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit ZeroTier address of a network controller followed by a 6-digit (24-bit) network number on that controller. Fans of software defined networking will recognize this as a spin on the familiar [separation of data plane and control plane](http://sdntutorials.com/difference-between-control-plane-and-data-plane/) SDN design pattern. +Every ZeroTier virtual network has a *network controller*. This is our reference implementation and is the same one we use to power our own hosted services at [my.zerotier.com](https://my.zerotier.com/). Network controllers act as configuration servers and certificate authorities for the members of networks. Controllers are located on the network by simply parsing out the first 10 digits of a network's 16-digit network ID: these are the address of the controller. -This code implements the *node/NetworkController.hpp* interface to provide an embedded microservice configurable via the same local HTTP control plane as ZeroTier One iteself. It is built by default in ZeroTier One in desktop and server builds. This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. +As of ZeroTier One version 1.2.0 this code is included in normal builds for desktop, laptop, and server (Linux, etc.) targets, allowing any device to create virtual networks without having to be rebuilt from source with special flags to enable this feature. While this does offer a convenient way to create ad-hoc networks or experiment, we recommend running a dedicated controller somewhere secure and stable for any "serious" use case. -Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, but they can be safely backed up or copied. +Controller data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, rsync'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, and if they are edited directly take care not to save corrupt JSON since that can also lead to data loss when the controller is restarted. Going through the API is strongly preferred to directly modifying these files. + +### Upgrading from Older Versions + +Older versions of this code used a SQLite database instead of in-filesystem JSON. A migration utility called `migrate-sqlite` is included here and *must* be used to migrate this data to the new format. If the controller is started with an old `controller.db` in its working directory it will terminate after printing an error to *stderr*. This is done to prevent "surprises" for those running DIY controllers using the old code. + +The migration tool is written in nodeJS and can be used like this: + + cd migrate-sqlite + npm install + node migrate-sqlite.js + +You may need to `sudo node ...` if the ZeroTier working directory is owned by root. + +This code will dump the contents of any `controller.db` in the ZeroTier working directory and recreate its data in the form of JSON objects under `controller.d`. The old `controller.db` will then be renamed to `controller.db.migrated` and the controller will start normally. + +After migrating make sure that the contents of `controller.d` are owned and writable by the user that will be running the ZeroTier controller process! (Usually this is root but controllers that don't also join networks are sometimes run as unprivileged users.) + +If you don't have nodeJS on the machine running ZeroTier it is perfectly fine to just copy `controller.db` to a directory on another machine and run the migration tool there. After running your migration the contents of `controller.d` can be tar'd up and copied back over to the controller. Just remember to rename or remove `controller.db` on the controller machine afterwords so the controller will start. ### Scalability and Reliability -Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. +Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. Since the controller uses the filesystem as its data store we recommend fast filesystems and fast SSD drives for heavily loaded controllers. Since ZeroTier nodes are mobile and do not need static IPs, implementing high availability fail-over for controllers is easy. Just replicate their working directories from master to backup and have something automatically fire up the backup if the master goes down. Many modern orchestration tools have built-in support for this. It would also be possible in theory to run controllers on a replicated or distributed filesystem, but we haven't tested this yet. @@ -154,6 +172,7 @@ The entry types and their additional fields are: | `ACTION_ACCEPT` | Accept any packets matching this rule | (none) | | `ACTION_TEE` | Send a copy of this packet to a node (rule parsing continues) | `zt` | | `ACTION_REDIRECT` | Redirect this packet to another node | `zt` | +| `ACTION_DEBUG_LOG` | Output debug info on match (if built with rules engine debug) | (none) | | `MATCH_SOURCE_ZEROTIER_ADDRESS` | Match VL1 ZeroTier address of packet sender. | `zt` | | `MATCH_DEST_ZEROTIER_ADDRESS` | Match VL1 ZeroTier address of recipient | `zt` | | `MATCH_ETHERTYPE` | Match Ethernet frame type | `etherType` | @@ -174,10 +193,12 @@ The entry types and their additional fields are: | `MATCH_TAGS_BITWISE_OR` | Match if both sides' tags OR to value | `id`,`value` | | `MATCH_TAGS_BITWISE_XOR` | Match if both sides` tags XOR to value | `id`,`value` | -Notes: +Important notes about rules engine behavior: - * `MATCH_DEST_ZEROTIER_ADDRESS` does not work for multicast packets since these have many and varying destinations and can take circuitous routes. Use MAC rules instead. - * IPv4 and IPv6 rules do not match for frames that are not IPv4 or IPv6 respectively. + * IPv4 and IPv6 IP address rules do not match for frames that are not IPv4 or IPv6 respectively. + * `ACTION_DEBUG_LOG` is a no-op on nodes not built with `ZT_RULES_ENGINE_DEBUGGING` enabled (see Network.cpp). If that is enabled nodes will dump a trace of rule evaluation results to *stdout* when this action is encountered but will otherwise keep evaluating rules. This is used for basic "smoke testing" of the rules engine. + * Multicast packets and packets destined for bridged devices treated a little differently. They are matched more than once. They are matched at the point of send with a NULL ZeroTier destination address, meaning that `MATCH_DEST_ZEROTIER_ADDRESS` is useless. That's because the true VL1 destination is not yet known. Then they are matched again for each true VL1 destination. On these later subsequent matches TEE actions are ignored and REDIRECT rules are interpreted as DROPs. This prevents multiple TEE or REDIRECT packets from being sent to third party devices. + * Rules in capabilities are always matched as if the current device is the sender (inbound == false). A capability specifies sender side rules that can be enforced on both sides. #### `/controller/network//member` @@ -228,6 +249,7 @@ Note that managed IP assignments are only used if they fall within a managed rou | clientMajorVersion | integer | Client major version or -1 if unknown | | clientMinorVersion | integer | Client minor version or -1 if unknown | | clientRevision | integer | Client revision or -1 if unknown | +| clientProtocolVersion | integer | ZeroTier protocol version reported by client | | fromAddr | string | Physical address if known | The controller can only know a member's `fromAddr` if it's able to establish a direct path to it. Members behind very restrictive firewalls may not have this information since the controller will be receiving the member's requests by way of a relay. ZeroTier does not back-trace IP paths as packets are relayed since this would add a lot of protocol overhead. diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index a3d82adb..9dafcaa6 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -101,6 +101,11 @@ extern "C" { */ #define ZT_MAX_NETWORK_MULTICAST_SUBSCRIPTIONS 4096 +/** + * Rules engine revision ID, which specifies rules engine capabilities + */ +#define ZT_RULES_ENGINE_REVISION 1 + /** * Maximum number of base (non-capability) network rules */ @@ -114,7 +119,7 @@ extern "C" { /** * Maximum number of per-member tags per network */ -#define ZT_MAX_NETWORK_TAGS 16 +#define ZT_MAX_NETWORK_TAGS 128 /** * Maximum number of direct network paths to a given peer diff --git a/node/Network.cpp b/node/Network.cpp index 1267f99c..0bbf070c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -38,36 +38,6 @@ // Uncomment to enable ZT_NETWORK_RULE_ACTION_DEBUG_LOG rule output to STDOUT #define ZT_RULES_ENGINE_DEBUGGING 1 -/* -{ - "name": "filter_log_test", - "private": true, - "v4AssignMode": { - "zt": true - }, - "v6AssignMode": { - "rfc4193": true, - "zt": false, - "6plane": false - }, - "routes": [ - { "target": "10.140.140.0/24", "via": null } - ], - "ipAssignmentPools": [ - { "ipRangeStart": "10.140.140.2", "ipRangeEnd": "10.140.140.254" } - ], - "rules": [ - { "type": "MATCH_ETHERTYPE", "etherType": 0x0800 }, - { "type": "ACTION_DEBUG_LOG" }, - - { "type": "MATCH_ETHERTYPE", "etherType": 0x0800, "not": true }, - { "type": "ACTION_DEBUG_LOG" }, - - { "type": "ACTION_ACCEPT" } - ] -} -*/ - namespace ZeroTier { #ifdef ZT_RULES_ENGINE_DEBUGGING @@ -162,7 +132,7 @@ static int _doZtFilter( #ifdef ZT_RULES_ENGINE_DEBUGGING std::vector dlog; char dpbuf[1024]; -#endif +#endif // ZT_RULES_ENGINE_DEBUGGING for(unsigned int rn=0;rn%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + printf(" _ " ZT_EOL_S); + for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) + printf(" | %s" ZT_EOL_S,m->c_str()); + printf(" + MATCH %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, ztSource.toString().c_str(), ztDest.toString().c_str(), (unsigned int)macSource[0], @@ -225,10 +207,8 @@ static int _doZtFilter( frameLen, etherType ); - for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) - printf(" %s" ZT_EOL_S,m->c_str()); - dlog.clear(); } + dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation continue; @@ -793,6 +773,7 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES,(uint64_t)ZT_MAX_CAPABILITY_RULES); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION); if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 22ffb1cf..67126d64 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -107,6 +107,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv" +// Rules engine revision +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV "revr" // Maximum number of rules per network this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES "mr" // Maximum number of capabilities this node can accept diff --git a/one.cpp b/one.cpp index 3cb6b775..7cddb376 100644 --- a/one.cpp +++ b/one.cpp @@ -1086,6 +1086,12 @@ int main(int argc,char **argv) } } + // This can be removed once the new controller code has been around for many versions + if (OSUtils::fileExists((homeDir + ZT_PATH_SEPARATOR_S + "controller.db").c_str(),true)) { + fprintf(stderr,"%s: FATAL: an old controller.db exists in %s -- see instructions in controller/README.md for how to migrate!" ZT_EOL_S,argv[0],homeDir.c_str()); + return 1; + } + #ifdef __UNIX_LIKE__ #ifndef ZT_ONE_NO_ROOT_CHECK if ((!skipRootCheck)&&(getuid() != 0)) { -- cgit v1.2.3 From 35ac995d0510b4bacf1bd813641e4403519f3122 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 14:04:27 -0700 Subject: Fix setting of v6AssignMode in controller. --- controller/EmbeddedNetworkController.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 45af30be..68990536 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1330,8 +1330,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } else if (b["v6AssignMode"].is_object()) { json &v6m = b["v6AssignMode"]; if (v6m.count("rfc4193")) nv6m["rfc4193"] = _jB(v6m["rfc4193"],false); - if (v6m.count("zt")) nv6m["rfc4193"] = _jB(v6m["zt"],false); - if (v6m.count("6plane")) nv6m["rfc4193"] = _jB(v6m["6plane"],false); + if (v6m.count("zt")) nv6m["zt"] = _jB(v6m["zt"],false); + if (v6m.count("6plane")) nv6m["6plane"] = _jB(v6m["6plane"],false); } if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; if (!nv6m.count("zt")) nv6m["zt"] = false; -- cgit v1.2.3 From 297b1b4258f6ec59f08590170e0dbad22dd6e790 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 14:16:55 -0700 Subject: Another tiny API bug fix. --- controller/EmbeddedNetworkController.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 68990536..c14fc101 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1319,6 +1319,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( std::vector v6m(Utils::split(_jS(b["v6AssignMode"],"").c_str(),",","","")); std::sort(v6m.begin(),v6m.end()); v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); + nv6m["rfc4193"] = false; + nv6m["zt"] = false; + nv6m["6plane"] = false; for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { if (*i == "rfc4193") nv6m["rfc4193"] = true; -- cgit v1.2.3 From 77c2bf3ad901b607c9097c58a20a529ef6f0d042 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 14:47:19 -0700 Subject: Kill dead field from network JSON. --- controller/EmbeddedNetworkController.hpp | 1 - 1 file changed, 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 5e942a7b..303a5355 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -165,7 +165,6 @@ private: if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; - if (!network.count("activeBridges")) network["activeBridges"] = nlohmann::json::array(); if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); if (!network.count("rules")) { -- cgit v1.2.3 From 914c42537c33a6ebf7952dd69587e3d1aa2b7e99 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 17:48:36 -0700 Subject: Type fixes. --- controller/EmbeddedNetworkController.cpp | 48 ++++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c14fc101..4dad4311 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -137,14 +137,14 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_ACTION_TEE: r["type"] = "ACTION_TEE"; r["address"] = Address(rule.v.fwd.address).toString(); - r["flags"] = (uint64_t)rule.v.fwd.flags; - r["length"] = (uint64_t)rule.v.fwd.length; + r["flags"] = (unsigned int)rule.v.fwd.flags; + r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; r["address"] = Address(rule.v.fwd.address).toString(); - r["flags"] = (uint64_t)rule.v.fwd.flags; - r["length"] = (uint64_t)rule.v.fwd.length; + r["flags"] = (unsigned int)rule.v.fwd.flags; + r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: r["type"] = "ACTION_DEBUG_LOG"; @@ -162,22 +162,22 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_MATCH_VLAN_ID: r["type"] = "MATCH_VLAN_ID"; r["not"] = ((rule.t & 0x80) != 0); - r["vlanId"] = (uint64_t)rule.v.vlanId; + r["vlanId"] = (unsigned int)rule.v.vlanId; break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: r["type"] = "MATCH_VLAN_PCP"; r["not"] = ((rule.t & 0x80) != 0); - r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; + r["vlanPcp"] = (unsigned int)rule.v.vlanPcp; break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: r["type"] = "MATCH_VLAN_DEI"; r["not"] = ((rule.t & 0x80) != 0); - r["vlanDei"] = (uint64_t)rule.v.vlanDei; + r["vlanDei"] = (unsigned int)rule.v.vlanDei; break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: r["type"] = "MATCH_ETHERTYPE"; r["not"] = ((rule.t & 0x80) != 0); - r["etherType"] = (uint64_t)rule.v.etherType; + r["etherType"] = (unsigned int)rule.v.etherType; break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; @@ -214,24 +214,24 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_MATCH_IP_TOS: r["type"] = "MATCH_IP_TOS"; r["not"] = ((rule.t & 0x80) != 0); - r["ipTos"] = (uint64_t)rule.v.ipTos; + r["ipTos"] = (unsigned int)rule.v.ipTos; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: r["type"] = "MATCH_IP_PROTOCOL"; r["not"] = ((rule.t & 0x80) != 0); - r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; + r["ipProtocol"] = (unsigned int)rule.v.ipProtocol; break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (uint64_t)rule.v.port[0]; - r["end"] = (uint64_t)rule.v.port[1]; + r["start"] = (unsigned int)rule.v.port[0]; + r["end"] = (unsigned int)rule.v.port[1]; break; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: r["type"] = "MATCH_IP_DEST_PORT_RANGE"; r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (uint64_t)rule.v.port[0]; - r["end"] = (uint64_t)rule.v.port[1]; + r["start"] = (unsigned int)rule.v.port[0]; + r["end"] = (unsigned int)rule.v.port[1]; break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; @@ -244,32 +244,32 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: r["type"] = "MATCH_FRAME_SIZE_RANGE"; r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (uint64_t)rule.v.frameSize[0]; - r["end"] = (uint64_t)rule.v.frameSize[1]; + r["start"] = (unsigned int)rule.v.frameSize[0]; + r["end"] = (unsigned int)rule.v.frameSize[1]; break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: r["type"] = "MATCH_TAGS_SAMENESS"; r["not"] = ((rule.t & 0x80) != 0); - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: r["type"] = "MATCH_TAGS_BITWISE_AND"; r["not"] = ((rule.t & 0x80) != 0); - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: r["type"] = "MATCH_TAGS_BITWISE_OR"; r["not"] = ((rule.t & 0x80) != 0); - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: r["type"] = "MATCH_TAGS_BITWISE_XOR"; r["not"] = ((rule.t & 0x80) != 0); - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; break; } return r; -- cgit v1.2.3 From 7a00036954b46d3b870d0658f2860a4b27acb601 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 18:10:02 -0700 Subject: Tweak log length to fit JSON for members within two 4096-kb blocks. --- controller/EmbeddedNetworkController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 4dad4311..5c52cfcb 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -50,7 +50,7 @@ using json = nlohmann::json; #define ZT_NETCONF_CONTROLLER_API_VERSION 3 // Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 32 +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 24 // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 -- cgit v1.2.3 From 8e3004591bae72e20232200b74fe145f5574d02e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 14:01:15 -0700 Subject: Add overlooked MATCH_ICMP to rule set. --- controller/EmbeddedNetworkController.cpp | 20 ++++++++++++ include/ZeroTierOne.h | 30 +++++++++++++----- node/Capability.hpp | 11 +++++++ node/Network.cpp | 52 ++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 8 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5c52cfcb..0a1ee2ef 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -221,6 +221,14 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["not"] = ((rule.t & 0x80) != 0); r["ipProtocol"] = (unsigned int)rule.v.ipProtocol; break; + case ZT_NETWORK_RULE_MATCH_ICMP: + r["type"] = "MATCH_ICMP"; + r["not"] = ((rule.t & 0x80) != 0); + r["type"] = (unsigned int)rule.v.icmp.type; + if ((rule.v.icmp.flags & 0x01) != 0) + r["code"] = (unsigned int)rule.v.icmp.code; + else r["code"] = json(); + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; r["not"] = ((rule.t & 0x80) != 0); @@ -375,6 +383,18 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; rule.v.ipProtocol = (uint8_t)(_jI(r["ipProtocol"],0ULL) & 0xffULL); return true; + } else if (t == "MATCH_ICMP") { + rule.t |= ZT_NETWORK_RULE_MATCH_ICMP; + rule.v.icmp.type = (uint8_t)(_jI(r["type"],0ULL) & 0xffULL); + json &code = r["code"]; + if (code.is_null()) { + rule.v.icmp.code = 0; + rule.v.icmp.flags = 0x00; + } else { + rule.v.icmp.code = (uint8_t)(_jI(code,0ULL) & 0xffULL); + rule.v.icmp.flags = 0x01; + } + return true; } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 9dafcaa6..121de7a2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -597,45 +597,50 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_MATCH_IP_PROTOCOL = 45, + /** + * ICMP type and possibly code (does not match if not ICMP) + */ + ZT_NETWORK_RULE_MATCH_ICMP = 46, + /** * IP source port range (start-end, inclusive) */ - ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 46, + ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 47, /** * IP destination port range (start-end, inclusive) */ - ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 47, + ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 48, /** * Packet characteristics (set of flags) */ - ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 48, + ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 49, /** * Frame size range (start-end, inclusive) */ - ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 49, + ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 50, /** * Match if local and remote tags differ by no more than value, use 0 to check for equality */ - ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS = 50, + ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS = 51, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 51, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 52, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 52, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 53, /** * Match if local and remote tags XORed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 53 + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 54 }; /** @@ -739,6 +744,15 @@ typedef struct */ uint16_t frameSize[2]; + /** + * ICMP type and code + */ + struct { + uint8_t type; // ICMP type, always matched + uint8_t code; // ICMP code if matched + uint8_t flags; // flag 0x01 means also match code, otherwise only match type + } icmp; + /** * For tag-related rules */ diff --git a/node/Capability.hpp b/node/Capability.hpp index 46fa9da6..f62ed30b 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -233,6 +233,12 @@ public: b.append((uint8_t)1); b.append((uint8_t)rules[i].v.ipProtocol); break; + case ZT_NETWORK_RULE_MATCH_ICMP: + b.append((uint8_t)3); + b.append((uint8_t)rules[i].v.icmp.type); + b.append((uint8_t)rules[i].v.icmp.code); + b.append((uint8_t)rules[i].v.icmp.flags); + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: b.append((uint8_t)4); @@ -312,6 +318,11 @@ public: case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: rules[ruleCount].v.ipProtocol = (uint8_t)b[p]; break; + case ZT_NETWORK_RULE_MATCH_ICMP: + rules[ruleCount].v.icmp.type = (uint8_t)b[p]; + rules[ruleCount].v.icmp.code = (uint8_t)b[p+1]; + rules[ruleCount].v.icmp.flags = (uint8_t)b[p+2]; + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: rules[ruleCount].v.port[0] = b.template at(p); diff --git a/node/Network.cpp b/node/Network.cpp index 0279bc53..026a07fc 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -64,6 +64,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_IPV6_DEST: return "MATCH_IPV6_DEST"; case ZT_NETWORK_RULE_MATCH_IP_TOS: return "MATCH_IP_TOS"; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: return "MATCH_IP_PROTOCOL"; + case ZT_NETWORK_RULE_MATCH_ICMP: return "MATCH_ICMP"; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: return "MATCH_IP_SOURCE_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: return "MATCH_CHARACTERISTICS"; @@ -362,6 +363,57 @@ static int _doZtFilter( FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; + case ZT_NETWORK_RULE_MATCH_ICMP: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + if (frameData[9] == 0x01) { + const unsigned int ihl = (frameData[0] & 0xf) * 32; + if (frameLen >= (ihl + 2)) { + if (rules[rn].v.icmp.type == frameData[ihl]) { + if ((rules[rn].v.icmp.flags & 0x01) != 0) { + thisRuleMatches = (uint8_t)(frameData[ihl+1] == rules[rn].v.icmp.code); + } else { + thisRuleMatches = 1; + } + } else { + thisRuleMatches = 0; + } + FILTER_TRACE("%u %s %c (IPv4) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[ihl],(int)rules[rn].v.icmp.type,(int)frameData[ihl+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [IPv4 frame invalid] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not ICMP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else if (etherType == ZT_ETHERTYPE_IPV6) { + unsigned int pos = 0,proto = 0; + if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { + if ((proto == 0x3a)&&(frameLen >= (pos+2))) { + if (rules[rn].v.icmp.type == frameData[pos]) { + if ((rules[rn].v.icmp.flags & 0x01) != 0) { + thisRuleMatches = (uint8_t)(frameData[pos+1] == rules[rn].v.icmp.code); + } else { + thisRuleMatches = 1; + } + } else { + thisRuleMatches = 0; + } + FILTER_TRACE("%u %s %c (IPv4) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[pos],(int)rules[rn].v.icmp.type,(int)frameData[pos+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not ICMPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + break; + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { -- cgit v1.2.3 From 54489a7f61a19b07eaa5a87d1df2ee30101f29ee Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 14:14:58 -0700 Subject: rename SAMENESS to DIFFERENCE which is less confusing --- controller/EmbeddedNetworkController.cpp | 8 +++---- include/ZeroTierOne.h | 2 +- node/Capability.hpp | 21 ++++++------------ node/Network.cpp | 37 ++++++++++++++++++-------------- 4 files changed, 33 insertions(+), 35 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0a1ee2ef..29dd8ad7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -255,8 +255,8 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["start"] = (unsigned int)rule.v.frameSize[0]; r["end"] = (unsigned int)rule.v.frameSize[1]; break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: - r["type"] = "MATCH_TAGS_SAMENESS"; + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: + r["type"] = "MATCH_TAGS_DIFFERENCE"; r["not"] = ((rule.t & 0x80) != 0); r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; @@ -431,8 +431,8 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.frameSize[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); rule.v.frameSize[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; - } else if (t == "MATCH_TAGS_SAMENESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; + } else if (t == "MATCH_TAGS_DIFFERENCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 121de7a2..73450006 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -625,7 +625,7 @@ enum ZT_VirtualNetworkRuleType /** * Match if local and remote tags differ by no more than value, use 0 to check for equality */ - ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS = 51, + ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 51, /** * Match if local and remote tags ANDed together equal value. diff --git a/node/Capability.hpp b/node/Capability.hpp index f62ed30b..8e749e80 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -38,9 +38,6 @@ class RuntimeEnvironment; /** * A set of grouped and signed network flow rules * - * The use of capabilities implements capability-based security on ZeroTIer - * virtual networks for efficient and manageable network micro-segmentation. - * * On the sending side the sender does the following for each packet: * * (1) Evaluates its capabilities in ascending order of ID to determine @@ -49,16 +46,12 @@ class RuntimeEnvironment; * receving peer ("presents" it). * (3) The sender then sends the packet. * - * On the receiving side the receiver does the following for each packet: - * - * (1) Evaluates the capabilities of the sender (that the sender has - * presented) to determine if it should received this packet. - * (2) Evaluates its own capabilities to determine if it should receive - * this packet. - * (3) If both check out, it receives the packet. + * On the receiving side the receiver evaluates the capabilities presented + * by the sender. If any valid un-expired capability allows this packet it + * is accepted. * - * Note that rules in capabilities can do other things as well such as TEE - * or REDIRECT packets. See filter code and ZT_VirtualNetworkRule. + * Note that this is after evaluation of network scope rules and only if + * network scope rules do not deliver an explicit match. */ class Capability { @@ -255,7 +248,7 @@ public: b.append((uint16_t)rules[i].v.frameSize[0]); b.append((uint16_t)rules[i].v.frameSize[1]); break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: @@ -336,7 +329,7 @@ public: rules[ruleCount].v.frameSize[0] = b.template at(p); rules[ruleCount].v.frameSize[1] = b.template at(p + 2); break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: diff --git a/node/Network.cpp b/node/Network.cpp index 026a07fc..f8b7c1d5 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -69,7 +69,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: return "MATCH_CHARACTERISTICS"; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: return "MATCH_FRAME_SIZE_RANGE"; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: return "MATCH_TAGS_SAMENESS"; + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: return "MATCH_TAGS_DIFFERENCE"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: return "MATCH_TAGS_BITWISE_AND"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: return "MATCH_TAGS_BITWISE_OR"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: return "MATCH_TAGS_BITWISE_XOR"; @@ -487,7 +487,7 @@ static int _doZtFilter( thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); FILTER_TRACE("%u %s %c %u in %u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),frameLen,(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1],(unsigned int)thisRuleMatches); break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { @@ -510,13 +510,18 @@ static int _doZtFilter( } } if (!rtv) { - thisRuleMatches = 0; - FILTER_TRACE("%u %s %c remote tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + if (inbound) { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } else { + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } } else { - if (rt == ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS) { - const uint32_t sameness = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); - thisRuleMatches = (uint8_t)(sameness <= rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u sameness:%u <= %u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,sameness,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); + if (rt == ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE) { + const uint32_t diff = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); + thisRuleMatches = (uint8_t)(diff <= rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u difference:%u<=%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,diff,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { thisRuleMatches = (uint8_t)((lt->value() & *rtv) == rules[rn].v.tag.value); FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); @@ -675,22 +680,22 @@ int Network::filterIncomingPacket( const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; - case 1: return 1; - case 2: return 2; + case -1: return 0; // DROP + case 1: return 1; // ACCEPT + case 2: return 2; // super-ACCEPT } Membership::CapabilityIterator mci(m); const Capability *c; while ((c = mci.next(_config))) { - switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; - case 1: return 1; - case 2: return 2; + switch(_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { + case -1: return 0; // DROP + case 1: return 1; // ACCEPT + case 2: return 2; // super-ACCEPT } } - return 0; + return 0; // DROP } bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const -- cgit v1.2.3 From 74afef8eb1d96aec291c6dfeca31e69a2ad33d69 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 16:50:22 -0700 Subject: Think through and refine a few things in rules, especially edge case TEE and REDIRECT behavior and semantics. --- controller/EmbeddedNetworkController.cpp | 2 - node/Capability.hpp | 2 +- node/IncomingPacket.cpp | 6 +- node/Network.cpp | 290 +++++++++++++++++++++++-------- node/Network.hpp | 11 +- node/OutboundMulticast.cpp | 5 +- node/Packet.hpp | 5 +- node/Switch.cpp | 5 +- 8 files changed, 229 insertions(+), 97 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 29dd8ad7..ff2f34ec 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -144,7 +144,6 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["type"] = "ACTION_REDIRECT"; r["address"] = Address(rule.v.fwd.address).toString(); r["flags"] = (unsigned int)rule.v.fwd.flags; - r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: r["type"] = "ACTION_DEBUG_LOG"; @@ -308,7 +307,6 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); - rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "ACTION_DEBUG_LOG") { rule.t |= ZT_NETWORK_RULE_ACTION_DEBUG_LOG; diff --git a/node/Capability.hpp b/node/Capability.hpp index 8e749e80..e23d7943 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -178,7 +178,7 @@ public: b.append((uint8_t)14); b.append((uint64_t)rules[i].v.fwd.address); b.append((uint32_t)rules[i].v.fwd.flags); - b.append((uint16_t)rules[i].v.fwd.length); + b.append((uint16_t)rules[i].v.fwd.length); // unused for redirect break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c75125d3..4b013078 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -560,7 +560,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr const MAC sourceMac(peer->address(),network->id()); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0)) + if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true); } @@ -625,7 +625,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); break; @@ -981,7 +981,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share } const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); - if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0)) { + if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { RR->node->putFrame(network->id(),network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); } } diff --git a/node/Network.cpp b/node/Network.cpp index f8b7c1d5..5a9b07cf 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -76,15 +76,14 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) default: return "BAD_RULE_TYPE"; } } -static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool noRedirect,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) +static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) { static volatile unsigned long cnt = 0; - printf("%.6lu %c %s inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + printf("%.6lu %c %s %s frameLen=%u etherType=%u" ZT_EOL_S, cnt++, ((thisSetMatches) ? 'Y' : '.'), ruleName, - (int)inbound, - (int)noRedirect, + ((inbound) ? "INBOUND" : "OUTBOUND"), frameLen, etherType ); @@ -143,14 +142,20 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig return false; // overflow == invalid } -// 0 == no match, -1 == match/drop, 1 == match/accept, 2 == match/accept even if bridged -static int _doZtFilter( +enum _doZtFilterResult +{ + DOZTFILTER_NO_MATCH = 0, + DOZTFILTER_DROP = 1, + DOZTFILTER_REDIRECT = 2, + DOZTFILTER_ACCEPT = 3, + DOZTFILTER_SUPER_ACCEPT = 4 +}; +static _doZtFilterResult _doZtFilter( const RuntimeEnvironment *RR, - const bool noRedirect, const NetworkConfig &nconf, const bool inbound, const Address &ztSource, - const Address &ztDest, + Address &ztDest, // MUTABLE const MAC &macSource, const MAC &macDest, const uint8_t *const frameData, @@ -163,7 +168,9 @@ static int _doZtFilter( const unsigned int localTagCount, const uint32_t *const remoteTagIds, const uint32_t *const remoteTagValues, - const unsigned int remoteTagCount) + const unsigned int remoteTagCount, + Address &cc, // MUTABLE + unsigned int &ccLength) // MUTABLE { // For each set of rules we start by assuming that they match (since no constraints // yields a 'match all' rule). @@ -181,75 +188,83 @@ static int _doZtFilter( case ZT_NETWORK_RULE_ACTION_DROP: if (thisSetMatches) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DROP",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_DROP",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - return -1; // match, drop packet + return DOZTFILTER_DROP; } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DROP",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_DROP",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // no match, evaluate next set + thisSetMatches = 1; } continue; + case ZT_NETWORK_RULE_ACTION_ACCEPT: if (thisSetMatches) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - return 1; // match, accept packet + return DOZTFILTER_ACCEPT; // match, accept packet } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // no match, evaluate next set + thisSetMatches = 1; } continue; + case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { const Address fwdAddr(rules[rn].v.fwd.address); - if (fwdAddr == RR->identity.address()) { - // If we are the TEE or REDIRECT destination, don't TEE or REDIRECT - // to self. We should also accept here instead of interpreting - // REDIRECT as DROP since we are the destination. + if (fwdAddr == ztSource) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT resulted in 'super-accept' since we are destination"); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored since source is target"); #endif // ZT_RULES_ENGINE_DEBUGGING - return 2; // we should "super-accept" this packet since we are the TEE or REDIRECT destination - } else { - if (!noRedirect) { - Packet outp(fwdAddr,RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(nconf.networkId); - outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); - macDest.appendTo(outp); - macSource.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); - outp.compress(); - RR->sw->send(outp,true); + thisSetMatches = 1; + } else if (fwdAddr == RR->identity.address()) { + if (inbound) { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT interpreted as super-accept since we are target"); +#endif // ZT_RULES_ENGINE_DEBUGGING + return DOZTFILTER_SUPER_ACCEPT; + } else { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored on outbound since we are target"); +#endif // ZT_RULES_ENGINE_DEBUGGING + thisSetMatches = 1; } - + } else if (fwdAddr == ztDest) { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored since destination is target"); +#endif // ZT_RULES_ENGINE_DEBUGGING + thisSetMatches = 1; + } else { if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(noRedirect) ? "second-pass match, not actually redirecting" : (const char *)0); + _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - return -1; // match, drop packet (we redirected it) + ztDest = fwdAddr; + return DOZTFILTER_REDIRECT; } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_TEE",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(noRedirect) ? "second-pass match, not actually teeing" : (const char *)0); + _dumpFilterTrace("ACTION_TEE",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // TEE does not terminate evaluation + cc = fwdAddr; + ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen; + thisSetMatches = 1; } } } continue; + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: // a no-op target specifically for debugging purposes #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation + thisSetMatches = 1; continue; default: break; @@ -547,7 +562,7 @@ static int _doZtFilter( thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); } - return 0; + return DOZTFILTER_NO_MATCH; } const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); @@ -614,7 +629,7 @@ Network::~Network() } bool Network::filterOutgoingPacket( - const bool noRedirect, + const bool noTee, const Address &ztSource, const Address &ztDest, const MAC &macSource, @@ -626,39 +641,94 @@ bool Network::filterOutgoingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + Address ztDest2(ztDest); + Address cc; + unsigned int ccLength = 0; + bool mainRuleTableMatch = false; + bool accept = false; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch(_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); + switch(_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + case DOZTFILTER_NO_MATCH: + break; + case DOZTFILTER_DROP: return false; - case 1: - case 2: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); - return true; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side + mainRuleTableMatch = true; + accept = true; + break; } - for(unsigned int c=0;c<_config.capabilityCount;++c) { - switch (_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); - return false; - case 1: - case 2: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,&(_config.capabilities[c])); - return true; + const Capability *relevantCap = (const Capability *)0; + if (!mainRuleTableMatch) { + for(unsigned int c=0;c<_config.capabilityCount;++c) { + ztDest2 = ztDest; // sanity check + Address cc2; + unsigned int ccLength2 = 0; + switch (_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + case DOZTFILTER_NO_MATCH: + case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern + break; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side + if ((!noTee)&&(cc2)) { + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength2); + outp.compress(); + RR->sw->send(outp,true); + } + relevantCap = &(_config.capabilities[c]); + accept = true; + break; + } + if (accept) + break; } } - return false; + if (accept) { + if (ztDest2) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); + + if ((!noTee)&&(cc)) { + Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength); + outp.compress(); + RR->sw->send(outp,true); + } + + if (ztDest != ztDest2) { + Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,frameLen); + outp.compress(); + RR->sw->send(outp,true); + return false; // DROP locally, since we redirected + } + } + + return accept; } int Network::filterIncomingPacket( @@ -673,29 +743,97 @@ int Network::filterIncomingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + Address ztDest2(ztDest); + Address cc; + unsigned int ccLength = 0; + bool mainRuleTableMatch = false; + int accept = 0; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; // DROP - case 1: return 1; // ACCEPT - case 2: return 2; // super-ACCEPT + switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + case DOZTFILTER_NO_MATCH: + break; + case DOZTFILTER_DROP: + return 0; // DROP + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + mainRuleTableMatch = true; + accept = 1; // ACCEPT + break; + case DOZTFILTER_SUPER_ACCEPT: + mainRuleTableMatch = true; + accept = 2; // super-ACCEPT + break; } - Membership::CapabilityIterator mci(m); - const Capability *c; - while ((c = mci.next(_config))) { - switch(_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; // DROP - case 1: return 1; // ACCEPT - case 2: return 2; // super-ACCEPT + if (!mainRuleTableMatch) { + Membership::CapabilityIterator mci(m); + const Capability *c; + while ((c = mci.next(_config))) { + ztDest2 = ztDest; // sanity check + Address cc2; + unsigned int ccLength2 = 0; + switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + case DOZTFILTER_NO_MATCH: + case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern + break; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + accept = 1; // ACCEPT + break; + case DOZTFILTER_SUPER_ACCEPT: + accept = 2; // super-ACCEPT + break; + } + if (accept) { + if (cc2) { + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength2); + outp.compress(); + RR->sw->send(outp,true); + } + break; + } + } + } + + if (accept) { + if (cc) { + Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength); + outp.compress(); + RR->sw->send(outp,true); + } + + if (ztDest != ztDest2) { + Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,frameLen); + outp.compress(); + RR->sw->send(outp,true); + return 0; // DROP locally, since we redirected } } - return 0; // DROP + return accept; } bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const diff --git a/node/Network.hpp b/node/Network.hpp index aa4b67f8..45a51bf2 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -82,11 +82,10 @@ public: * Apply filters to an outgoing packet * * This applies filters from our network config and, if that doesn't match, - * our capabilities in ascending order of capability ID. If there is a match - * certain actions may be taken such as pushing credentials to ztDest and - * sending a copy of the packet to a TEE or REDIRECT target. + * our capabilities in ascending order of capability ID. Additional actions + * such as TEE may be taken, and credentials may be pushed. * - * @param noRedirect If true, do not TEE or REDIRECT -- this is set for secondary filtrations done in multicast and bridge send paths + * @param noTee If true, do not TEE anything anywhere * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -95,10 +94,10 @@ public: * @param frameLen Ethernet frame payload length * @param etherType 16-bit ethernet type ID * @param vlanId 16-bit VLAN ID - * @return True if packet should be sent to destination peer + * @return True if packet should be sent, false if dropped or redirected */ bool filterOutgoingPacket( - const bool noRedirect, + const bool noTee, const Address &ztSource, const Address &ztDest, const MAC &macSource, diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 6b583e7c..33c28f65 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -86,10 +86,11 @@ void OutboundMulticast::init( void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) { const SharedPtr nw(RR->node->network(_nwid)); - if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { + Address toAddr2(toAddr); + if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); - _packet.setDestination(toAddr); + _packet.setDestination(toAddr2); RR->sw->send(_packet,true); } } diff --git a/node/Packet.hpp b/node/Packet.hpp index 570bace9..27e289fd 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -655,9 +655,8 @@ public: * * Flags: * 0x01 - Certificate of network membership attached (DEPRECATED) - * 0x02 - Packet is a TEE'd packet - * 0x04 - Packet is a REDIRECT'ed packet - * 0x08 - TEE/REDIRECT'ed packet is on inbound side of connection + * 0x02 - This is a TEE'd or REDIRECT'ed packet + * 0x04 - TEE/REDIRECT'ed packet is from inbound side * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we diff --git a/node/Switch.cpp b/node/Switch.cpp index 546c9157..8e41c89f 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -437,10 +437,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),mg.toString().c_str(),etherTypeName(etherType),len); - // We filter with a NULL destination ZeroTier address first. Filtrations - // for each ZT destination are also done in OutboundMulticast, but these - // set noRedirect to true. This prevents multiple TEEs and REDIRECTs for - // multicast packets. + // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; -- cgit v1.2.3 From c9ee8612e496d833b287f00c548f76ee5879bfef Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 12:12:52 -0700 Subject: Credential TTL (tags/capabilities) should be credential time max delta, since we could get pushed one that is newer. --- controller/EmbeddedNetworkController.cpp | 12 ++++++------ node/Membership.hpp | 5 +++-- node/NetworkConfig.cpp | 4 ++-- node/NetworkConfig.hpp | 16 ++++++++-------- 4 files changed, 19 insertions(+), 18 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index ff2f34ec..cf6bd7c9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -654,16 +654,16 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // for both.) This is computed by reference to the last time we deauthorized // a member, since within the time period since this event any temporal // differences are not particularly relevant. - uint64_t credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL; + uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; if (now > nmi.mostRecentDeauthTime) - credentialTtl += (now - nmi.mostRecentDeauthTime); - if (credentialTtl > ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL) - credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL; + credentialtmd += (now - nmi.mostRecentDeauthTime); + if (credentialtmd > ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) + credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; nc.networkId = nwid; nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; - nc.credentialTimeToLive = credentialTtl; + nc.credentialTimeMaxDelta = credentialtmd; nc.revision = _jI(network["revision"],0ULL); nc.issuedTo = identity.address(); if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; @@ -925,7 +925,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } if (_jB(network["private"],true)) { - CertificateOfMembership com(now,credentialTtl,nwid,identity.address()); + CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); if (com.sign(signingId)) { nc.com = com; } else { diff --git a/node/Membership.hpp b/node/Membership.hpp index 5e5efc50..209f6158 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -144,7 +144,7 @@ public: } /** - * Check whether a capability or tag is expired + * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time * * @param cred Credential to check -- must have timestamp() accessor method * @return True if credential is NOT expired @@ -153,7 +153,8 @@ public: inline bool isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred) const { const uint64_t ts = cred.timestamp(); - return ( ( (ts >= nconf.timestamp) || ((nconf.timestamp - ts) <= nconf.credentialTimeToLive) ) && (ts > _blacklistBefore) ); + const uint64_t delta = (ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts); + return ((delta <= nconf.credentialTimeMaxDelta)&&(ts > _blacklistBefore)); } /** diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 0c9c05ca..6acc48ea 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -37,7 +37,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,this->networkId)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,this->credentialTimeToLive)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; @@ -193,7 +193,7 @@ bool NetworkConfig::fromDictionary(const Dictionarytimestamp = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,0); - this->credentialTimeToLive = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,0); + this->credentialTimeMaxDelta = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,0); this->revision = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REVISION,0); this->issuedTo = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,0); if (!this->issuedTo) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index e2bacb07..b5ab9ccb 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -41,12 +41,12 @@ #include "Identity.hpp" /** - * Default maximum credential TTL and maxDelta for COM timestamps + * Default maximum time delta for COMs, tags, and capabilities * * The current value is two hours, providing ample time for a controller to * experience fail-over, etc. */ -#define ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL 7200000ULL +#define ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA 7200000ULL /** * Default minimum credential TTL and maxDelta for COM timestamps @@ -54,7 +54,7 @@ * This is just slightly over three minutes and provides three retries for * all currently online members to refresh. */ -#define ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL 185000ULL +#define ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA 185000ULL /** * Flag: allow passive bridging (experimental) @@ -148,8 +148,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_TYPE "t" // text #define ZT_NETWORKCONFIG_DICT_KEY_NAME "n" -// credential time to live in ms -#define ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL "cttl" +// credential time max delta in ms +#define ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA "ctmd" // binary serialized certificate of membership #define ZT_NETWORKCONFIG_DICT_KEY_COM "C" // specialists (binary array of uint64_t) @@ -372,7 +372,7 @@ public: { printf("networkId==%.16llx\n",networkId); printf("timestamp==%llu\n",timestamp); - printf("credentialTimeToLive==%llu\n",credentialTimeToLive); + printf("credentialTimeMaxDelta==%llu\n",credentialTimeMaxDelta); printf("revision==%llu\n",revision); printf("issuedTo==%.10llx\n",issuedTo.toInt()); printf("multicastLimit==%u\n",multicastLimit); @@ -407,9 +407,9 @@ public: uint64_t timestamp; /** - * TTL for capabilities and tags + * Max difference between timestamp and tag/capability timestamp */ - uint64_t credentialTimeToLive; + uint64_t credentialTimeMaxDelta; /** * Controller-side revision counter for this configuration -- cgit v1.2.3 From 0d4109a9f1f119e336d73039251ad17c0e2a56f4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 9 Sep 2016 08:43:58 -0700 Subject: More refactoring to clean up code, and add a gate function to make sure we do not handle OK packets we did not expect. This hardens up a few potential edge cases around security, since such messages might be used to e.g. pollute a cache and DOS under certain conditions. --- controller/EmbeddedNetworkController.cpp | 2 - include/ZeroTierOne.h | 12 +-- node/IncomingPacket.cpp | 156 ++++++++++++++++++------------- node/Membership.hpp | 6 +- node/Multicaster.cpp | 1 + node/Network.cpp | 10 ++ node/Network.hpp | 6 ++ node/Node.cpp | 3 + node/Node.hpp | 34 +++++++ node/OutboundMulticast.cpp | 1 + node/Packet.hpp | 5 +- node/Peer.cpp | 2 + node/Switch.cpp | 11 +-- 13 files changed, 168 insertions(+), 81 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index cf6bd7c9..79560dcc 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1585,7 +1585,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes "\t\"upstream\": \"%.10llx\"," ZT_EOL_S "\t\"current\": \"%.10llx\"," ZT_EOL_S "\t\"receivedTimestamp\": %llu," ZT_EOL_S - "\t\"remoteTimestamp\": %llu," ZT_EOL_S "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S "\t\"flags\": %llu," ZT_EOL_S "\t\"sourcePacketHopCount\": %u," ZT_EOL_S @@ -1606,7 +1605,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes (unsigned long long)report->upstream, (unsigned long long)report->current, (unsigned long long)OSUtils::now(), - (unsigned long long)report->remoteTimestamp, (unsigned long long)report->sourcePacketId, (unsigned long long)report->flags, report->sourcePacketHopCount, diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 0c22ae9d..633db7cf 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -154,6 +154,11 @@ extern "C" { */ #define ZT_CIRCUIT_TEST_MAX_HOP_BREADTH 8 +/** + * Circuit test report flag: upstream peer authorized in path (e.g. by network COM) + */ +#define ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH 0x0000000000000001ULL + /** * Maximum number of cluster members (and max member ID plus one) */ @@ -1218,18 +1223,13 @@ typedef struct { */ uint64_t timestamp; - /** - * Timestamp on remote device - */ - uint64_t remoteTimestamp; - /** * 64-bit packet ID of packet received by the reporting device */ uint64_t sourcePacketId; /** - * Flags (currently unused, will be zero) + * Flags */ uint64_t flags; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ac04ce96..c8364415 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -156,6 +156,17 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; + case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { + SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + if ((network)&&(network->recentlyAllowedOnNetwork(peer))) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + network->config().com.serialize(outp); + outp.append((uint8_t)0); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); + } + } break; + case Packet::ERROR_NETWORK_ACCESS_DENIED_: { SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) @@ -163,10 +174,12 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr } break; case Packet::ERROR_UNWANTED_MULTICAST: { - uint64_t nwid = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); - MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); - TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",nwid,peer->address().toString().c_str(),mg.toString().c_str()); - RR->mc->remove(nwid,mg,peer->address()); + SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + if ((network)&&(network->gate(peer,verb(),packetId()))) { + MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); + TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str()); + RR->mc->remove(network->id(),mg,peer->address()); + } } break; default: break; @@ -352,7 +365,12 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); - //TRACE("%s(%s): OK(%s)",source().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); + if (!RR->node->expectingReplyTo(inRePacketId)) { + TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId()); + return true; + } + + //TRACE("%s(%s): OK(%s)",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); switch(inReVerb) { @@ -424,10 +442,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); - const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); - //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); - const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + SharedPtr network(RR->node->network(nwid)); + if ((network)&&(network->gate(peer,verb(),packetId()))) { + const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); + //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); + const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); + RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + } } break; case Packet::VERB_MULTICAST_FRAME: { @@ -437,24 +458,26 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p //TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags); - unsigned int offset = 0; + SharedPtr network(RR->node->network(nwid)); + if (network) { + unsigned int offset = 0; - if ((flags & 0x01) != 0) { // deprecated but still used by older peers - CertificateOfMembership com; - offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); - if (com) { - SharedPtr network(RR->node->network(com.networkId())); - if (network) + if ((flags & 0x01) != 0) { // deprecated but still used by older peers + CertificateOfMembership com; + offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); + if (com) network->addCredential(com); } - } - if ((flags & 0x02) != 0) { - // OK(MULTICAST_FRAME) includes implicit gather results - offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; - unsigned int totalKnown = at(offset); offset += 4; - unsigned int count = at(offset); offset += 2; - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); + if (network->gate(peer,verb(),packetId())) { + if ((flags & 0x02) != 0) { + // OK(MULTICAST_FRAME) includes implicit gather results + offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; + unsigned int totalKnown = at(offset); offset += 4; + unsigned int count = at(offset); offset += 2; + RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); + } + } } } break; @@ -515,27 +538,29 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - const SharedPtr rendezvousWith(RR->topology->getPeer(with)); - if (rendezvousWith) { - const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); - const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; - if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { - const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - if (!RR->topology->isUpstream(peer->identity())) { - TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); - } else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { - RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now()); - TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + if (!RR->topology->isUpstream(peer->identity())) { + TRACE("RENDEZVOUS from %s ignored since source is not upstream",peer->address().toString().c_str()); + } else { + const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); + const SharedPtr rendezvousWith(RR->topology->getPeer(with)); + if (rendezvousWith) { + const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); + const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; + if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { + const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); + if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { + RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now()); + TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + } else { + TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + } } else { - TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str()); } } else { - TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str()); + TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } - } else { - TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { @@ -549,25 +574,25 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr try { const uint64_t nwid = at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); - bool approved = false; + bool trustEstablished = false; if (network) { - if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { - if (!network->gate(peer,verb(),packetId())) { - TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - } else { + if (!network->gate(peer,verb(),packetId())) { + TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + } else { + trustEstablished = true; + if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); const MAC sourceMac(peer->address(),nwid); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); - approved = true; // this means approved on the network in general, not this packet per se } } } else { TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } - peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,approved); + peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -580,23 +605,23 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); if (network) { - if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { - const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS]; + const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS]; - unsigned int comLen = 0; - if ((flags & 0x01) != 0) { // deprecated but still used by old peers - CertificateOfMembership com; - comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM); - if (com) - network->addCredential(com); - } + unsigned int comLen = 0; + if ((flags & 0x01) != 0) { // inline COM with EXT_FRAME is deprecated but still used with old peers + CertificateOfMembership com; + comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM); + if (com) + network->addCredential(com); + } - if (!network->gate(peer,verb(),packetId())) { - TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); - return true; - } + if (!network->gate(peer,verb(),packetId())) { + TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + return true; + } + if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { const unsigned int etherType = at(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE); const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO); const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM); @@ -604,7 +629,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC %s",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),from.toString().c_str()); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -1139,6 +1164,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt // Add length of second "additional fields" section. vlf += at(ZT_PACKET_IDX_PAYLOAD + 29 + vlf); + uint64_t reportFlags = 0; + // Check credentials (signature already verified) if (originatorCredentialNetworkId) { SharedPtr network(RR->node->network(originatorCredentialNetworkId)); @@ -1147,6 +1174,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } + if (network->gate(peer,verb(),packetId())) + reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH; } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); @@ -1188,7 +1217,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED); outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED); outp.append((uint16_t)0); // error code, currently unused - outp.append((uint64_t)0); // flags, currently unused + outp.append((uint64_t)reportFlags); outp.append((uint64_t)packetId()); peer->address().appendTo(outp); outp.append((uint8_t)hops()); @@ -1237,7 +1266,6 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S report.upstream = Address(field(ZT_PACKET_IDX_PAYLOAD + 52,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); report.testId = at(ZT_PACKET_IDX_PAYLOAD + 8); report.timestamp = at(ZT_PACKET_IDX_PAYLOAD); - report.remoteTimestamp = at(ZT_PACKET_IDX_PAYLOAD + 16); report.sourcePacketId = at(ZT_PACKET_IDX_PAYLOAD + 44); report.flags = at(ZT_PACKET_IDX_PAYLOAD + 36); report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 57]; // end of fixed length headers: 58 diff --git a/node/Membership.hpp b/node/Membership.hpp index 55355fda..d67c6822 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -163,8 +163,10 @@ public: return true; if (_com) { const uint64_t a = _com.timestamp().first; - const std::pair b(nconf.com.timestamp()); - return ((a <= b.first) ? ((b.first - a) <= ZT_PEER_ACTIVITY_TIMEOUT) : true); + if ((_blacklistBefore)&&(a <= _blacklistBefore)) + return false; + const uint64_t b = nconf.com.timestamp().first; + return ((a <= b) ? ((b - a) <= ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) : true); } return false; } diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index a6bff6aa..36d7d2d0 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -253,6 +253,7 @@ void Multicaster::send( outp.append((uint32_t)gatherLimit); if (com) com->serialize(outp); + RR->node->expectReplyTo(outp.packetId()); RR->sw->send(outp,true); } } diff --git a/node/Network.cpp b/node/Network.cpp index 2a5f213c..710e70dd 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1054,6 +1054,7 @@ void Network::requestConfiguration() } else { outp.append((unsigned char)0,16); } + RR->node->expectReplyTo(outp.packetId()); outp.compress(); RR->sw->send(outp,true); @@ -1092,6 +1093,15 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin return false; } +bool Network::recentlyAllowedOnNetwork(const SharedPtr &peer) const +{ + Mutex::Lock _l(_lock); + const Membership *m = _memberships.get(peer->address()); + if (m) + return m->recentlyAllowedOnNetwork(_config); + return false; +} + void Network::clean() { const uint64_t now = RR->node->now(); diff --git a/node/Network.hpp b/node/Network.hpp index c80f1cba..e8d6e2a5 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -257,6 +257,12 @@ public: */ bool gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); + /** + * @param peer Peer to check + * @return True if peer has recently been a valid member of this network + */ + bool recentlyAllowedOnNetwork(const SharedPtr &peer) const; + /** * Perform cleanup and possibly save state */ diff --git a/node/Node.cpp b/node/Node.cpp index 415385f7..e8279c62 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -75,6 +75,9 @@ Node::Node( { _online = false; + memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); + memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); + // Use Salsa20 alone as a high-quality non-crypto PRNG { char foo[32]; diff --git a/node/Node.hpp b/node/Node.hpp index 3c0a5e92..315b5248 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -44,6 +44,10 @@ #define TRACE(f,...) {} #endif +// Bit mask for "expecting reply" hash +#define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255 +#define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31 + namespace ZeroTier { /** @@ -250,6 +254,33 @@ public: void postCircuitTestReport(const ZT_CircuitTestReport *report); void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); + /** + * Register that we are expecting a reply to a packet ID + * + * @param packetId Packet ID to expect reply to + */ + inline void expectReplyTo(const uint64_t packetId) + { + const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); + _expectingRepliesTo[bucket][_expectingRepliesToBucketPtr[bucket]++ & ZT_EXPECTING_REPLIES_BUCKET_MASK2] = packetId; + } + + /** + * Check whether a given packet ID is something we are expecting a reply to + * + * @param packetId Packet ID to check + * @return True if we're expecting a reply + */ + inline bool expectingReplyTo(const uint64_t packetId) const + { + const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); + for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) { + if (_expectingRepliesTo[bucket][i] == packetId) + return true; + } + return false; + } + private: inline SharedPtr _network(uint64_t nwid) const { @@ -266,6 +297,9 @@ private: void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P + uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1]; + uint64_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1]; + ZT_DataStoreGetFunction _dataStoreGetFunction; ZT_DataStorePutFunction _dataStorePutFunction; ZT_WirePacketSendFunction _wirePacketSendFunction; diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 33c28f65..6e811581 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -91,6 +91,7 @@ void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toA //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); _packet.setDestination(toAddr2); + RR->node->expectReplyTo(_packet.packetId()); RR->sw->send(_packet,true); } } diff --git a/node/Packet.hpp b/node/Packet.hpp index 5ead2c3d..2ca73a84 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -965,7 +965,7 @@ public: * <[2] 16-bit reporter OS/platform or 0 if not specified> * <[2] 16-bit reporter architecture or 0 if not specified> * <[2] 16-bit error code (set to 0, currently unused)> - * <[8] 64-bit report flags (set to 0, currently unused)> + * <[8] 64-bit report flags> * <[8] 64-bit packet ID of received CIRCUIT_TEST packet> * <[5] upstream ZeroTier address from which CIRCUIT_TEST was received> * <[1] 8-bit packet hop count of received CIRCUIT_TEST> @@ -980,6 +980,9 @@ public: * <[5] ZeroTier address of next hop> * <[...] current best direct path address, if any, 0 if none> * + * Report flags: + * 0x1 - Upstream peer in circuit test path allowed in path (e.g. network COM valid) + * * Circuit test reports can be sent by hops in a circuit test to report * back results. They should include information about the sender as well * as about the paths to which next hops are being sent. diff --git a/node/Peer.cpp b/node/Peer.cpp index 58faab3b..a7a9fcc3 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -266,6 +266,7 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u atAddress.serialize(outp); outp.append((uint64_t)RR->topology->worldId()); outp.append((uint64_t)RR->topology->worldTimestamp()); + RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,false); // HELLO is sent in the clear RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } @@ -274,6 +275,7 @@ void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &at { if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); + RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,true); RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } else { diff --git a/node/Switch.cpp b/node/Switch.cpp index 21d0b3c9..f2a0d35b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -734,13 +734,12 @@ unsigned long Switch::doTimerTasks(uint64_t now) Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) { - SharedPtr root(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); - if (root) { - Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS); + SharedPtr upstream(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); + if (upstream) { + Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS); addr.appendTo(outp); - outp.armor(root->key(),true); - if (root->sendDirect(outp.data(),outp.size(),RR->node->now(),true)) - return root->address(); + RR->node->expectReplyTo(outp.packetId()); + send(outp,true); } return Address(); } -- cgit v1.2.3 From ab9afbc749f24f08f25dcf8bd6f4263b97c79bb9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 9 Sep 2016 11:36:10 -0700 Subject: (1) Public networks now get COMs even though they do not gate with them since they will need them to push auth for multicast stuff, (2) added a bunch of rate limit circuit breakers for anti-DOS, (3) cleanup. --- controller/EmbeddedNetworkController.cpp | 12 +-- node/Constants.hpp | 20 ++++ node/IncomingPacket.cpp | 152 ++++++++++++++++++----------- node/IncomingPacket.hpp | 2 +- node/Membership.cpp | 2 +- node/Multicaster.cpp | 80 ++++++++++------ node/Multicaster.hpp | 40 ++++++++ node/Network.cpp | 72 ++++++++------ node/Network.hpp | 20 ++-- node/Node.cpp | 2 +- node/Path.hpp | 15 +++ node/Peer.cpp | 160 +++++++++++++++---------------- node/Peer.hpp | 43 ++++++++- 13 files changed, 393 insertions(+), 227 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 79560dcc..861792ed 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -924,13 +924,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - if (_jB(network["private"],true)) { - CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); - if (com.sign(signingId)) { - nc.com = com; - } else { - return NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } + CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); + if (com.sign(signingId)) { + nc.com = com; + } else { + return NETCONF_QUERY_INTERNAL_SERVER_ERROR; } _writeJson(memberJP,member); diff --git a/node/Constants.hpp b/node/Constants.hpp index a625b480..05cd765a 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -236,6 +236,11 @@ */ #define ZT_MULTICAST_EXPLICIT_GATHER_DELAY (ZT_MULTICAST_LIKE_EXPIRE / 10) +/** + * Expiration for credentials presented for MULTICAST_LIKE or MULTICAST_GATHER (for non-network-members) + */ +#define ZT_MULTICAST_CREDENTIAL_EXPIRATON ZT_MULTICAST_LIKE_EXPIRE + /** * Timeout for outgoing multicasts * @@ -263,6 +268,11 @@ */ #define ZT_PATH_MIN_REACTIVATE_INTERVAL 2500 +/** + * Do not accept HELLOs over a given path more often than this + */ +#define ZT_PATH_HELLO_RATE_LIMIT 1000 + /** * Delay between full-fledge pings of directly connected peers */ @@ -283,6 +293,11 @@ */ #define ZT_PEER_ACTIVITY_TIMEOUT 500000 +/** + * General rate limit timeout for multiple packet types (HELLO, etc.) + */ +#define ZT_PEER_GENERAL_INBOUND_RATE_LIMIT 1000 + /** * Delay between requests for updated network autoconf information * @@ -326,6 +341,11 @@ */ #define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000 +/** + * General rate limit for other kinds of rate-limited packets (HELLO, credential request, etc.) both inbound and outbound + */ +#define ZT_PEER_GENERAL_RATE_LIMIT 1000 + /** * Maximum number of direct path pushes within cutoff time * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 1ce942c9..7f996dab 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -62,11 +62,8 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) return true; } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { - // A null pointer for peer to _doHELLO() tells it to run its own - // special internal authentication logic. This is done for unencrypted - // HELLOs to learn new identities, etc. - SharedPtr tmp; - return _doHELLO(RR,tmp); + // Only HELLO is allowed in the clear, but will still have a MAC + return _doHELLO(RR,false); } SharedPtr peer(RR->topology->getPeer(sourceAddress)); @@ -91,7 +88,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); return true; - case Packet::VERB_HELLO: return _doHELLO(RR,peer); + case Packet::VERB_HELLO: return _doHELLO(RR,true); case Packet::VERB_ERROR: return _doERROR(RR,peer); case Packet::VERB_OK: return _doOK(RR,peer); case Packet::VERB_WHOIS: return _doWHOIS(RR,peer); @@ -192,16 +189,16 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr return true; } -bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer) +bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated) { - /* Note: this is the only packet ever sent in the clear, and it's also - * the only packet that we authenticate via a different path. Authentication - * occurs here and is based on the validity of the identity and the - * integrity of the packet's MAC, but it must be done after we check - * the identity since HELLO is a mechanism for learning new identities - * in the first place. */ - try { + const uint64_t now = RR->node->now(); + + if (!_path->rateGateHello(now)) { + TRACE("dropped HELLO from %s(%s): rate limiting circuit breaker for HELLO on this path tripped",source().toString().c_str(),_path->address().toString().c_str()); + return true; + } + const uint64_t pid = packetId(); const Address fromAddress(source()); const unsigned int protoVersion = (*this)[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION]; @@ -228,20 +225,19 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } } - if (protoVersion < ZT_PROTO_VERSION_MIN) { - TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); - return true; - } if (fromAddress != id.address()) { TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_path->address().toString().c_str()); return true; } + if (protoVersion < ZT_PROTO_VERSION_MIN) { + TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; + } - if (!peer) { // peer == NULL is the normal case here - peer = RR->topology->getPeer(id.address()); - if (peer) { - // We already have an identity with this address -- check for collisions - + SharedPtr peer(RR->topology->getPeer(id.address())); + if (peer) { + // We already have an identity with this address -- check for collisions + if (!alreadyAuthenticated) { if (peer->identity() != id) { // Identity is different from the one we already have -- address collision @@ -273,31 +269,37 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer // Continue at // VALID } - } else { - // We don't already have an identity with this address -- validate and learn it + } // else continue at // VALID + } else { + // We don't already have an identity with this address -- validate and learn it - // Check identity proof of work - if (!id.locallyValidate()) { - TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); - return true; - } + // Sanity check: this basically can't happen + if (alreadyAuthenticated) { + TRACE("dropped HELLO from %s(%s): somehow already authenticated with unknown peer?",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; + } - // Check packet integrity and authentication - SharedPtr newPeer(new Peer(RR,RR->identity,id)); - if (!dearmor(newPeer->key())) { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); - return true; - } - peer = RR->topology->addPeer(newPeer); + // Check identity proof of work + if (!id.locallyValidate()) { + TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; + } - // Continue at // VALID + // Check packet integrity and authentication + SharedPtr newPeer(new Peer(RR,RR->identity,id)); + if (!dearmor(newPeer->key())) { + TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; } + peer = RR->topology->addPeer(newPeer); - // VALID -- if we made it here, packet passed identity and authenticity checks! + // Continue at // VALID } + // VALID -- if we made it here, packet passed identity and authenticity checks! + if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now()); + RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_HELLO); @@ -349,7 +351,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,outp.data(),outp.size(),now); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); @@ -443,7 +445,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); SharedPtr network(RR->node->network(nwid)); - if ((network)&&(network->gateMulticastGather(peer,verb(),packetId()))) { + if ((network)&&(network->gateMulticastGatherReply(peer,verb(),packetId()))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); @@ -469,7 +471,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p network->addCredential(com); } - if (network->gateMulticastGather(peer,verb(),packetId())) { + if (network->gateMulticastGatherReply(peer,verb(),packetId())) { if ((flags & 0x02) != 0) { // OK(MULTICAST_FRAME) includes implicit gather results offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; @@ -494,6 +496,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { + if (!peer->rateGateInboundWhoisRequest(RR->node->now())) { + TRACE("dropped WHOIS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + return true; + } + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_WHOIS); outp.append(packetId()); @@ -672,6 +679,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

&peer) { try { + if (!peer->rateGateEchoRequest(RR->node->now())) { + TRACE("dropped ECHO from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + return true; + } + const uint64_t pid = packetId(); Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_ECHO); @@ -680,6 +692,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true); _path->send(RR,outp.data(),outp.size(),RR->node->now()); + peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -692,11 +705,35 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared try { const uint64_t now = RR->node->now(); + uint64_t authOnNetwork[256]; + unsigned int authOnNetworkCount = 0; + SharedPtr network; + // Iterate through 18-byte network,MAC,ADI tuples for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr(ptr); - const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); - RR->mc->add(now,nwid,group,peer->address()); + + bool auth = false; + for(unsigned int i=0;iid() != nwid)) + network = RR->node->network(nwid); + if ( ((network)&&(network->gate(peer,verb(),packetId()))) || RR->mc->cacheAuthorized(peer->address(),nwid,now) ) { + auth = true; + if (authOnNetworkCount < 256) // sanity check, packets can't really be this big + authOnNetwork[authOnNetworkCount++] = nwid; + } + } + + if (auth) { + const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); + RR->mc->add(now,nwid,group,peer->address()); + } } peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); @@ -721,7 +758,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S if (network) { if (network->addCredential(com) == 1) return false; // wait for WHOIS - } + } else RR->mc->addCredential(com,false); } } ++p; // skip trailing 0 after COMs if present @@ -759,22 +796,21 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons { try { const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); - - const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); - const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); - const Dictionary metaData(metaDataBytes,metaDataLength); - const unsigned int hopCount = hops(); const uint64_t requestPacketId = packetId(); - bool netconfOk = false; + bool trustEstablished = false; if (RR->localNetworkController) { + const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); + const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); + const Dictionary metaData(metaDataBytes,metaDataLength); + NetworkConfig *netconf = new NetworkConfig(); try { switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _path->address(),RR->identity,peer->identity(),nwid,metaData,*netconf)) { case NetworkController::NETCONF_QUERY_OK: { - netconfOk = true; + trustEstablished = true; Dictionary *dconf = new Dictionary(); try { if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { @@ -846,7 +882,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk); + peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,trustEstablished); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -897,21 +933,23 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar //TRACE("<address().toString().c_str(),gatherLimit,nwid,mg.toString().c_str()); + const SharedPtr network(RR->node->network(nwid)); + if ((flags & 0x01) != 0) { try { CertificateOfMembership com; com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM); if (com) { - SharedPtr network(RR->node->network(nwid)); if (network) network->addCredential(com); + else RR->mc->addCredential(com,false); } } catch ( ... ) { TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str()); } } - if (gatherLimit) { + if ( ( ((network)&&(network->gate(peer,verb(),packetId()))) || (RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now())) ) && (gatherLimit > 0) ) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); outp.append(packetId()); @@ -1043,7 +1081,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha const uint64_t now = RR->node->now(); // First, subject this to a rate limit - if (!peer->shouldRespondToDirectPathPush(now)) { + if (!peer->rateGatePushDirectPaths(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 35438f4f..dbaf67b8 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -136,7 +136,7 @@ private: // These are called internally to handle packet contents once it has // been authenticated, decrypted, decompressed, and classified. bool _doERROR(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer); // can be called with NULL peer, while all others cannot + bool _doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated); bool _doOK(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer); diff --git a/node/Membership.cpp b/node/Membership.cpp index 4ca008e3..8c2ba673 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -71,7 +71,7 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint } capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - const bool needCom = ((nconf.isPrivate())&&(nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); + const bool needCom = ((nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); if ( (needCom) || (appendedCaps) || (appendedTags) ) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (needCom) { diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 36d7d2d0..fc8fa1bd 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -34,8 +34,8 @@ namespace ZeroTier { Multicaster::Multicaster(const RuntimeEnvironment *renv) : RR(renv), - _groups(1024), - _groups_m() + _groups(256), + _gatherAuth(256) { } @@ -244,7 +244,7 @@ void Multicaster::send( } for(unsigned int k=0;kconfig())&&(network->config().isPrivate())) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0; + const CertificateOfMembership *com = (network) ? ((network->config().com) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0; Packet outp(explicitGatherPeers[k],RR->identity.address(),Packet::VERB_MULTICAST_GATHER); outp.append(nwid); outp.append((uint8_t)((com) ? 0x01 : 0x00)); @@ -301,42 +301,62 @@ void Multicaster::send( void Multicaster::clean(uint64_t now) { - Mutex::Lock _l(_groups_m); - - Multicaster::Key *k = (Multicaster::Key *)0; - MulticastGroupStatus *s = (MulticastGroupStatus *)0; - Hashtable::Iterator mm(_groups); - while (mm.next(k,s)) { - for(std::list::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) { - if ((tx->expired(now))||(tx->atLimit())) - s->txQueue.erase(tx++); - else ++tx; - } + { + Mutex::Lock _l(_groups_m); + Multicaster::Key *k = (Multicaster::Key *)0; + MulticastGroupStatus *s = (MulticastGroupStatus *)0; + Hashtable::Iterator mm(_groups); + while (mm.next(k,s)) { + for(std::list::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) { + if ((tx->expired(now))||(tx->atLimit())) + s->txQueue.erase(tx++); + else ++tx; + } - unsigned long count = 0; - { - std::vector::iterator reader(s->members.begin()); - std::vector::iterator writer(reader); - while (reader != s->members.end()) { - if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) { - *writer = *reader; - ++writer; - ++count; + unsigned long count = 0; + { + std::vector::iterator reader(s->members.begin()); + std::vector::iterator writer(reader); + while (reader != s->members.end()) { + if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) { + *writer = *reader; + ++writer; + ++count; + } + ++reader; } - ++reader; + } + + if (count) { + s->members.resize(count); + } else if (s->txQueue.empty()) { + _groups.erase(*k); + } else { + s->members.clear(); } } + } - if (count) { - s->members.resize(count); - } else if (s->txQueue.empty()) { - _groups.erase(*k); - } else { - s->members.clear(); + { + Mutex::Lock _l(_gatherAuth_m); + _GatherAuthKey *k = (_GatherAuthKey *)0; + uint64_t *ts = (uint64_t *)ts; + Hashtable<_GatherAuthKey,uint64_t>::Iterator i(_gatherAuth); + while (i.next(k,ts)) { + if ((now - *ts) >= ZT_MULTICAST_CREDENTIAL_EXPIRATON) + _gatherAuth.erase(*k); } } } +void Multicaster::addCredential(const CertificateOfMembership &com,bool alreadyValidated) +{ + if ((alreadyValidated)||(com.verify(RR) == 0)) { + Mutex::Lock _l(_gatherAuth_m); + _gatherAuth[_GatherAuthKey(com.networkId(),com.issuedTo())] = RR->node->now(); + } +} + void Multicaster::_add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) { // assumes _groups_m is locked diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 51dabc69..8be3b736 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -179,12 +179,52 @@ public: */ void clean(uint64_t now); + /** + * Add an authorization credential + * + * The Multicaster keeps its own track of when valid credentials of network + * membership are presented. This allows it to control MULTICAST_LIKE + * GATHER authorization for networks this node does not belong to. + * + * @param com Certificate of membership + * @param alreadyValidated If true, COM has already been checked and found to be valid and signed + */ + void addCredential(const CertificateOfMembership &com,bool alreadyValidated); + + /** + * Check authorization for GATHER and LIKE for non-network-members + * + * @param a Address of peer + * @param nwid Network ID + * @param now Current time + * @return True if GATHER and LIKE should be allowed + */ + bool cacheAuthorized(const Address &a,const uint64_t nwid,const uint64_t now) const + { + Mutex::Lock _l(_gatherAuth_m); + const uint64_t *p = _gatherAuth.get(_GatherAuthKey(nwid,a)); + return ((p)&&((now - *p) < ZT_MULTICAST_CREDENTIAL_EXPIRATON)); + } + private: void _add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); const RuntimeEnvironment *RR; + Hashtable _groups; Mutex _groups_m; + + struct _GatherAuthKey + { + _GatherAuthKey() : member(0),networkId(0) {} + _GatherAuthKey(const uint64_t nwid,const Address &a) : member(a.toInt()),networkId(nwid) {} + inline unsigned long hashCode() const { return (member ^ networkId); } + inline bool operator==(const _GatherAuthKey &k) const { return ((member == k.member)&&(networkId == k.networkId)); } + uint64_t member; + uint64_t networkId; + }; + Hashtable< _GatherAuthKey,uint64_t > _gatherAuth; + Mutex _gatherAuth_m; }; } // namespace ZeroTier diff --git a/node/Network.cpp b/node/Network.cpp index a9b14942..146f2962 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -866,31 +866,24 @@ bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBr return true; else if (includeBridgedGroups) return _multicastGroupsBehindMe.contains(mg); - else return false; + return false; } void Network::multicastSubscribe(const MulticastGroup &mg) { - { - Mutex::Lock _l(_lock); - if (std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) - return; - _myMulticastGroups.push_back(mg); - std::sort(_myMulticastGroups.begin(),_myMulticastGroups.end()); - _pushStateToMembers(&mg); + Mutex::Lock _l(_lock); + if (!std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) { + _myMulticastGroups.insert(std::upper_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg),mg); + _sendUpdatesToMembers(&mg); } } void Network::multicastUnsubscribe(const MulticastGroup &mg) { Mutex::Lock _l(_lock); - std::vector nmg; - for(std::vector::const_iterator i(_myMulticastGroups.begin());i!=_myMulticastGroups.end();++i) { - if (*i != mg) - nmg.push_back(*i); - } - if (nmg.size() != _myMulticastGroups.size()) - _myMulticastGroups.swap(nmg); + std::vector::iterator i(std::lower_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)); + if ( (i != _myMulticastGroups.end()) && (*i == mg) ) + _myMulticastGroups.erase(i); } bool Network::applyConfiguration(const NetworkConfig &conf) @@ -1054,30 +1047,29 @@ void Network::requestConfiguration() } else { outp.append((unsigned char)0,16); } - RR->node->expectReplyTo(outp.packetId()); - outp.compress(); - RR->sw->send(outp,true); - // Expect replies with this in-re packet ID - _inboundConfigPacketId = outp.packetId(); + RR->node->expectReplyTo(_inboundConfigPacketId = outp.packetId()); _inboundConfigChunks.clear(); + + outp.compress(); + RR->sw->send(outp,true); } bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) { + const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); try { if (_config) { Membership &m = _membership(peer->address()); const bool allow = m.isAllowedOnNetwork(_config); if (allow) { - const uint64_t now = RR->node->now(); m.sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0); if (m.shouldLikeMulticasts(now)) { _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); m.likingMulticasts(now); } - } else if (m.recentlyAllowedOnNetwork(_config)) { + } else if (m.recentlyAllowedOnNetwork(_config)&&peer->rateGateRequestCredentials(now)) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((uint8_t)verb); outp.append(packetId); @@ -1093,7 +1085,7 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin return false; } -bool Network::gateMulticastGather(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) +bool Network::gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) { return ( (peer->address() == controller()) || RR->topology->isUpstream(peer->identity()) || gate(peer,verb,packetId) || _config.isAnchor(peer->address()) ); } @@ -1180,7 +1172,22 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); _multicastGroupsBehindMe.set(mg,now); if (tmp != _multicastGroupsBehindMe.size()) - _pushStateToMembers(&mg); + _sendUpdatesToMembers(&mg); +} + +int Network::addCredential(const CertificateOfMembership &com) +{ + if (com.networkId() != _id) + return -1; + const Address a(com.issuedTo()); + Mutex::Lock _l(_lock); + Membership &m = _membership(a); + const int result = m.addCredential(RR,com); + if (result == 0) { + m.sendCredentialsIfNeeded(RR,RR->node->now(),a,_config,(const Capability *)0); + RR->mc->addCredential(com,true); + } + return result; } void Network::destroy() @@ -1245,7 +1252,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const } } -void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup) +void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup) { // Assumes _lock is locked const uint64_t now = RR->node->now(); @@ -1263,7 +1270,7 @@ void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup) // them our COM so that MULTICAST_GATHER can be authenticated properly. const std::vector

upstreams(RR->topology->upstreamAddresses()); for(std::vector
::const_iterator a(upstreams.begin());a!=upstreams.end();++a) { - if ((_config.isPrivate())&&(_config.com)) { + if (_config.com) { Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); _config.com.serialize(outp); outp.append((uint8_t)0x00); @@ -1272,12 +1279,17 @@ void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup) _announceMulticastGroupsTo(*a,groups); } - // Announce to controller, which does not need our COM since it obviously - // knows if we are a member. Of course if we already did or are going to - // below then we can skip it here. + // Also announce to controller, and send COM to simplify and generalize behavior even though in theory it does not need it const Address c(controller()); - if ( (std::find(upstreams.begin(),upstreams.end(),c) == upstreams.end()) && (!_memberships.contains(c)) ) + if ( (std::find(upstreams.begin(),upstreams.end(),c) == upstreams.end()) && (!_memberships.contains(c)) ) { + if (_config.com) { + Packet outp(c,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + _config.com.serialize(outp); + outp.append((uint8_t)0x00); + RR->sw->send(outp,true); + } _announceMulticastGroupsTo(c,groups); + } } // Make sure that all "network anchors" have Membership records so we will diff --git a/node/Network.hpp b/node/Network.hpp index d80b13b9..7a4065ff 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -260,7 +260,7 @@ public: /** * Check whether this peer is allowed to provide multicast info for this network */ - bool gateMulticastGather(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); + bool gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); /** * @param peer Peer to check @@ -276,10 +276,10 @@ public: /** * Push state to members such as multicast group memberships and latest COM (if needed) */ - inline void pushStateToMembers() + inline void sendUpdatesToMembers() { Mutex::Lock _l(_lock); - _pushStateToMembers((const MulticastGroup *)0); + _sendUpdatesToMembers((const MulticastGroup *)0); } /** @@ -332,9 +332,7 @@ public: { Mutex::Lock _l(_lock); const Address *const br = _remoteBridgeRoutes.get(mac); - if (br) - return *br; - return Address(); + return ((br) ? *br : Address()); } /** @@ -357,13 +355,7 @@ public: * @param com Certificate of membership * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - inline int addCredential(const CertificateOfMembership &com) - { - if (com.networkId() != _id) - return -1; - Mutex::Lock _l(_lock); - return _membership(com.issuedTo()).addCredential(RR,com); - } + int addCredential(const CertificateOfMembership &com); /** * @param cap Capability @@ -418,7 +410,7 @@ private: ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); - void _pushStateToMembers(const MulticastGroup *const newMulticastGroup); + void _sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup); void _announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; Membership &_membership(const Address &a); diff --git a/node/Node.cpp b/node/Node.cpp index e8279c62..59794854 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -266,7 +266,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) needConfig.push_back(n->second); - n->second->pushStateToMembers(); + n->second->sendUpdatesToMembers(); } } for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) diff --git a/node/Path.hpp b/node/Path.hpp index 27cff645..6278532d 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -104,6 +104,7 @@ public: Path() : _lastOut(0), _lastIn(0), + _lastHello(0), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) @@ -113,6 +114,7 @@ public: Path(const InetAddress &localAddress,const InetAddress &addr) : _lastOut(0), _lastIn(0), + _lastHello(0), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) @@ -229,9 +231,22 @@ public: */ inline uint64_t lastIn() const { return _lastIn; } + /** + * @return True if we should allow HELLO via this path + */ + inline bool rateGateHello(const uint64_t now) + { + if ((now - _lastHello) >= ZT_PATH_HELLO_RATE_LIMIT) { + _lastHello = now; + return true; + } + return false; + } + private: uint64_t _lastOut; uint64_t _lastIn; + uint64_t _lastHello; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often diff --git a/node/Peer.cpp b/node/Peer.cpp index a7a9fcc3..0e6ef333 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -47,6 +47,9 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastMulticastFrame(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), + _lastCredentialRequestSent(0), + _lastWhoisRequestReceived(0), + _lastEchoRequestReceived(0), RR(renv), _remoteClusterOptimal4(0), _vProto(0), @@ -194,7 +197,80 @@ void Peer::received( } } else if (trustEstablished) { // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) - _pushDirectPaths(path,now); +#ifdef ZT_ENABLE_CLUSTER + // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection + const bool haveCluster = (RR->cluster); +#else + const bool haveCluster = false; +#endif + if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) { + _lastDirectPathPushSent = now; + + std::vector pathsToPush; + + std::vector dps(RR->node->directPaths()); + for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) + pathsToPush.push_back(*i); + + std::vector sym(RR->sa->getSymmetricNatPredictions()); + for(unsigned long i=0,added=0;inode->prng() % sym.size()]); + if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { + pathsToPush.push_back(tmp); + if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) + break; + } + } + + if (pathsToPush.size() > 0) { +#ifdef ZT_TRACE + std::string ps; + for(std::vector::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) { + if (ps.length() > 0) + ps.push_back(','); + ps.append(p->toString()); + } + TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str()); +#endif + + std::vector::const_iterator p(pathsToPush.begin()); + while (p != pathsToPush.end()) { + Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); + outp.addSize(2); // leave room for count + + unsigned int count = 0; + while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) { + uint8_t addressType = 4; + switch(p->ss_family) { + case AF_INET: + break; + case AF_INET6: + addressType = 6; + break; + default: // we currently only push IP addresses + ++p; + continue; + } + + outp.append((uint8_t)0); // no flags + outp.append((uint16_t)0); // no extensions + outp.append(addressType); + outp.append((uint8_t)((addressType == 4) ? 6 : 18)); + outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16)); + outp.append((uint16_t)p->port()); + + ++count; + ++p; + } + + if (count) { + outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); + outp.armor(_key,true); + path->send(RR,outp.data(),outp.size(),now); + } + } + } + } } } @@ -368,86 +444,4 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) v6 = _paths[bestp6].path->address(); } -bool Peer::_pushDirectPaths(const SharedPtr &path,uint64_t now) -{ -#ifdef ZT_ENABLE_CLUSTER - // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection - if (RR->cluster) - return false; -#endif - - if ((now - _lastDirectPathPushSent) < ZT_DIRECT_PATH_PUSH_INTERVAL) - return false; - else _lastDirectPathPushSent = now; - - std::vector pathsToPush; - - std::vector dps(RR->node->directPaths()); - for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) - pathsToPush.push_back(*i); - - std::vector sym(RR->sa->getSymmetricNatPredictions()); - for(unsigned long i=0,added=0;inode->prng() % sym.size()]); - if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { - pathsToPush.push_back(tmp); - if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) - break; - } - } - if (pathsToPush.empty()) - return false; - -#ifdef ZT_TRACE - { - std::string ps; - for(std::vector::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) { - if (ps.length() > 0) - ps.push_back(','); - ps.append(p->toString()); - } - TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str()); - } -#endif - - std::vector::const_iterator p(pathsToPush.begin()); - while (p != pathsToPush.end()) { - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); - outp.addSize(2); // leave room for count - - unsigned int count = 0; - while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) { - uint8_t addressType = 4; - switch(p->ss_family) { - case AF_INET: - break; - case AF_INET6: - addressType = 6; - break; - default: // we currently only push IP addresses - ++p; - continue; - } - - outp.append((uint8_t)0); // no flags - outp.append((uint16_t)0); // no extensions - outp.append(addressType); - outp.append((uint8_t)((addressType == 4) ? 6 : 18)); - outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16)); - outp.append((uint16_t)p->port()); - - ++count; - ++p; - } - - if (count) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); - outp.armor(_key,true); - path->send(RR,outp.data(),outp.size(),now); - } - } - - return true; -} - } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 2e64fb4d..d714b937 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -348,7 +348,7 @@ public: * @param now Current time * @return True if we should respond */ - inline bool shouldRespondToDirectPathPush(const uint64_t now) + inline bool rateGatePushDirectPaths(const uint64_t now) { if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME) ++_directPathPushCutoffCount; @@ -357,6 +357,42 @@ public: return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT); } + /** + * Rate limit gate for sending of ERROR_NEED_MEMBERSHIP_CERTIFICATE + */ + inline bool rateGateRequestCredentials(const uint64_t now) + { + if ((now - _lastCredentialRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastCredentialRequestSent = now; + return true; + } + return false; + } + + /** + * Rate limit gate for inbound WHOIS requests + */ + inline bool rateGateInboundWhoisRequest(const uint64_t now) + { + if ((now - _lastWhoisRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastWhoisRequestReceived = now; + return true; + } + return false; + } + + /** + * Rate limit gate for inbound ECHO requests + */ + inline bool rateGateEchoRequest(const uint64_t now) + { + if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastEchoRequestReceived = now; + return true; + } + return false; + } + /** * Find a common set of addresses by which two peers can link, if any * @@ -378,8 +414,6 @@ public: } private: - bool _pushDirectPaths(const SharedPtr &path,uint64_t now); - inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const { uint64_t s = ZT_PEER_PING_PERIOD + _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)); @@ -415,6 +449,9 @@ private: uint64_t _lastMulticastFrame; uint64_t _lastDirectPathPushSent; uint64_t _lastDirectPathPushReceive; + uint64_t _lastCredentialRequestSent; + uint64_t _lastWhoisRequestReceived; + uint64_t _lastEchoRequestReceived; const RuntimeEnvironment *RR; uint32_t _remoteClusterOptimal4; uint16_t _vProto; -- cgit v1.2.3 From 68e549233ddba17ec686a0462e2630580ee3d2a7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 15 Sep 2016 13:17:37 -0700 Subject: Revise bearer token code in controller, and add relay policy as a meta-data item presented to controller by nodes (to facilitate future meshiness). --- controller/EmbeddedNetworkController.cpp | 103 +++++++++++++++++++------------ controller/EmbeddedNetworkController.hpp | 4 +- controller/README.md | 4 +- node/Network.cpp | 2 + node/NetworkConfig.hpp | 10 ++- 5 files changed, 77 insertions(+), 46 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 861792ed..53b345b4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -566,42 +566,69 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; - if (!_jB(network["private"],true)) { + if (_jB(member["authorized"],false)) { + authorizedBy = "memberIsAuthorized"; + } else if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; - // If member already has an authorized field, leave it alone. That way its state is - // preserved if the user toggles the network back to private. Otherwise set it to - // true by default for new members of public nets. if (!member.count("authorized")) { member["authorized"] = true; - member["lastAuthorizedTime"] = now; - member["lastAuthorizedBy"] = authorizedBy; + json ah; + ah["a"] = true; + ah["by"] = authorizedBy; + ah["ts"] = now; + ah["ct"] = json(); + ah["c"] = json(); + member["authHistory"].push_back(ah); member["lastModified"] = now; - auto revj = member["revision"]; + json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } - } else if (_jB(member["authorized"],false)) { - authorizedBy = "memberIsAuthorized"; } else { - char atok[256]; - if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { - atok[255] = (char)0; // not necessary but YDIFLO - if (strlen(atok) > 0) { // extra sanity check since we never want to compare a null token on either side - auto authTokens = network["authTokens"]; + char presentedAuth[512]; + if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) { + presentedAuth[511] = (char)0; // sanity check + + // Check for bearer token presented by member + if ((strlen(presentedAuth) > 6)&&(!strncmp(presentedAuth,"token:",6))) { + const char *const presentedToken = presentedAuth + 6; + + json &authTokens = network["authTokens"]; if (authTokens.is_array()) { for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { - authorizedBy = "token"; - member["authorized"] = true; // tokens actually change member authorization state - member["lastAuthorizedTime"] = now; - member["lastAuthorizedBy"] = authorizedBy; - member["lastModified"] = now; - auto revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - break; + json &token = authTokens[i]; + if (token.is_object()) { + const uint64_t expires = _jI(token["expires"],0ULL); + const uint64_t maxUses = _jI(token["maxUsesPerMember"],0ULL); + std::string tstr = _jS(token["token"],""); + + if (((expires == 0ULL)||(expires > now))&&(tstr == presentedToken)) { + bool usable = (maxUses == 0); + if (!usable) { + uint64_t useCount = 0; + json &ahist = member["authHistory"]; + if (ahist.is_array()) { + for(unsigned long j=0;j 0) { json t = json::object(); t["token"] = tstr; t["expires"] = _jI(token["expires"],0ULL); + t["maxUsesPerMember"] = _jI(token["maxUsesPerMember"],0ULL); nat.push_back(t); } } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 303a5355..1bfd9577 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -143,9 +143,7 @@ private: inline void _initMember(nlohmann::json &member) { if (!member.count("authorized")) member["authorized"] = false; - if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL; - if (!member.count("lastAuthorizedBy")) member["lastAuthorizedBy"] = ""; - if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL; + if (!member.count("authHistory")) member["authHistory"] = nlohmann::json::array(); if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array(); if (!member.count("activeBridge")) member["activeBridge"] = false; diff --git a/controller/README.md b/controller/README.md index 1eb0ca0d..805641d9 100644 --- a/controller/README.md +++ b/controller/README.md @@ -229,9 +229,7 @@ This returns an object containing all currently online members and the most rece | nwid | string | 16-digit network ID | no | | clock | integer | Current clock, ms since epoch | no | | authorized | boolean | Is member authorized? (for private networks) | YES | -| lastAuthorizedTime | integer | Time 'authorized' was last set to 'true' | no | -| lastAuthorizedBy | string | What last set 'authorized' to 'true'? | no | -| lastDeauthorizedTime | integer | Time 'authorized' was last set to 'false' | no | +| authHistory | array[object] | History of auth changes, latest at end | no | | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | | ipAssignments | array[string] | Managed IP address assignments | YES | diff --git a/node/Network.cpp b/node/Network.cpp index 22aca0d8..197841d9 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1001,6 +1001,7 @@ void Network::requestConfiguration() Dictionary rmd; rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_VENDOR,(uint64_t)ZT_VENDOR_ZEROTIER); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MAJOR); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MINOR); @@ -1011,6 +1012,7 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY,(uint64_t)RR->node->relayPolicy()); if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index ad1cafa5..5ad86855 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -108,9 +108,13 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION "v" // Protocol version (see Packet.hpp) #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION "pv" -// Software major, minor, revision +// Software vendor +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_VENDOR "vend" +// Software major version #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv" +// Software minor version #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv" +// Software revision #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv" // Rules engine revision #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV "revr" @@ -123,9 +127,11 @@ namespace ZeroTier { // Maximum number of tags this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS "mt" // Network join authorization token (if any) -#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN "atok" +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH "a" // Network configuration meta-data flags #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS "f" +// Relay policy for this node +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY "rp" // These dictionary keys are short so they don't take up much room. // By convention we use upper case for binary blobs, but it doesn't really matter. -- cgit v1.2.3 From 1f74dd4589017bd5bc34b31ff6e09e6875d5e1c7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 23 Sep 2016 16:08:38 -0700 Subject: Revocation work in progress, add WATCH which is TEE with implicit rate sync (thanks JG@DCVC!), and clean up some cruft in Network. --- controller/EmbeddedNetworkController.cpp | 12 ++ include/ZeroTierOne.h | 9 +- node/Capability.hpp | 2 + node/IncomingPacket.cpp | 76 ++++---- node/Membership.cpp | 234 +++++++++++++++---------- node/Membership.hpp | 263 +++++++++++++--------------- node/Network.cpp | 289 +++++++++++++------------------ node/Network.hpp | 189 +++++++------------- node/Packet.hpp | 63 ++++--- node/Revocation.cpp | 46 +++++ node/Revocation.hpp | 178 +++++++++++++++++++ node/Tag.cpp | 2 +- node/Tag.hpp | 30 +++- objects.mk | 1 + 14 files changed, 803 insertions(+), 591 deletions(-) create mode 100644 node/Revocation.cpp create mode 100644 node/Revocation.hpp (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 53b345b4..5ba8cf98 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -140,6 +140,12 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["flags"] = (unsigned int)rule.v.fwd.flags; r["length"] = (unsigned int)rule.v.fwd.length; break; + case ZT_NETWORK_RULE_ACTION_WATCH: + r["type"] = "ACTION_WATCH"; + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (unsigned int)rule.v.fwd.flags; + r["length"] = (unsigned int)rule.v.fwd.length; + break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; r["address"] = Address(rule.v.fwd.address).toString(); @@ -303,6 +309,12 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; + } else if (t == "ACTION_WATCH") { + rule.t |= ZT_NETWORK_RULE_ACTION_WATCH; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); + return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e0f6ca28..e43c8541 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -516,15 +516,20 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_ACTION_TEE = 2, + /** + * Exactly like TEE but frames are dropped if previous TEEs were not acknowledged by the observer + */ + ZT_NETWORK_RULE_ACTION_WATCH = 3, + /** * Drop and redirect this frame to another node (by ZT address) */ - ZT_NETWORK_RULE_ACTION_REDIRECT = 3, + ZT_NETWORK_RULE_ACTION_REDIRECT = 4, /** * Log if match and if rule debugging is enabled in the build, otherwise does nothing (for developers) */ - ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 4, + ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 5, /** * Maximum ID for an ACTION, anything higher is a MATCH diff --git a/node/Capability.hpp b/node/Capability.hpp index e23d7943..2cf54b5c 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -174,6 +174,7 @@ public: b.append((uint8_t)0); break; case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_WATCH: case ZT_NETWORK_RULE_ACTION_REDIRECT: b.append((uint8_t)14); b.append((uint64_t)rules[i].v.fwd.address); @@ -270,6 +271,7 @@ public: default: break; case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_WATCH: case ZT_NETWORK_RULE_ACTION_REDIRECT: rules[ruleCount].v.fwd.address = b.template at(p); rules[ruleCount].v.fwd.flags = b.template at(p + 8); diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b3925773..12766fe2 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -39,6 +39,7 @@ #include "CertificateOfMembership.hpp" #include "Capability.hpp" #include "Tag.hpp" +#include "Revocation.hpp" namespace ZeroTier { @@ -162,13 +163,8 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr // Peers can send this in response to frames if they do not have a recent enough COM from us SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); const uint64_t now = RR->node->now(); - if ( (network) && (network->config().com) && (peer->rateGateComRequest(now)) ) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - network->config().com.serialize(outp); - outp.append((uint8_t)0); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),now); - } + if ( (network) && (network->config().com) && (peer->rateGateComRequest(now)) ) + network->pushCredentialsNow(peer->address(),now); } break; case Packet::ERROR_NETWORK_ACCESS_DENIED_: { @@ -681,9 +677,17 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

node->putFrame(nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); break; } + } - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + if ((flags & 0x10) != 0) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((uint8_t)Packet::VERB_EXT_FRAME); + outp.append((uint64_t)packetId()); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } + + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); @@ -775,6 +779,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S CertificateOfMembership com; Capability cap; Tag tag; + Revocation revocation; bool trustEstablished = false; unsigned int p = ZT_PACKET_IDX_PAYLOAD; @@ -784,8 +789,14 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S SharedPtr network(RR->node->network(com.networkId())); if (network) { switch (network->addCredential(com)) { - case 0: trustEstablished = true; break; - case 1: return false; // wait for WHOIS + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } else RR->mc->addCredential(com,false); } @@ -799,8 +810,14 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S SharedPtr network(RR->node->network(cap.networkId())); if (network) { switch (network->addCredential(cap)) { - case 0: trustEstablished = true; break; - case 1: return false; // wait for WHOIS + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } } @@ -811,11 +828,25 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S SharedPtr network(RR->node->network(tag.networkId())); if (network) { switch (network->addCredential(tag)) { - case 0: trustEstablished = true; break; - case 1: return false; // wait for WHOIS + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } } + + const unsigned int numRevocations = at(p); p += 2; + for(unsigned int i=0;i network(RR->node->network(revocation.networkId())); + if (network) { + } + } } peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); @@ -932,24 +963,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons const uint64_t nwid = at(ZT_PACKET_IDX_PAYLOAD); bool trustEstablished = false; - if (Network::controllerFor(nwid) == peer->address()) { - SharedPtr network(RR->node->network(nwid)); - if (network) { - network->requestConfiguration(); - trustEstablished = true; - } else { - TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); - return true; - } - const unsigned int blacklistCount = at(ZT_PACKET_IDX_PAYLOAD + 8); - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 10; - for(unsigned int i=0;iblacklistBefore(Address(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH),at(ptr + 5)); - ptr += 13; - } - } peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { diff --git a/node/Membership.cpp b/node/Membership.cpp index 8c2ba673..d579d303 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -16,6 +16,8 @@ * along with this program. If not, see . */ +#include + #include "Membership.hpp" #include "RuntimeEnvironment.hpp" #include "Peer.hpp" @@ -28,28 +30,43 @@ namespace ZeroTier { -void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap) +Membership::Membership() : + _lastUpdatedMulticast(0), + _lastPushAttempt(0), + _lastPushedCom(0), + _comRevocationThreshold(0) +{ + for(unsigned int i=0;i= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); do { - unfinished = false; Buffer capsAndTags; unsigned int appendedCaps = 0; - if (cap) { + if (localCapabilityIndex >= 0) { capsAndTags.addSize(2); - std::map::iterator cs(_caps.find(cap->id())); - if ((cs != _caps.end())&&((now - cs->second.lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY)) { - cap->serialize(capsAndTags); - cs->second.lastPushed = now; + + if ( (_localCaps[localCapabilityIndex].id != nconf.capabilities[localCapabilityIndex].id()) || ((now - _localCaps[localCapabilityIndex].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCaps[localCapabilityIndex].lastPushed = now; + _localCaps[localCapabilityIndex].id = nconf.capabilities[localCapabilityIndex].id(); + nconf.capabilities[localCapabilityIndex].serialize(capsAndTags); ++appendedCaps; } + capsAndTags.setAt(0,(uint16_t)appendedCaps); + localCapabilityIndex = -1; // don't send this cap again on subsequent loops if force is true } else { capsAndTags.append((uint16_t)0); } @@ -57,22 +74,17 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint unsigned int appendedTags = 0; const unsigned int tagCountPos = capsAndTags.size(); capsAndTags.addSize(2); - for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { - if ((capsAndTags.size() + sizeof(Tag)) >= (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) { - unfinished = true; + for(;localTagPtr= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + if ((capsAndTags.size() + sizeof(Tag)) >= (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) break; - } - nconf.tags[i].serialize(capsAndTags); - ts->lastPushed = now; + nconf.tags[localTagPtr].serialize(capsAndTags); ++appendedTags; } } capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - const bool needCom = ((nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); - if ( (needCom) || (appendedCaps) || (appendedTags) ) { + if (needCom||appendedCaps||appendedTags) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (needCom) { nconf.com.serialize(outp); @@ -80,110 +92,148 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint } outp.append((uint8_t)0x00); outp.append(capsAndTags.data(),capsAndTags.size()); + outp.append((uint16_t)0); // no revocations, these propagate differently outp.compress(); RR->sw->send(outp,true); + needCom = false; // don't send COM again on subsequent loops if force is true } - } while (unfinished); // if there are many tags, etc., we can send more than one + } while (localTagPtr < nconf.tagCount); } catch ( ... ) { TRACE("unable to send credentials due to unexpected exception"); } } -int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) +const Capability *Membership::getCapability(const NetworkConfig &nconf,const uint32_t id) const { - if (_com == com) { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); - return 0; + const _RemoteCapability *const *c = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)id,_RemoteCredentialSorter<_RemoteCapability>()); + return ( ((c != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*c)->id == (uint64_t)id)) ? ((((*c)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*c)->cap,**c))) ? &((*c)->cap) : (const Capability *)0) : (const Capability *)0); +} + +const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) const +{ + const _RemoteTag *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialSorter<_RemoteTag>()); + return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*t)->tag,**t))) ? &((*t)->tag) : (const Tag *)0) : (const Tag *)0); +} + +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com) +{ + const uint64_t newts = com.timestamp().first; + if (newts <= _comRevocationThreshold) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (revoked)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_REJECTED; } - const int vr = com.verify(RR); + const uint64_t oldts = _com.timestamp().first; + if (newts < oldts) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (older than current)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_REJECTED; + } + if ((newts == oldts)&&(_com == com)) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } - if (vr == 0) { - if (com.timestamp().first >= _com.timestamp().first) { + switch(com.verify(RR)) { + default: + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (invalid signature or object)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_REJECTED; + case 0: TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); _com = com; - } else { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED but not used (OK but older than current)",com.issuedTo().toString().c_str(),com.networkId()); - } - } else { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; } - - return vr; } -int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag) { - TState *t = _tags.get(tag.id()); - if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); - return 0; + _RemoteTag *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialSorter<_RemoteTag>()); + _RemoteTag *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteTag *)0; + if (have) { + if ( (!_isCredentialTimestampValid(nconf,tag,*have)) || (have->tag.timestamp() > tag.timestamp()) ) { + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + } + if (have->tag == tag) { + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } } - const int vr = tag.verify(RR); - if (vr == 0) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); - if (!t) { - while (_tags.size() >= ZT_MAX_NETWORK_TAGS) { - uint32_t oldest = 0; - uint64_t oldestLastReceived = 0xffffffffffffffffULL; - uint32_t *i = (uint32_t *)0; - TState *ts = (TState *)0; - Hashtable::Iterator tsi(_tags); - while (tsi.next(i,ts)) { - if (ts->lastReceived < oldestLastReceived) { - oldestLastReceived = ts->lastReceived; - oldest = *i; + + switch(tag.verify(RR)) { + default: + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + case 0: + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); + if (have) { + have->lastReceived = RR->node->now(); + have->tag = tag; + } else { + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == 0xffffffffffffffffULL) { + have = _remoteTags[i]; + break; + } else if (_remoteTags[i]->lastReceived <= minlr) { + have = _remoteTags[i]; + minlr = _remoteTags[i]->lastReceived; } } - if (oldestLastReceived != 0xffffffffffffffffULL) - _tags.erase(oldest); + have->lastReceived = RR->node->now(); + have->tag = tag; + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); } - t = &(_tags[tag.id()]); - } - if (t->tag.timestamp() <= tag.timestamp()) { - t->lastReceived = RR->node->now(); - t->tag = tag; - } - } else { - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (%d)",tag.issuedTo().toString().c_str(),tag.networkId(),vr); + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; } - return vr; } -int Membership::addCredential(const RuntimeEnvironment *RR,const Capability &cap) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap) { - std::map::iterator c(_caps.find(cap.id())); - if ((c != _caps.end())&&(c->second.lastReceived != 0)&&(c->second.cap == cap)) { - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); - return 0; + _RemoteCapability *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialSorter<_RemoteCapability>()); + _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCapability *)0; + if (have) { + if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->cap.timestamp() > cap.timestamp()) ) { + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + } + if (have->cap == cap) { + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } } - const int vr = cap.verify(RR); - if (vr == 0) { - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); - if (c == _caps.end()) { - while (_caps.size() >= ZT_MAX_NETWORK_CAPABILITIES) { - std::map::iterator oldest; - uint64_t oldestLastReceived = 0xffffffffffffffffULL; - for(std::map::iterator i(_caps.begin());i!=_caps.end();++i) { - if (i->second.lastReceived < oldestLastReceived) { - oldestLastReceived = i->second.lastReceived; - oldest = i; + + switch(cap.verify(RR)) { + default: + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + case 0: + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); + if (have) { + have->lastReceived = RR->node->now(); + have->cap = cap; + } else { + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == 0xffffffffffffffffULL) { + have = _remoteCaps[i]; + break; + } else if (_remoteCaps[i]->lastReceived <= minlr) { + have = _remoteCaps[i]; + minlr = _remoteCaps[i]->lastReceived; } } - if (oldestLastReceived != 0xffffffffffffffffULL) - _caps.erase(oldest); + have->lastReceived = RR->node->now(); + have->cap = cap; + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); } - CState &c2 = _caps[cap.id()]; - c2.lastReceived = RR->node->now(); - c2.cap = cap; - } else if (c->second.cap.timestamp() <= cap.timestamp()) { - c->second.lastReceived = RR->node->now(); - c->second.cap = cap; - } - } else { - TRACE("addCredential(Capability) for %s on %.16llx REJECTED (%d)",cap.issuedTo().toString().c_str(),cap.networkId(),vr); + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; } - return vr; } } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 5eb68d34..421e3ee8 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -21,14 +21,12 @@ #include -#include - #include "Constants.hpp" #include "../include/ZeroTierOne.h" #include "CertificateOfMembership.hpp" #include "Capability.hpp" #include "Tag.hpp" -#include "Hashtable.hpp" +#include "Revocation.hpp" #include "NetworkConfig.hpp" namespace ZeroTier { @@ -40,77 +38,135 @@ class Network; * A container for certificates of membership and other network credentials * * This is kind of analogous to a join table between Peer and Network. It is - * presently held by the Network object for each participating Peer. + * held by the Network object for each participating Peer. * - * This is not thread safe. It must be locked externally. + * This class is not thread safe. It must be locked externally. */ class Membership { private: // Tags and related state - struct TState + struct _RemoteTag { - TState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed OUR tag to this peer (with this ID) - uint64_t lastPushed; + _RemoteTag() : id(0xffffffffffffffffULL),lastReceived(0),revocationThreshold(0) {} + // Tag ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) + uint64_t id; // Last time we received THEIR tag (with this ID) uint64_t lastReceived; + // Revocation blacklist threshold or 0 if none + uint64_t revocationThreshold; // THEIR tag Tag tag; }; // Credentials and related state - struct CState + struct _RemoteCapability { - CState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed OUR capability to this peer (with this ID) - uint64_t lastPushed; + _RemoteCapability() : id(0xffffffffffffffffULL),lastReceived(0),revocationThreshold(0) {} + // Capability ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) + uint64_t id; // Last time we received THEIR capability (with this ID) uint64_t lastReceived; + // Revocation blacklist threshold or 0 if none + uint64_t revocationThreshold; // THEIR capability Capability cap; }; + // Comparison operator for remote credential entries + template + struct _RemoteCredentialSorter + { + inline bool operator()(const T *a,const T *b) const { return (a->id < b->id); } + inline bool operator()(const uint64_t a,const T *b) const { return (a < b->id); } + inline bool operator()(const T *a,const uint64_t b) const { return (a->id < b); } + inline bool operator()(const uint64_t a,const uint64_t b) const { return (a < b); } + }; + + // Used to track push state for network config tags[] and capabilities[] entries + struct _LocalCredentialPushState + { + _LocalCredentialPushState() : lastPushed(0),id(0) {} + uint64_t lastPushed; + uint32_t id; + }; + public: + enum AddCredentialResult + { + ADD_REJECTED, + ADD_ACCEPTED_NEW, + ADD_ACCEPTED_REDUNDANT, + ADD_DEFERRED_FOR_WHOIS + }; + /** - * A wrapper to iterate through member capabilities in ascending order of capability ID and return only valid ones + * Iterator to scan forward through capabilities in ascending order of ID */ class CapabilityIterator { public: - CapabilityIterator(const Membership &m) : - _m(m), - _i(m._caps.begin()), - _e(m._caps.end()) - { - } + CapabilityIterator(const Membership &m,const NetworkConfig &nconf) : + _m(&m), + _c(&nconf), + _i(&(m._remoteCaps[0])) {} - inline const Capability *next(const NetworkConfig &nconf) + inline const Capability *next() { - while (_i != _e) { - if ((_i->second.lastReceived)&&(_m.isCredentialTimestampValid(nconf,_i->second.cap))) - return &((_i++)->second.cap); - else ++_i; + for(;;) { + if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != 0xffffffffffffffffULL)) { + const Capability *tmp = &((*_i)->cap); + if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { + ++_i; + return tmp; + } else ++_i; + } else { + return (const Capability *)0; + } } - return (const Capability *)0; } private: - const Membership &_m; - std::map::const_iterator _i,_e; + const Membership *_m; + const NetworkConfig *_c; + const _RemoteCapability *const *_i; }; friend class CapabilityIterator; - Membership() : - _lastUpdatedMulticast(0), - _lastPushAttempt(0), - _lastPushedCom(0), - _blacklistBefore(0), - _com(), - _caps(), - _tags(8) + /** + * Iterator to scan forward through tags in ascending order of ID + */ + class TagIterator { - } + public: + TagIterator(const Membership &m,const NetworkConfig &nconf) : + _m(&m), + _c(&nconf), + _i(&(m._remoteTags[0])) {} + + inline const Tag *next() + { + for(;;) { + if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != 0xffffffffffffffffULL)) { + const Tag *tmp = &((*_i)->tag); + if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { + ++_i; + return tmp; + } else ++_i; + } else { + return (const Tag *)0; + } + } + } + + private: + const Membership *_m; + const NetworkConfig *_c; + const _RemoteTag *const *_i; + }; + friend class TagIterator; + + Membership(); /** * Send COM and other credentials to this peer if needed @@ -122,9 +178,10 @@ public: * @param now Current time * @param peerAddress Address of member peer (the one that this Membership describes) * @param nconf My network config - * @param cap Capability to send or 0 if none + * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none + * @param force If true, send objects regardless of last push time */ - void sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap); + void pushCredentials(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** * Check whether we should push MULTICAST_LIKEs to this peer @@ -142,6 +199,8 @@ public: inline void likingMulticasts(const uint64_t now) { _lastUpdatedMulticast = now; } /** + * Check whether the peer represented by this Membership should be allowed on this network at all + * * @param nconf Our network config * @return True if this peer is allowed on this network at all */ @@ -149,126 +208,48 @@ public: { if (nconf.isPublic()) return true; - if ((_blacklistBefore)&&(_com.timestamp().first <= _blacklistBefore)) + if ((_comRevocationThreshold)&&(_com.timestamp().first <= _comRevocationThreshold)) return false; return nconf.com.agreesWith(_com); } /** - * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time - * - * @param cred Credential to check -- must have timestamp() accessor method - * @return True if credential is NOT expired + * @param nconf Network configuration + * @param id Capablity ID + * @return Pointer to capability or NULL if not found */ - template - inline bool isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred) const - { - const uint64_t ts = cred.timestamp(); - const uint64_t delta = (ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts); - return ((delta <= nconf.credentialTimeMaxDelta)&&(ts > _blacklistBefore)); - } + const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const; /** * @param nconf Network configuration * @param id Tag ID * @return Pointer to tag or NULL if not found */ - inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const - { - const TState *t = _tags.get(id); - return ((t) ? (((t->lastReceived != 0)&&(isCredentialTimestampValid(nconf,t->tag))) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); - } - - /** - * @param nconf Network configuration - * @param ids Array to store IDs into - * @param values Array to store values into - * @param maxTags Capacity of ids[] and values[] - * @return Number of tags added to arrays - */ - inline unsigned int getAllTags(const NetworkConfig &nconf,uint32_t *ids,uint32_t *values,unsigned int maxTags) const - { - unsigned int n = 0; - uint32_t *id = (uint32_t *)0; - TState *ts = (TState *)0; - Hashtable::Iterator i(const_cast(this)->_tags); - while (i.next(id,ts)) { - if ((ts->lastReceived)&&(isCredentialTimestampValid(nconf,ts->tag))) { - if (n >= maxTags) - return n; - ids[n] = *id; - values[n] = ts->tag.value(); - } - } - return n; - } - - /** - * @param nconf Network configuration - * @param id Capablity ID - * @return Pointer to capability or NULL if not found - */ - inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const - { - std::map::const_iterator c(_caps.find(id)); - return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(isCredentialTimestampValid(nconf,c->second.cap))) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); - } + const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const; /** * Validate and add a credential if signature is okay and it's otherwise good - * - * @param RR Runtime environment - * @param com Certificate of membership - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good - * - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const Tag &tag); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag); /** * Validate and add a credential if signature is okay and it's otherwise good - * - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const Capability &cap); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap); - /** - * Blacklist COM, tags, and capabilities before this time - * - * @param ts Blacklist cutoff - */ - inline void blacklistBefore(const uint64_t ts) { _blacklistBefore = ts; } - - /** - * Clean up old or stale entries - * - * @param nconf Network config - */ - inline void clean(const NetworkConfig &nconf) +private: + template + inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred,const CS &state) const { - for(std::map::iterator i(_caps.begin());i!=_caps.end();) { - if (!isCredentialTimestampValid(nconf,i->second.cap)) { - _caps.erase(i++); - } else { - ++i; - } - } - - uint32_t *i = (uint32_t *)0; - TState *ts = (TState *)0; - Hashtable::Iterator tsi(_tags); - while (tsi.next(i,ts)) { - if (!isCredentialTimestampValid(nconf,ts->tag)) - _tags.erase(*i); - } + const uint64_t ts = cred.timestamp(); + return ( (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) && (ts > state.revocationThreshold) ); } -private: // Last time we pushed MULTICAST_LIKE(s) uint64_t _lastUpdatedMulticast; @@ -278,17 +259,23 @@ private: // Last time we pushed our COM to this peer uint64_t _lastPushedCom; - // Time before which to blacklist credentials from this peer - uint64_t _blacklistBefore; + // Revocation threshold for COM or 0 if none + uint64_t _comRevocationThreshold; - // COM from this peer + // Remote member's latest network COM CertificateOfMembership _com; - // Capability-related state (we need an ordered container here, hence std::map) - std::map _caps; + // Sorted (in ascending order of ID) arrays of pointers to remote tags and capabilities + _RemoteTag *_remoteTags[ZT_MAX_NETWORK_TAGS]; + _RemoteCapability *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; + + // This is the RAM allocated for remote tags and capabilities from which the sorted arrays are populated + _RemoteTag _tagMem[ZT_MAX_NETWORK_TAGS]; + _RemoteCapability _capMem[ZT_MAX_NETWORK_CAPABILITIES]; - // Tag-related state - Hashtable _tags; + // Local credential push state tracking + _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; + _LocalCredentialPushState _localCaps[ZT_MAX_NETWORK_CAPABILITIES]; }; } // namespace ZeroTier diff --git a/node/Network.cpp b/node/Network.cpp index 455e185e..0fab6a27 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -36,7 +36,7 @@ #include "Peer.hpp" // Uncomment to make the rules engine dump trace info to stdout -//#define ZT_RULES_ENGINE_DEBUGGING 1 +#define ZT_RULES_ENGINE_DEBUGGING 1 namespace ZeroTier { @@ -155,24 +155,21 @@ enum _doZtFilterResult static _doZtFilterResult _doZtFilter( const RuntimeEnvironment *RR, const NetworkConfig &nconf, + const Membership *membership, // can be NULL const bool inbound, const Address &ztSource, - Address &ztDest, // MUTABLE + Address &ztDest, // MUTABLE -- is changed on REDIRECT actions const MAC &macSource, const MAC &macDest, const uint8_t *const frameData, const unsigned int frameLen, const unsigned int etherType, const unsigned int vlanId, - const ZT_VirtualNetworkRule *rules, + const ZT_VirtualNetworkRule *rules, // cannot be NULL const unsigned int ruleCount, - const Tag *localTags, - const unsigned int localTagCount, - const uint32_t *const remoteTagIds, - const uint32_t *const remoteTagValues, - const unsigned int remoteTagCount, - Address &cc, // MUTABLE - unsigned int &ccLength) // MUTABLE + Address &cc, // MUTABLE -- set to TEE destination if TEE action is taken or left alone otherwise + unsigned int &ccLength, // MUTABLE -- set to length of packet payload to TEE + bool &ccWatch) // MUTABLE -- set to true for WATCH target as opposed to normal TEE { #ifdef ZT_RULES_ENGINE_DEBUGGING char dpbuf[1024]; // used by FILTER_TRACE macro @@ -204,6 +201,7 @@ static _doZtFilterResult _doZtFilter( // These are initially handled together since preliminary logic is common case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_WATCH: case ZT_NETWORK_RULE_ACTION_REDIRECT: { const Address fwdAddr(rules[rn].v.fwd.address); if (fwdAddr == ztSource) { @@ -242,6 +240,7 @@ static _doZtFilterResult _doZtFilter( #endif // ZT_RULES_ENGINE_DEBUGGING cc = fwdAddr; ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen; + ccWatch = (rt == ZT_NETWORK_RULE_ACTION_WATCH); } } } continue; @@ -508,50 +507,40 @@ static _doZtFilterResult _doZtFilter( case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { - const Tag *lt = (const Tag *)0; - for(unsigned int i=0;i 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); - } else { - const uint32_t *rtv = (const uint32_t *)0; - for(unsigned int i=0;i 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); - } else { - thisRuleMatches = 1; - FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); - } - } else { + const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); + if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { + const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0); + if (remoteTag) { + const uint32_t ltv = localTag->value(); + const uint32_t rtv = remoteTag->value(); if (rt == ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE) { - const uint32_t diff = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); + const uint32_t diff = (ltv > rtv) ? (ltv - rtv) : (rtv - ltv); thisRuleMatches = (uint8_t)(diff <= rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u difference:%u<=%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,diff,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); + FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u difference:%u<=%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,diff,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { - thisRuleMatches = (uint8_t)((lt->value() & *rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + thisRuleMatches = (uint8_t)((ltv & rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) { - thisRuleMatches = (uint8_t)((lt->value() | *rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x | remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + thisRuleMatches = (uint8_t)((ltv | rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x | remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { - thisRuleMatches = (uint8_t)((lt->value() ^ *rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + thisRuleMatches = (uint8_t)((ltv ^ rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { // sanity check, can't really happen thisRuleMatches = 0; } + } else { + if (inbound) { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } else { + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } break; @@ -582,7 +571,6 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : _portInitialized(false), _inboundConfigPacketId(0), _lastConfigUpdate(0), - _lastRequestedConfiguration(0), _destroyed(false), _netconfFailure(NETCONF_FAILURE_NONE), _portError(0) @@ -598,7 +586,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : if (conf.length()) { dconf->load(conf.c_str()); if (nconf->fromDictionary(*dconf)) { - this->setConfiguration(*nconf,false); + this->_setConfiguration(*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; } @@ -646,32 +634,27 @@ bool Network::filterOutgoingPacket( const unsigned int etherType, const unsigned int vlanId) { - uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; - uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + const uint64_t now = RR->node->now(); Address ztFinalDest(ztDest); - Address cc; - const Capability *relevantCap = (const Capability *)0; - unsigned int ccLength = 0; + int localCapabilityIndex = -1; bool accept = false; - const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); - Membership *m = (Membership *)0; - unsigned int remoteTagCount = 0; - if (ztDest) { - m = &(_memberships[ztDest]); - remoteTagCount = m->getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - } + Membership *const membership = (ztDest) ? _memberships.get(ztDest) : (Membership *)0; - switch(_doZtFilter(RR,_config,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + Address cc; + unsigned int ccLength = 0; + bool ccWatch = false; + switch(_doZtFilter(RR,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { case DOZTFILTER_NO_MATCH: for(unsigned int c=0;c<_config.capabilityCount;++c) { - ztFinalDest = ztDest; // sanity check + ztFinalDest = ztDest; // sanity check, shouldn't be possible if there was no match Address cc2; unsigned int ccLength2 = 0; - switch (_doZtFilter(RR,_config,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + bool ccWatch2 = false; + switch (_doZtFilter(RR,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),cc2,ccLength2,ccWatch2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -679,16 +662,16 @@ bool Network::filterOutgoingPacket( case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side - relevantCap = &(_config.capabilities[c]); + localCapabilityIndex = (int)c; accept = true; if ((!noTee)&&(cc2)) { Membership &m2 = _membership(cc2); - m2.sendCredentialsIfNeeded(RR,now,cc2,_config,relevantCap); + m2.pushCredentials(RR,now,cc2,_config,localCapabilityIndex,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + outp.append((uint8_t)(ccWatch2 ? 0x16 : 0x02)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -715,13 +698,16 @@ bool Network::filterOutgoingPacket( } if (accept) { + if (membership) + membership->pushCredentials(RR,now,ztDest,_config,localCapabilityIndex,false); + if ((!noTee)&&(cc)) { Membership &m2 = _membership(cc); - m2.sendCredentialsIfNeeded(RR,now,cc,_config,relevantCap); + m2.pushCredentials(RR,now,cc,_config,localCapabilityIndex,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + outp.append((uint8_t)(ccWatch ? 0x16 : 0x02)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -732,11 +718,11 @@ bool Network::filterOutgoingPacket( if ((ztDest != ztFinalDest)&&(ztFinalDest)) { Membership &m2 = _membership(ztFinalDest); - m2.sendCredentialsIfNeeded(RR,now,ztFinalDest,_config,relevantCap); + m2.pushCredentials(RR,now,ztFinalDest,_config,localCapabilityIndex,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + outp.append((uint8_t)0x04); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -745,11 +731,9 @@ bool Network::filterOutgoingPacket( RR->sw->send(outp,true); return false; // DROP locally, since we redirected - } else if (m) { - m->sendCredentialsIfNeeded(RR,now,ztDest,_config,relevantCap); + } else { + return true; } - - return true; } else { return false; } @@ -765,28 +749,27 @@ int Network::filterIncomingPacket( const unsigned int etherType, const unsigned int vlanId) { - uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; - uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; Address ztFinalDest(ztDest); - Address cc; - unsigned int ccLength = 0; int accept = 0; Mutex::Lock _l(_lock); - Membership &m = _membership(sourcePeer->address()); - const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); + Membership &membership = _membership(sourcePeer->address()); - switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + Address cc; + unsigned int ccLength = 0; + bool ccWatch = false; + switch (_doZtFilter(RR,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { case DOZTFILTER_NO_MATCH: { - Membership::CapabilityIterator mci(m); + Membership::CapabilityIterator mci(membership,_config); const Capability *c; - while ((c = mci.next(_config))) { - ztFinalDest = ztDest; // sanity check + while ((c = mci.next())) { + ztFinalDest = ztDest; // sanity check, should be unmodified if there was no match Address cc2; unsigned int ccLength2 = 0; - switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + bool ccWatch2 = false; + switch(_doZtFilter(RR,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),cc2,ccLength2,ccWatch2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -801,11 +784,11 @@ int Network::filterIncomingPacket( if (accept) { if (cc2) { - _membership(cc2).sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,(const Capability *)0); + _membership(cc2).pushCredentials(RR,RR->node->now(),cc2,_config,-1,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + outp.append((uint8_t)(ccWatch2 ? 0x1c : 0x08)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -832,11 +815,11 @@ int Network::filterIncomingPacket( if (accept) { if (cc) { - _membership(cc).sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,(const Capability *)0); + _membership(cc).pushCredentials(RR,RR->node->now(),cc,_config,-1,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + outp.append((uint8_t)(ccWatch ? 0x1c : 0x08)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -846,11 +829,11 @@ int Network::filterIncomingPacket( } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { - _membership(ztFinalDest).sendCredentialsIfNeeded(RR,RR->node->now(),ztFinalDest,_config,(const Capability *)0); + _membership(ztFinalDest).pushCredentials(RR,RR->node->now(),ztFinalDest,_config,-1,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + outp.append((uint8_t)0x0a); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -892,60 +875,6 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) _myMulticastGroups.erase(i); } -bool Network::applyConfiguration(const NetworkConfig &conf) -{ - if (_destroyed) // sanity check - return false; - try { - if ((conf.networkId == _id)&&(conf.issuedTo == RR->identity.address())) { - ZT_VirtualNetworkConfig ctmp; - bool portInitialized; - { - Mutex::Lock _l(_lock); - _config = conf; - _lastConfigUpdate = RR->node->now(); - _netconfFailure = NETCONF_FAILURE_NONE; - _externalConfig(&ctmp); - portInitialized = _portInitialized; - _portInitialized = true; - } - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(portInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); - return true; - } else { - TRACE("ignored invalid configuration for network %.16llx (configuration contains mismatched network ID or issued-to address)",(unsigned long long)_id); - } - } catch (std::exception &exc) { - TRACE("ignored invalid configuration for network %.16llx (%s)",(unsigned long long)_id,exc.what()); - } catch ( ... ) { - TRACE("ignored invalid configuration for network %.16llx (unknown exception)",(unsigned long long)_id); - } - return false; -} - -int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) -{ - try { - { - Mutex::Lock _l(_lock); - if (_config == nconf) - return 1; // OK config, but duplicate of what we already have - } - if (applyConfiguration(nconf)) { - if (saveToDisk) { - char n[64]; - Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - Dictionary d; - if (nconf.toDictionary(d,false)) - RR->node->dataStorePut(n,(const void *)d.data(),d.sizeBytes(),true); - } - return 2; // OK and configuration has changed - } - } catch ( ... ) { - TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); - } - return 0; -} - void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize) { std::string newConfig; @@ -979,7 +908,8 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d Identity controllerId(RR->topology->getIdentity(this->controller())); if (controllerId) { if (nc->fromDictionary(*dict)) { - this->setConfiguration(*nc,true); + Mutex::Lock _l(_lock); + this->_setConfiguration(*nc,true); } else { TRACE("error parsing new config with length %u: deserialization of NetworkConfig failed (certificate error?)",(unsigned int)newConfig.length()); } @@ -997,12 +927,6 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d void Network::requestConfiguration() { - // Sanity limit: do not request more often than once per second - const uint64_t now = RR->node->now(); - if ((now - _lastRequestedConfiguration) < 1000ULL) - return; - _lastRequestedConfiguration = RR->node->now(); - const Address ctrl(controller()); Dictionary rmd; @@ -1024,9 +948,10 @@ void Network::requestConfiguration() if (RR->localNetworkController) { NetworkConfig nconf; switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,nconf)) { - case NetworkController::NETCONF_QUERY_OK: - this->setConfiguration(nconf,true); - return; + case NetworkController::NETCONF_QUERY_OK: { + Mutex::Lock _l(_lock); + this->_setConfiguration(nconf,true); + } return; case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: this->setNotFound(); return; @@ -1073,7 +998,7 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) { if (!m) m = &(_membership(peer->address())); - m->sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0); + m->pushCredentials(RR,now,peer->address(),_config,-1,false); if (m->shouldLikeMulticasts(now)) { _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); m->likingMulticasts(now); @@ -1124,9 +1049,8 @@ void Network::clean() Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - if (RR->topology->getPeerNoCache(*a)) - m->clean(_config); - else _memberships.erase(*a); + if (!RR->topology->getPeerNoCache(*a)) + _memberships.erase(*a); } } } @@ -1177,21 +1101,25 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) _sendUpdatesToMembers(&mg); } -int Network::addCredential(const CertificateOfMembership &com) +Membership::AddCredentialResult Network::addCredential(const CertificateOfMembership &com) { if (com.networkId() != _id) - return -1; + return Membership::ADD_REJECTED; const Address a(com.issuedTo()); Mutex::Lock _l(_lock); Membership &m = _membership(a); - const int result = m.addCredential(RR,com); - if (result == 0) { - m.sendCredentialsIfNeeded(RR,RR->node->now(),a,_config,(const Capability *)0); + const Membership::AddCredentialResult result = m.addCredential(RR,_config,com); + if ((result == Membership::ADD_ACCEPTED_NEW)||(result == Membership::ADD_ACCEPTED_REDUNDANT)) { + m.pushCredentials(RR,RR->node->now(),a,_config,-1,false); RR->mc->addCredential(com,true); } return result; } +Membership::AddCredentialResult Network::addCredential(const Revocation &rev) +{ +} + void Network::destroy() { Mutex::Lock _l(_lock); @@ -1215,6 +1143,39 @@ ZT_VirtualNetworkStatus Network::_status() const } } +int Network::_setConfiguration(const NetworkConfig &nconf,bool saveToDisk) +{ + // assumes _lock is locked + try { + if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) + return 0; + if (_config == nconf) + return 1; // OK config, but duplicate of what we already have + + ZT_VirtualNetworkConfig ctmp; + _config = nconf; + _lastConfigUpdate = RR->node->now(); + _netconfFailure = NETCONF_FAILURE_NONE; + _externalConfig(&ctmp); + const bool oldPortInitialized = _portInitialized; + _portInitialized = true; + _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + + if (saveToDisk) { + char n[64]; + Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); + Dictionary d; + if (nconf.toDictionary(d,false)) + RR->node->dataStorePut(n,(const void *)d.data(),d.sizeBytes(),true); + } + + return 2; // OK and configuration has changed + } catch ( ... ) { + TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); + } + return 0; +} + void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const { // assumes _lock is locked @@ -1308,7 +1269,7 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - m->sendCredentialsIfNeeded(RR,now,*a,_config,(const Capability *)0); + m->pushCredentials(RR,now,*a,_config,-1,false); if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { if (!newMulticastGroup) m->likingMulticasts(now); diff --git a/node/Network.hpp b/node/Network.hpp index c85e5993..a151fb88 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -62,6 +62,11 @@ public: */ static const MulticastGroup BROADCAST; + /** + * Compute primary controller device ID from network ID + */ + static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); } + /** * Construct a new network * @@ -76,14 +81,24 @@ public: ~Network(); + inline uint64_t id() const { return _id; } + inline Address controller() const { return Address(_id >> 24); } + inline bool multicastEnabled() const { return (_config.multicastLimit > 0); } + inline bool hasConfig() const { return (_config); } + inline uint64_t lastConfigUpdate() const throw() { return _lastConfigUpdate; } + inline ZT_VirtualNetworkStatus status() const { Mutex::Lock _l(_lock); return _status(); } + inline const NetworkConfig &config() const { return _config; } + inline const MAC &mac() const { return _mac; } + /** * Apply filters to an outgoing packet * * This applies filters from our network config and, if that doesn't match, * our capabilities in ascending order of capability ID. Additional actions - * such as TEE may be taken, and credentials may be pushed. + * such as TEE may be taken, and credentials may be pushed, so this is not + * side-effect-free. It's basically step one in sending something over VL2. * - * @param noTee If true, do not TEE anything anywhere + * @param noTee If true, do not TEE anything anywhere (for two-pass filtering as done with multicast and bridging) * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -134,42 +149,10 @@ public: const unsigned int vlanId); /** - * @return Network ID - */ - inline uint64_t id() const throw() { return _id; } - - /** - * @return Address of network's controller (most significant 40 bits of ID) - */ - inline Address controller() const throw() { return Address(_id >> 24); } - - /** - * @param nwid Network ID - * @return Address of network's controller - */ - static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); } - - /** - * @return Multicast group memberships for this network's port (local, not learned via bridging) - */ - inline std::vector multicastGroups() const - { - Mutex::Lock _l(_lock); - return _myMulticastGroups; - } - - /** - * @return All multicast groups including learned groups that are behind any bridges we're attached to - */ - inline std::vector allMulticastGroups() const - { - Mutex::Lock _l(_lock); - return _allMulticastGroups(); - } - - /** + * Check whether we are subscribed to a multicast group + * * @param mg Multicast group - * @param includeBridgedGroups If true, also include any groups we've learned via bridging + * @param includeBridgedGroups If true, also check groups we've learned via bridging * @return True if this network endpoint / peer is a member */ bool subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const; @@ -188,37 +171,19 @@ public: */ void multicastUnsubscribe(const MulticastGroup &mg); - /** - * Apply a NetworkConfig to this network - * - * @param conf Configuration in NetworkConfig form - * @return True if configuration was accepted - */ - bool applyConfiguration(const NetworkConfig &conf); - - /** - * Set or update this network's configuration - * - * @param nconf Network configuration - * @param saveToDisk IF true (default), write config to disk - * @return 0 -- rejected, 1 -- accepted but not new, 2 -- accepted new config - */ - int setConfiguration(const NetworkConfig &nconf,bool saveToDisk); - /** * Handle an inbound network config chunk * - * Only chunks whose inRePacketId matches the packet ID of the last request - * are handled. If this chunk completes the config, it is decoded and - * setConfiguration() is called. + * This is called from IncomingPacket when we receive a chunk from a network + * controller. * - * @param inRePacketId In-re packet ID from OK(NETWORK_CONFIG_REQUEST) + * @param requestId An ID for grouping chunks, e.g. in-re packet ID for OK(NETWORK_CONFIG_REQUEST) * @param data Chunk data * @param chunkSize Size of data[] * @param chunkIndex Index of chunk in full config * @param totalSize Total size of network config */ - void handleInboundConfigChunk(const uint64_t inRePacketId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize); + void handleInboundConfigChunk(const uint64_t requestId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize); /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this @@ -230,7 +195,7 @@ public: } /** - * Set netconf failure to 'not found' -- called by PacketDecider when controller reports this + * Set netconf failure to 'not found' -- called by IncomingPacket when controller reports this */ inline void setNotFound() { @@ -240,10 +205,6 @@ public: /** * Causes this network to request an updated configuration from its master node now - * - * There is a circuit breaker here to prevent this from being done more often - * than once per second. This is to prevent things like NETWORK_CONFIG_REFRESH - * from causing multiple requests. */ void requestConfiguration(); @@ -251,7 +212,7 @@ public: * Determine whether this peer is permitted to communicate on this network * * This also performs certain periodic actions such as pushing renewed - * credentials to peers or requesting them if not present. + * credentials to peers, so like the filters it is not side-effect-free. * * @param peer Peer to check * @param verb Packet verb @@ -266,7 +227,7 @@ public: bool gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); /** - * Perform cleanup and possibly save state + * Do periodic cleanup and housekeeping tasks */ void clean(); @@ -279,46 +240,6 @@ public: _sendUpdatesToMembers((const MulticastGroup *)0); } - /** - * @return Time of last updated configuration or 0 if none - */ - inline uint64_t lastConfigUpdate() const throw() { return _lastConfigUpdate; } - - /** - * @return Status of this network - */ - inline ZT_VirtualNetworkStatus status() const - { - Mutex::Lock _l(_lock); - return _status(); - } - - /** - * @param ec Buffer to fill with externally-visible network configuration - */ - inline void externalConfig(ZT_VirtualNetworkConfig *ec) const - { - Mutex::Lock _l(_lock); - _externalConfig(ec); - } - - /** - * Get current network config - * - * @return Network configuration (may be a null config if we don't have one yet) - */ - inline const NetworkConfig &config() const { return _config; } - - /** - * @return True if this network has a valid config - */ - inline bool hasConfig() const { return (_config); } - - /** - * @return Ethernet MAC address for this network's local interface - */ - inline const MAC &mac() const { return _mac; } - /** * Find the node on this network that has this MAC behind it (if any) * @@ -349,44 +270,47 @@ public: void learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now); /** - * @param com Certificate of membership - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + * Validate a credential and learn it if it passes certificate and other checks */ - int addCredential(const CertificateOfMembership &com); + Membership::AddCredentialResult addCredential(const CertificateOfMembership &com); /** - * @param cap Capability - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + * Validate a credential and learn it if it passes certificate and other checks */ - inline int addCredential(const Capability &cap) + inline Membership::AddCredentialResult addCredential(const Capability &cap) { if (cap.networkId() != _id) - return -1; + return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(cap.issuedTo()).addCredential(RR,cap); + return _membership(cap.issuedTo()).addCredential(RR,_config,cap); } /** - * @param cap Tag - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + * Validate a credential and learn it if it passes certificate and other checks */ - inline int addCredential(const Tag &tag) + inline Membership::AddCredentialResult addCredential(const Tag &tag) { if (tag.networkId() != _id) - return -1; + return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(tag.issuedTo()).addCredential(RR,tag); + return _membership(tag.issuedTo()).addCredential(RR,_config,tag); } /** - * Blacklist COM, tags, and capabilities before this time + * Validate a credential and learn it if it passes certificate and other checks + */ + Membership::AddCredentialResult addCredential(const Revocation &rev); + + /** + * Force push credentials (COM, etc.) to a peer now * - * @param ts Blacklist cutoff + * @param to Destination peer address + * @param now Current time */ - inline void blacklistBefore(const Address &peerAddress,const uint64_t ts) + inline void pushCredentialsNow(const Address &to,const uint64_t now) { Mutex::Lock _l(_lock); - _membership(peerAddress).blacklistBefore(ts); + _membership(to).pushCredentials(RR,now,to,_config,-1,true); } /** @@ -399,11 +323,23 @@ public: void destroy(); /** - * @return Pointer to user PTR (modifiable user ptr used in API) + * Get this network's config for export via the ZT core API + * + * @param ec Buffer to fill with externally-visible network configuration + */ + inline void externalConfig(ZT_VirtualNetworkConfig *ec) const + { + Mutex::Lock _l(_lock); + _externalConfig(ec); + } + + /** + * @return Externally usable pointer-to-pointer exported via the core API */ inline void **userPtr() throw() { return &_uPtr; } private: + int _setConfiguration(const NetworkConfig &nconf,bool saveToDisk); ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); @@ -412,9 +348,9 @@ private: std::vector _allMulticastGroups() const; Membership &_membership(const Address &a); - const RuntimeEnvironment *RR; + const RuntimeEnvironment *const RR; void *_uPtr; - uint64_t _id; + const uint64_t _id; uint64_t _lastAnnouncedMulticastGroupsUpstream; MAC _mac; // local MAC address volatile bool _portInitialized; @@ -428,7 +364,6 @@ private: NetworkConfig _config; volatile uint64_t _lastConfigUpdate; - volatile uint64_t _lastRequestedConfiguration; volatile bool _destroyed; diff --git a/node/Packet.hpp b/node/Packet.hpp index 2ca73a84..03b9b113 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -655,15 +655,27 @@ public: * * Flags: * 0x01 - Certificate of network membership attached (DEPRECATED) - * 0x02 - This is a TEE'd or REDIRECT'ed packet - * 0x04 - TEE/REDIRECT'ed packet is from inbound side - * + * 0x02 - Most significant bit of subtype (see below) + * 0x04 - Middle bit of subtype (see below) + * 0x08 - Least significant bit of subtype (see below) + * 0x10 - ACK requested in the form of OK(EXT_FRAME) + * + * Subtypes (0..7): + * 0x0 - Normal frame (bridging can be determined by checking MAC) + * 0x1 - TEEd outbound frame + * 0x2 - REDIRECTed outbound frame + * 0x3 - WATCHed outbound frame (TEE with ACK, ACK bit also set) + * 0x4 - TEEd inbound frame + * 0x5 - REDIRECTed inbound frame + * 0x6 - WATCHed inbound frame + * 0x7 - (reserved for future use) + * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we * want to attach a certificate since FRAME does not support that. * - * ERROR may be generated if a membership certificate is needed for a - * closed network. Payload will be network ID. + * If the ACK flag (0x08) is set, an OK(EXT_FRAME) is sent with + * no payload to acknowledge receipt of the frame. */ VERB_EXT_FRAME = 0x07, @@ -698,7 +710,7 @@ public: VERB_MULTICAST_LIKE = 0x09, /** - * Network membership credential push: + * Network credentials push: * <[...] serialized certificate of membership> * [<[...] additional certificates of membership>] * <[1] 0x00, null byte marking end of COM array> @@ -706,12 +718,12 @@ public: * <[...] one or more serialized Capability> * <[2] 16-bit number of tags> * <[...] one or more serialized Tags> + * <[2] 16-bit number of revocations> + * <[...] one or more serialized Revocations> * - * This is sent in response to ERROR_NEED_MEMBERSHIP_CERTIFICATE and may - * be pushed at any other time to keep exchanged certificates up to date. - * - * COMs and other credentials need not be for the same network, since each - * includes its own network ID and signature. + * This can be sent by anyone at any time to push network credentials. + * These will of course only be accepted if they are properly signed. + * Credentials can be for any number of networks. * * OK/ERROR are not generated. */ @@ -742,23 +754,18 @@ public: VERB_NETWORK_CONFIG_REQUEST = 0x0b, /** - * Network configuration update push: - * <[8] network ID to refresh> - * <[2] 16-bit number of address/timestamp pairs to blacklist> - * [<[5] ZeroTier address of peer being revoked>] - * [<[8] blacklist credentials older than this timestamp>] - * [<[...] additional address/timestamp pairs>] - * - * This can be sent by a network controller to both request that a network - * config be updated and push instantaneous revocations of specific peers - * or peer credentials. - * - * Specific revocations can be pushed to blacklist a specific peer's - * credentials (COM, tags, and capabilities) if older than a specified - * timestamp. This can be used to accomplish expedited revocation of - * a peer's access to things on a network or to the network itself among - * those other peers that can currently reach the controller. This is not - * the only mechanism for revocation of course, but it's the fastest. + * Network configuration push: + * <[8] 64-bit network ID> + * <[8] 64-bit value used to group chunks in this push> + * <[2] 16-bit length of network configuration dictionary chunk> + * <[...] network configuration dictionary (may be incomplete)> + * <[4] 32-bit total length of assembled dictionary> + * <[4] 32-bit index of chunk in this reply> + * + * This is a direct push variant for network config updates. It otherwise + * carries the same payload as OK(NETWORK_CONFIG_REQUEST). There is an + * extra number after network ID in this version that is used in place of + * the in-re packet ID sent with OKs to group chunks together. */ VERB_NETWORK_CONFIG_REFRESH = 0x0c, diff --git a/node/Revocation.cpp b/node/Revocation.cpp new file mode 100644 index 00000000..420476a4 --- /dev/null +++ b/node/Revocation.cpp @@ -0,0 +1,46 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "Revocation.hpp" +#include "RuntimeEnvironment.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Switch.hpp" +#include "Network.hpp" + +namespace ZeroTier { + +int Revocation::verify(const RuntimeEnvironment *RR) const +{ + if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) + return -1; + const Identity id(RR->topology->getIdentity(_signedBy)); + if (!id) { + RR->sw->requestWhois(_signedBy); + return 1; + } + try { + Buffer tmp; + this->serialize(tmp,true); + return (id.verify(tmp.data(),tmp.size(),_signature) ? 0 : -1); + } catch ( ... ) { + return -1; + } +} + +} // namespace ZeroTier diff --git a/node/Revocation.hpp b/node/Revocation.hpp new file mode 100644 index 00000000..58757465 --- /dev/null +++ b/node/Revocation.hpp @@ -0,0 +1,178 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_REVOCATION_HPP +#define ZT_REVOCATION_HPP + +#include +#include +#include +#include + +#include "Constants.hpp" +#include "../include/ZeroTierOne.h" +#include "Address.hpp" +#include "C25519.hpp" +#include "Utils.hpp" +#include "Buffer.hpp" +#include "Identity.hpp" + +/** + * Flag: fast propagation via rumor mill algorithm + */ +#define ZT_REVOCATION_FLAG_FAST_PROPAGATE 0x1ULL + +namespace ZeroTier { + +class RuntimeEnvironment; + +/** + * Revocation certificate to instantaneously revoke a COM, capability, or tag + */ +class Revocation +{ +public: + enum CredentialType + { + CREDENTIAL_TYPE_NIL = 0, + CREDENTIAL_TYPE_COM = 1, + CREDENTIAL_TYPE_CAPABILITY = 2, + CREDENTIAL_TYPE_TAG = 3 + }; + + Revocation() + { + memset(this,0,sizeof(Revocation)); + } + + Revocation(const uint64_t i,const uint64_t nwid,const uint64_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const CredentialType ct) : + _id(i), + _networkId(nwid), + _credentialId(cid), + _threshold(thr), + _flags(fl), + _target(tgt), + _signedBy(), + _type(ct) {} + + inline uint64_t id() const { return _id; } + inline uint64_t networkId() const { return _networkId; } + inline uint64_t credentialId() const { return _credentialId; } + inline uint64_t threshold() const { return _threshold; } + inline const Address &target() const { return _target; } + inline const Address &signer() const { return _signedBy; } + inline CredentialType type() const { return _type; } + + inline bool fastPropagate() const { return ((_flags & ZT_REVOCATION_FLAG_FAST_PROPAGATE) != 0); } + + /** + * @param signer Signing identity, must have private key + * @return True if signature was successful + */ + inline bool sign(const Identity &signer) + { + if (signer.hasPrivate()) { + Buffer tmp; + this->serialize(tmp,true); + _signedBy = signer.address(); + _signature = signer.sign(tmp.data(),tmp.size()); + return true; + } + return false; + } + + /** + * Verify this revocation's signature + * + * @param RR Runtime environment to provide for peer lookup, etc. + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain + */ + int verify(const RuntimeEnvironment *RR) const; + + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append(_id); + b.append(_networkId); + b.append(_credentialId); + b.append(_threshold); + b.append(_flags); + _target.appendTo(b); + _signedBy.appendTo(b); + b.append((uint8_t)_type); + + if (!forSign) { + b.append((uint8_t)1); // 1 == Ed25519 signature + b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); + b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); + } + + // This is the size of any additional fields, currently 0. + b.append((uint16_t)0); + + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + } + + template + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + memset(this,0,sizeof(Revocation)); + + unsigned int p = startAt; + + _id = b.template at(p); p += 8; + _networkId = b.template at(p); p += 8; + _credentialId = b.template at(p); p += 8; + _threshold = b.template at(p); p += 8; + _flags = b.template at(p); p += 8; + _target.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + _type = (CredentialType)b[p++]; + + if (b[p++] == 1) { + if (b.template at(p) == ZT_C25519_SIGNATURE_LEN) { + p += 2; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); + p += ZT_C25519_SIGNATURE_LEN; + } else throw std::runtime_error("invalid signature"); + } + + p += 2 + b.template at(p); + if (p > b.size()) + throw std::runtime_error("extended field overflow"); + + return (p - startAt); + } + +private: + uint64_t _id; + uint64_t _networkId; + uint64_t _credentialId; + uint64_t _threshold; + uint64_t _flags; + Address _target; + Address _signedBy; + CredentialType _type; + C25519::Signature _signature; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/Tag.cpp b/node/Tag.cpp index 352ecde8..eb4026bc 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -27,7 +27,7 @@ namespace ZeroTier { int Tag::verify(const RuntimeEnvironment *RR) const { - if ((!_signedBy)||(_signedBy != Network::controllerFor(_nwid))) + if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; const Identity id(RR->topology->getIdentity(_signedBy)); if (!id) { diff --git a/node/Tag.hpp b/node/Tag.hpp index 14cc3a5d..97228157 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -67,7 +67,7 @@ public: * @param value Tag value */ Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : - _nwid(nwid), + _networkId(nwid), _ts(ts), _id(id), _value(value), @@ -76,7 +76,7 @@ public: { } - inline uint64_t networkId() const { return _nwid; } + inline uint64_t networkId() const { return _networkId; } inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } @@ -91,13 +91,13 @@ public: */ inline bool sign(const Identity &signer) { - try { - Buffer<(sizeof(Tag) * 2)> tmp; + if (signer.hasPrivate()) { + Buffer tmp; _signedBy = signer.address(); this->serialize(tmp,true); _signature = signer.sign(tmp.data(),tmp.size()); return true; - } catch ( ... ) {} + } return false; } @@ -115,7 +115,7 @@ public: if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); // These are the same between Tag and Capability - b.append(_nwid); + b.append(_networkId); b.append(_ts); b.append(_id); @@ -140,7 +140,7 @@ public: unsigned int p = startAt; // These are the same between Tag and Capability - _nwid = b.template at(p); p += 8; + _networkId = b.template at(p); p += 8; _ts = b.template at(p); p += 8; _id = b.template at(p); p += 4; @@ -168,8 +168,22 @@ public: inline bool operator==(const Tag &t) const { return (memcmp(this,&t,sizeof(Tag)) == 0); } inline bool operator!=(const Tag &t) const { return (memcmp(this,&t,sizeof(Tag)) != 0); } + // For searching sorted arrays or lists of Tags by ID + struct IdComparePredicate + { + inline bool operator()(const Tag &a,const Tag &b) const { return (a.id() < b.id()); } + inline bool operator()(const uint32_t a,const Tag &b) const { return (a < b.id()); } + inline bool operator()(const Tag &a,const uint32_t b) const { return (a.id() < b); } + inline bool operator()(const Tag *a,const Tag *b) const { return (a->id() < b->id()); } + inline bool operator()(const Tag *a,const Tag &b) const { return (a->id() < b.id()); } + inline bool operator()(const Tag &a,const Tag *b) const { return (a.id() < b->id()); } + inline bool operator()(const uint32_t a,const Tag *b) const { return (a < b->id()); } + inline bool operator()(const Tag *a,const uint32_t b) const { return (a->id() < b); } + inline bool operator()(const uint32_t a,const uint32_t b) const { return (a < b); } + }; + private: - uint64_t _nwid; + uint64_t _networkId; uint64_t _ts; uint32_t _id; uint32_t _value; diff --git a/objects.mk b/objects.mk index f92a907e..5738e769 100644 --- a/objects.mk +++ b/objects.mk @@ -17,6 +17,7 @@ OBJS=\ node/Path.o \ node/Peer.o \ node/Poly1305.o \ + node/Revocation.o \ node/Salsa20.o \ node/SelfAwareness.o \ node/SHA512.o \ -- cgit v1.2.3 From 7e4b6b594b9529565b8bb3acb6d99e37c1f3db1b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 26 Sep 2016 17:05:39 -0700 Subject: It now builds. --- controller/EmbeddedNetworkController.cpp | 4 ++-- include/ZeroTierOne.h | 21 --------------------- node/IncomingPacket.cpp | 2 +- node/Node.cpp | 32 -------------------------------- node/Node.hpp | 1 - node/Packet.hpp | 11 ++++++++--- 6 files changed, 11 insertions(+), 60 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5ba8cf98..cd8ce8bf 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -516,8 +516,8 @@ void EmbeddedNetworkController::threadMain() Mutex::Lock _l(_refreshQueue_m); while (_refreshQueue.size() > 0) { _Refresh &r = _refreshQueue.front(); - if (_node) - _node->pushNetworkRefresh(r.dest,r.nwid,r.blacklistAddresses,r.blacklistThresholds,r.numBlacklistEntries); + //if (_node) + // _node->pushNetworkRefresh(r.dest,r.nwid,r.blacklistAddresses,r.blacklistThresholds,r.numBlacklistEntries); _refreshQueue.pop_front(); if (++count >= 50) break; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 591ff1fe..c66b9079 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1928,27 +1928,6 @@ enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,v */ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test); -/** - * Push a network refresh - * - * This is used by network controller implementations to send a - * NETWORK_CONFIG_REFRESH message to tell a node to refresh its - * config and to optionally push one or more credential timestamp - * blacklist thresholds for members of the network. - * - * Code outside a controller implementation will have no use for - * this as these messages are ignored if they do not come from a - * controller. - * - * @param node Node instance - * @param dest ZeroTier address of destination to which to send NETWORK_CONFIG_REFRESH - * @param nwid Network ID - * @param blacklistAddresses Array of ZeroTier addresses of network members to set timestamp blacklists for - * @param blacklistBeforeTimestamps Timestamps before which to blacklist credentials for each corresponding address in blacklistAddresses[] - * @param blacklistCount Size of blacklistAddresses[] and blacklistBeforeTimestamps[] - */ -void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); - /** * Initialize cluster operation * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c50db794..72dfbfd8 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -982,7 +982,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,trustEstablished); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } diff --git a/node/Node.cpp b/node/Node.cpp index 2533eeb6..db9b8ea0 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -552,31 +552,6 @@ void Node::circuitTestEnd(ZT_CircuitTest *test) } } -void Node::pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) -{ - Packet outp(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); - outp.append(nwid); - outp.addSize(2); - unsigned int c = 0; - for(unsigned int i=0;i= ZT_PROTO_MAX_PACKET_LENGTH) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); - RR->sw->send(outp,true); - outp = Packet(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); - outp.append(nwid); - outp.addSize(2); - c = 0; - } - Address(blacklistAddresses[i]).appendTo(outp); - outp.append(blacklistBeforeTimestamps[i]); - ++c; - } - if (c > 0) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); - RR->sw->send(outp,true); - } -} - ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -973,13 +948,6 @@ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test) } catch ( ... ) {} } -void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) -{ - try { - reinterpret_cast(node)->pushNetworkRefresh(dest,nwid,blacklistAddresses,blacklistBeforeTimestamps,blacklistCount); - } catch ( ... ) {} -} - enum ZT_ResultCode ZT_Node_clusterInit( ZT_Node *node, unsigned int myId, diff --git a/node/Node.hpp b/node/Node.hpp index 56869816..11462531 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -107,7 +107,6 @@ public: void setNetconfMaster(void *networkControllerInstance); ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); - void pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); ZT_ResultCode clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, diff --git a/node/Packet.hpp b/node/Packet.hpp index e76cb96c..b03ec327 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -670,9 +670,11 @@ public: * 0x6 - WATCHed inbound frame * 0x7 - (reserved for future use) * - * An extended frame carries full MAC addressing, making them a - * superset of VERB_FRAME. They're used for bridging or when we - * want to attach a certificate since FRAME does not support that. + * An extended frame carries full MAC addressing, making it a + * superset of VERB_FRAME. It is used for bridged traffic, + * redirected or observed traffic via rules, and can in theory + * be used for multicast though MULTICAST_FRAME exists for that + * purpose and has additional options and capabilities. * * OK payload (if ACK flag is set): * <[8] 64-bit network ID> @@ -725,6 +727,9 @@ public: * These will of course only be accepted if they are properly signed. * Credentials can be for any number of networks. * + * The use of a zero byte to terminate the COM section is for legacy + * backward compatiblity. Newer fields are prefixed with a length. + * * OK/ERROR are not generated. */ VERB_NETWORK_CREDENTIALS = 0x0a, -- cgit v1.2.3 From 2fc3d12fb66d94387d537c4d2c4c1e7a7ca4beeb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 29 Sep 2016 14:48:39 -0700 Subject: Minor tweaks to member code in controller, and fix Linux build. --- controller/EmbeddedNetworkController.cpp | 7 +------ controller/EmbeddedNetworkController.hpp | 4 ++++ osdep/ManagedRoute.cpp | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index cd8ce8bf..a93aa027 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1004,12 +1004,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( if (!member.size()) return 404; - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - - // Add non-persisted fields - member["clock"] = OSUtils::now(); - + _addMemberNonPersistedFields(member,now); responseBody = member.dump(2); responseContentType = "application/json"; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 1bfd9577..78d65c3a 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -181,6 +181,10 @@ private: network["activeMemberCount"] = nmi.activeMemberCount; network["totalMemberCount"] = nmi.totalMemberCount; } + inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now) + { + member["clock"] = now; + } // These are const after construction Node *const _node; diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 127f1b7d..1fc6c78e 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -524,11 +524,11 @@ void ManagedRoute::remove() #endif // __BSD__ ------------------------------------------------------------ #ifdef __LINUX__ // ---------------------------------------------------------- - _routeCmd("del",*r,_via,(_via) ? (const char *)0 : _device); + _routeCmd("del",r->first,_via,(_via) ? (const char *)0 : _device); #endif // __LINUX__ ---------------------------------------------------------- #ifdef __WINDOWS__ // -------------------------------------------------------- - _winRoute(true,interfaceLuid,interfaceIndex,*r,_via); + _winRoute(true,interfaceLuid,interfaceIndex,r->first,_via); #endif // __WINDOWS__ -------------------------------------------------------- } -- cgit v1.2.3 From 1eeebba2f7710e6f1ae4c120473c1d0184a920ea Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 29 Sep 2016 17:59:27 -0700 Subject: Drop old /active path from network. --- controller/EmbeddedNetworkController.cpp | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index a93aa027..94d1a7c9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -56,7 +56,7 @@ using json = nlohmann::json; #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 // Nodes are considered active if they've queried in less than this long -#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) +#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD (ZT_NETWORK_AUTOCONF_DELAY * 2) namespace ZeroTier { @@ -1030,32 +1030,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( return 200; } - } else if ((path[2] == "active")&&(path.size() == 3)) { - - responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - auto recentLog = member["recentLog"]; - if ((recentLog.is_array())&&(recentLog.size() > 0)) { - auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&(_jI(mostRecentLog["ts"],0ULL) >= threshold)) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - responseBody.append(mostRecentLog.dump()); - } - } - } - } - } - responseBody.push_back('}'); - responseContentType = "application/json"; - return 200; - } else if ((path[2] == "test")&&(path.size() >= 4)) { Mutex::Lock _l(_circuitTests_m); -- cgit v1.2.3 From f0794e09b70bea92ac0fab65506bc52ddbfb0f65 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Sep 2016 13:04:26 -0700 Subject: Controller cleanup. --- controller/EmbeddedNetworkController.cpp | 43 ++++++++++++++++---------------- controller/EmbeddedNetworkController.hpp | 1 + 2 files changed, 23 insertions(+), 21 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 94d1a7c9..c60f7322 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1004,7 +1004,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( if (!member.size()) return 404; - _addMemberNonPersistedFields(member,now); + _addMemberNonPersistedFields(member,OSUtils::now()); responseBody = member.dump(2); responseContentType = "application/json"; @@ -1324,28 +1324,28 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); if (b.count("v4AssignMode")) { - json &nv4m = network["v4AssignMode"]; - if (!nv4m.is_object()) nv4m = json::object(); - if (b["v4AssignMode"].is_string()) { // backward compatibility - nv4m["zt"] = (_jS(b["v4AssignMode"],"") == "zt"); - } else if (b["v4AssignMode"].is_object()) { - json &v4m = b["v4AssignMode"]; - if (v4m.count("zt")) nv4m["zt"] = _jB(v4m["zt"],false); - } - if (!nv4m.count("zt")) nv4m["zt"] = false; + json nv4m; + json &v4m = b["v4AssignMode"]; + if (v4m.is_string()) { // backward compatibility + nv4m["zt"] = (_jS(v4m,"") == "zt"); + } else if (v4m.is_object()) { + nv4m["zt"] = _jB(v4m["zt"],false); + } else nv4m["zt"] = false; + network["v4AssignMode"] = nv4m; } if (b.count("v6AssignMode")) { - json &nv6m = network["v6AssignMode"]; + json nv6m; + json &v6m = b["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); - if (b["v6AssignMode"].is_string()) { // backward compatibility - std::vector v6m(Utils::split(_jS(b["v6AssignMode"],"").c_str(),",","","")); - std::sort(v6m.begin(),v6m.end()); - v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); + if (v6m.is_string()) { // backward compatibility + std::vector v6ms(Utils::split(_jS(v6m,"").c_str(),",","","")); + std::sort(v6ms.begin(),v6ms.end()); + v6ms.erase(std::unique(v6ms.begin(),v6ms.end()),v6ms.end()); nv6m["rfc4193"] = false; nv6m["zt"] = false; nv6m["6plane"] = false; - for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { + for(std::vector::iterator i(v6ms.begin());i!=v6ms.end();++i) { if (*i == "rfc4193") nv6m["rfc4193"] = true; else if (*i == "zt") @@ -1353,15 +1353,16 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( else if (*i == "6plane") nv6m["6plane"] = true; } - } else if (b["v6AssignMode"].is_object()) { - json &v6m = b["v6AssignMode"]; + } else if (v6m.is_object()) { if (v6m.count("rfc4193")) nv6m["rfc4193"] = _jB(v6m["rfc4193"],false); if (v6m.count("zt")) nv6m["zt"] = _jB(v6m["zt"],false); if (v6m.count("6plane")) nv6m["6plane"] = _jB(v6m["6plane"],false); + } else { + nv6m["rfc4193"] = false; + nv6m["zt"] = false; + nv6m["6plane"] = false; } - if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; - if (!nv6m.count("zt")) nv6m["zt"] = false; - if (!nv6m.count("6plane")) nv6m["6plane"] = false; + network["v6AssignMode"] = nv6m; } if (b.count("routes")) { diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 78d65c3a..bce8890c 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -165,6 +165,7 @@ private: if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); + if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment network["rules"] = { -- cgit v1.2.3 From 988049f39bf7731c0189544b316850b81b004de3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Sep 2016 14:07:00 -0700 Subject: Add new rule to rules engine: random match. --- controller/EmbeddedNetworkController.cpp | 8 ++++++++ include/ZeroTierOne.h | 18 ++++++++++++++---- node/Capability.hpp | 7 +++++++ node/Network.cpp | 4 ++++ 4 files changed, 33 insertions(+), 4 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c60f7322..d656bad3 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -260,6 +260,11 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["start"] = (unsigned int)rule.v.frameSize[0]; r["end"] = (unsigned int)rule.v.frameSize[1]; break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + r["type"] = "MATCH_RANDOM"; + r["not"] = ((rule.t & 0x80) != 0); + r["probability"] = (unsigned long)rule.v.randomProbability; + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: r["type"] = "MATCH_TAGS_DIFFERENCE"; r["not"] = ((rule.t & 0x80) != 0); @@ -441,6 +446,9 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.frameSize[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); rule.v.frameSize[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; + } else if (t == "MATCH_RANDOM") { + rule.t |= ZT_NETWORK_RULE_MATCH_RANDOM; + rule.v.randomProbability = (uint32_t)(_jI(r["probability"],0ULL) & 0xffffffffULL); } else if (t == "MATCH_TAGS_DIFFERENCE") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index c66b9079..ee03c3b1 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -633,25 +633,30 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 50, + /** + * Random match with selectable probability + */ + ZT_NETWORK_RULE_MATCH_RANDOM = 51, + /** * Match if local and remote tags differ by no more than value, use 0 to check for equality */ - ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 51, + ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 52, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 52, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 53, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 53, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 54, /** * Match if local and remote tags XORed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 54, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 55, /** * Maximum ID allowed for a MATCH entry in the rules table @@ -720,6 +725,11 @@ typedef struct */ uint64_t zt; + /** + * 0 = never, UINT32_MAX = always + */ + uint32_t randomProbability; + /** * 48-bit Ethernet MAC address in big-endian order */ diff --git a/node/Capability.hpp b/node/Capability.hpp index 2cf54b5c..e808ad40 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -249,6 +249,10 @@ public: b.append((uint16_t)rules[i].v.frameSize[0]); b.append((uint16_t)rules[i].v.frameSize[1]); break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + b.append((uint8_t)4); + b.append((uint32_t)rules[i].v.randomProbability); + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: @@ -331,6 +335,9 @@ public: rules[ruleCount].v.frameSize[0] = b.template at(p); rules[ruleCount].v.frameSize[1] = b.template at(p + 2); break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + rules[ruleCount].v.randomProbability = b.template at(p); + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: diff --git a/node/Network.cpp b/node/Network.cpp index d38a3fdd..fe899dcc 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -504,6 +504,10 @@ static _doZtFilterResult _doZtFilter( thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); FILTER_TRACE("%u %s %c %u in %u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),frameLen,(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1],(unsigned int)thisRuleMatches); break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + thisRuleMatches = (uint8_t)((uint32_t)(RR->node->prng() & 0xffffffffULL) <= rules[rn].v.randomProbability); + FILTER_TRACE("%u %s %c -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)thisRuleMatches); + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: -- cgit v1.2.3 From adeb7e7da0e5d1e267c272a4f1d1c9b731e291d9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 5 Oct 2016 12:54:46 -0700 Subject: Make capability flags match more user-friendly and appropriate since "match any flag" is generally what we want. --- controller/EmbeddedNetworkController.cpp | 19 ++++--------------- include/ZeroTierOne.h | 2 +- node/Capability.hpp | 8 +++----- node/Network.cpp | 2 +- 4 files changed, 9 insertions(+), 22 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d656bad3..7fa66224 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -249,10 +249,8 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); r["mask"] = tmp; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); - r["value"] = tmp; break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: r["type"] = "MATCH_FRAME_SIZE_RANGE"; @@ -423,21 +421,12 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; if (r.count("mask")) { - auto v = r["mask"]; + json &v = r["mask"]; if (v.is_number()) { - rule.v.characteristics[0] = v; + rule.v.characteristics = v; } else { std::string tmp = v; - rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); - } - } - if (r.count("value")) { - auto v = r["value"]; - if (v.is_number()) { - rule.v.characteristics[1] = v; - } else { - std::string tmp = v; - rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); + rule.v.characteristics = Utils::hexStrToU64(tmp.c_str()); } } return true; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index ee03c3b1..e231ae62 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -713,7 +713,7 @@ typedef struct /** * Packet characteristic flags being matched */ - uint64_t characteristics[2]; + uint64_t characteristics; /** * IP port range -- start-end inclusive -- host byte order diff --git a/node/Capability.hpp b/node/Capability.hpp index e808ad40..f757639d 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -240,9 +240,8 @@ public: b.append((uint16_t)rules[i].v.port[1]); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - b.append((uint8_t)16); - b.append((uint64_t)rules[i].v.characteristics[0]); - b.append((uint64_t)rules[i].v.characteristics[1]); + b.append((uint8_t)8); + b.append((uint64_t)rules[i].v.characteristics); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: b.append((uint8_t)4); @@ -328,8 +327,7 @@ public: rules[ruleCount].v.port[1] = b.template at(p + 2); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - rules[ruleCount].v.characteristics[0] = b.template at(p); - rules[ruleCount].v.characteristics[1] = b.template at(p + 8); + rules[ruleCount].v.characteristics = b.template at(p); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: rules[ruleCount].v.frameSize[0] = b.template at(p); diff --git a/node/Network.cpp b/node/Network.cpp index fe899dcc..8b9f6e3d 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -497,7 +497,7 @@ static _doZtFilterResult _doZtFilter( } } } - thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); + thisRuleMatches = (uint8_t)((cf | rules[rn].v.characteristics) != 0); FILTER_TRACE("%u %s %c (%.16llx & %.16llx)==%.16llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],(unsigned int)thisRuleMatches); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: -- cgit v1.2.3 From 45c4ccb15362e17ec7030287a314df19a830f0f3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 5 Oct 2016 16:38:42 -0700 Subject: Add a tags both equal match. --- controller/EmbeddedNetworkController.cpp | 11 +++++++++++ include/ZeroTierOne.h | 5 +++++ node/Capability.hpp | 1 + node/Network.cpp | 6 +++++- 4 files changed, 22 insertions(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7fa66224..aa95ac64 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -287,6 +287,12 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; break; + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + r["type"] = "MATCH_TAGS_EQUAL"; + r["not"] = ((rule.t & 0x80) != 0); + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; } return r; } @@ -458,6 +464,11 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; + } else if (t == "MATCH_TAGS_EQUAL") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_EQUAL; + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); + return true; } return false; } diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e231ae62..8d7b0cd4 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -658,6 +658,11 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 55, + /** + * Match if local and remote tags both equal a value + */ + ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 56, + /** * Maximum ID allowed for a MATCH entry in the rules table */ diff --git a/node/Capability.hpp b/node/Capability.hpp index f757639d..99980ce7 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -340,6 +340,7 @@ public: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: rules[ruleCount].v.tag.id = b.template at(p); rules[ruleCount].v.tag.value = b.template at(p + 4); break; diff --git a/node/Network.cpp b/node/Network.cpp index 8b9f6e3d..00c201ba 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -511,7 +511,8 @@ static _doZtFilterResult _doZtFilter( case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: { const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0); @@ -531,6 +532,9 @@ static _doZtFilterResult _doZtFilter( } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { thisRuleMatches = (uint8_t)((ltv ^ rtv) == rules[rn].v.tag.value); FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_EQUAL) { + thisRuleMatches = (uint8_t)((ltv == rules[rn].v.tag.value)&&(rtv == rules[rn].v.tag.value)); + FILTER_TRACE("%u %s %c TAG %u local:%.8x and remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { // sanity check, can't really happen thisRuleMatches = 0; } -- cgit v1.2.3 From e53f63ca8700a526b15c2e7d05076d685734bcf6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 11 Oct 2016 12:00:16 -0700 Subject: Broke down and added an OR to the rules engine. It is now possible to have a series of MATCHes that are ORed. --- controller/EmbeddedNetworkController.cpp | 282 +++++++++++++++---------------- include/ZeroTierOne.h | 163 ++++-------------- node/Capability.hpp | 18 +- node/Network.cpp | 22 +-- 4 files changed, 193 insertions(+), 292 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index aa95ac64..7559cf83 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -127,7 +127,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; json r = json::object(); - switch((rule.t) & 0x7f) { + const ZT_VirtualNetworkRuleType rt = (ZT_VirtualNetworkRuleType)(rule.t & 0x3f); + + switch(rt) { case ZT_NETWORK_RULE_ACTION_DROP: r["type"] = "ACTION_DROP"; break; @@ -154,146 +156,136 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: r["type"] = "ACTION_DEBUG_LOG"; break; - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; - r["not"] = ((rule.t & 0x80) != 0); - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; - r["not"] = ((rule.t & 0x80) != 0); - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: - r["type"] = "MATCH_VLAN_ID"; - r["not"] = ((rule.t & 0x80) != 0); - r["vlanId"] = (unsigned int)rule.v.vlanId; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - r["type"] = "MATCH_VLAN_PCP"; - r["not"] = ((rule.t & 0x80) != 0); - r["vlanPcp"] = (unsigned int)rule.v.vlanPcp; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - r["type"] = "MATCH_VLAN_DEI"; - r["not"] = ((rule.t & 0x80) != 0); - r["vlanDei"] = (unsigned int)rule.v.vlanDei; - break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - r["type"] = "MATCH_ETHERTYPE"; - r["not"] = ((rule.t & 0x80) != 0); - r["etherType"] = (unsigned int)rule.v.etherType; - break; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - r["type"] = "MATCH_MAC_SOURCE"; - r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_MAC_DEST: - r["type"] = "MATCH_MAC_DEST"; - r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - r["type"] = "MATCH_IPV4_SOURCE"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - r["type"] = "MATCH_IPV4_DEST"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - r["type"] = "MATCH_IPV6_SOURCE"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - r["type"] = "MATCH_IPV6_DEST"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + default: break; - case ZT_NETWORK_RULE_MATCH_IP_TOS: - r["type"] = "MATCH_IP_TOS"; - r["not"] = ((rule.t & 0x80) != 0); - r["ipTos"] = (unsigned int)rule.v.ipTos; - break; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - r["type"] = "MATCH_IP_PROTOCOL"; - r["not"] = ((rule.t & 0x80) != 0); - r["ipProtocol"] = (unsigned int)rule.v.ipProtocol; - break; - case ZT_NETWORK_RULE_MATCH_ICMP: - r["type"] = "MATCH_ICMP"; - r["not"] = ((rule.t & 0x80) != 0); - r["type"] = (unsigned int)rule.v.icmp.type; - if ((rule.v.icmp.flags & 0x01) != 0) - r["code"] = (unsigned int)rule.v.icmp.code; - else r["code"] = json(); - break; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: - r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; - r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (unsigned int)rule.v.port[0]; - r["end"] = (unsigned int)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - r["type"] = "MATCH_IP_DEST_PORT_RANGE"; - r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (unsigned int)rule.v.port[0]; - r["end"] = (unsigned int)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - r["type"] = "MATCH_CHARACTERISTICS"; - r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); - r["mask"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - r["type"] = "MATCH_FRAME_SIZE_RANGE"; - r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (unsigned int)rule.v.frameSize[0]; - r["end"] = (unsigned int)rule.v.frameSize[1]; - break; - case ZT_NETWORK_RULE_MATCH_RANDOM: - r["type"] = "MATCH_RANDOM"; - r["not"] = ((rule.t & 0x80) != 0); - r["probability"] = (unsigned long)rule.v.randomProbability; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: - r["type"] = "MATCH_TAGS_DIFFERENCE"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: - r["type"] = "MATCH_TAGS_BITWISE_AND"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: - r["type"] = "MATCH_TAGS_BITWISE_OR"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: - r["type"] = "MATCH_TAGS_BITWISE_XOR"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: - r["type"] = "MATCH_TAGS_EQUAL"; + } + + if (r.size() == 0) { + switch(rt) { + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + r["type"] = "MATCH_VLAN_ID"; + r["vlanId"] = (unsigned int)rule.v.vlanId; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + r["type"] = "MATCH_VLAN_PCP"; + r["vlanPcp"] = (unsigned int)rule.v.vlanPcp; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + r["type"] = "MATCH_VLAN_DEI"; + r["vlanDei"] = (unsigned int)rule.v.vlanDei; + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + r["type"] = "MATCH_MAC_SOURCE"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + r["type"] = "MATCH_MAC_DEST"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + r["type"] = "MATCH_IPV4_SOURCE"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + r["type"] = "MATCH_IPV4_DEST"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + r["type"] = "MATCH_IPV6_SOURCE"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + r["type"] = "MATCH_IPV6_DEST"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + r["type"] = "MATCH_IP_TOS"; + r["ipTos"] = (unsigned int)rule.v.ipTos; + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + r["type"] = "MATCH_IP_PROTOCOL"; + r["ipProtocol"] = (unsigned int)rule.v.ipProtocol; + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + r["type"] = "MATCH_ETHERTYPE"; + r["etherType"] = (unsigned int)rule.v.etherType; + break; + case ZT_NETWORK_RULE_MATCH_ICMP: + r["type"] = "MATCH_ICMP"; + r["type"] = (unsigned int)rule.v.icmp.type; + if ((rule.v.icmp.flags & 0x01) != 0) + r["code"] = (unsigned int)rule.v.icmp.code; + else r["code"] = json(); + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["start"] = (unsigned int)rule.v.port[0]; + r["end"] = (unsigned int)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["start"] = (unsigned int)rule.v.port[0]; + r["end"] = (unsigned int)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + r["type"] = "MATCH_CHARACTERISTICS"; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); + r["mask"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["start"] = (unsigned int)rule.v.frameSize[0]; + r["end"] = (unsigned int)rule.v.frameSize[1]; + break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + r["type"] = "MATCH_RANDOM"; + r["probability"] = (unsigned long)rule.v.randomProbability; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: + r["type"] = "MATCH_TAGS_DIFFERENCE"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + r["type"] = "MATCH_TAGS_EQUAL"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + default: + break; + } + + if (r.size() > 0) { r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; + r["or"] = ((rule.t & 0x40) != 0); + } } + return r; } @@ -301,11 +293,16 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) { if (!r.is_object()) return false; + const std::string t(_jS(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); + if (_jB(r["not"],false)) rule.t = 0x80; else rule.t = 0x00; + if (_jB(r["or"],false)) + rule.t |= 0x40; + if (t == "ACTION_DROP") { rule.t |= ZT_NETWORK_RULE_ACTION_DROP; return true; @@ -352,10 +349,6 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; rule.v.vlanDei = (uint8_t)(_jI(r["vlanDei"],0ULL) & 0xffULL); return true; - } else if (t == "MATCH_ETHERTYPE") { - rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; - rule.v.etherType = (uint16_t)(_jI(r["etherType"],0ULL) & 0xffffULL); - return true; } else if (t == "MATCH_MAC_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; const std::string mac(_jS(r["mac"],"0")); @@ -402,6 +395,10 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; rule.v.ipProtocol = (uint8_t)(_jI(r["ipProtocol"],0ULL) & 0xffULL); return true; + } else if (t == "MATCH_ETHERTYPE") { + rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; + rule.v.etherType = (uint16_t)(_jI(r["etherType"],0ULL) & 0xffffULL); + return true; } else if (t == "MATCH_ICMP") { rule.t |= ZT_NETWORK_RULE_MATCH_ICMP; rule.v.icmp.type = (uint8_t)(_jI(r["type"],0ULL) & 0xffULL); @@ -470,6 +467,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } + return false; } diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 8d7b0cd4..17112e90 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -491,15 +491,15 @@ enum ZT_VirtualNetworkType /** * The type of a virtual network rules table entry * - * These must range from 0 to 127 (0x7f) because the most significant bit - * is reserved as a NOT flag. + * These must be from 0 to 63 since the most significant two bits of each + * rule type are NOT (MSB) and AND/OR. * * Each rule is composed of zero or more MATCHes followed by an ACTION. * An ACTION with no MATCHes is always taken. */ enum ZT_VirtualNetworkRuleType { - // 0 to 31 reserved for actions + // 0 to 15 reserved for actions /** * Drop frame @@ -534,139 +534,40 @@ enum ZT_VirtualNetworkRuleType /** * Maximum ID for an ACTION, anything higher is a MATCH */ - ZT_NETWORK_RULE_ACTION__MAX_ID = 31, - - // 32 to 127 reserved for match criteria - - /** - * Source ZeroTier address -- analogous to an Ethernet port ID on a switch - */ - ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS = 32, - - /** - * Destination ZeroTier address -- analogous to an Ethernet port ID on a switch - */ - ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS = 33, - - /** - * Ethernet VLAN ID - */ - ZT_NETWORK_RULE_MATCH_VLAN_ID = 34, - - /** - * Ethernet VLAN PCP - */ - ZT_NETWORK_RULE_MATCH_VLAN_PCP = 35, - - /** - * Ethernet VLAN DEI - */ - ZT_NETWORK_RULE_MATCH_VLAN_DEI = 36, - - /** - * Ethernet frame type - */ + ZT_NETWORK_RULE_ACTION__MAX_ID = 15, + + // 16 to 63 reserved for match criteria + + ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS = 24, + ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS = 25, + ZT_NETWORK_RULE_MATCH_VLAN_ID = 26, + ZT_NETWORK_RULE_MATCH_VLAN_PCP = 27, + ZT_NETWORK_RULE_MATCH_VLAN_DEI = 28, + ZT_NETWORK_RULE_MATCH_MAC_SOURCE = 29, + ZT_NETWORK_RULE_MATCH_MAC_DEST = 30, + ZT_NETWORK_RULE_MATCH_IPV4_SOURCE = 31, + ZT_NETWORK_RULE_MATCH_IPV4_DEST = 32, + ZT_NETWORK_RULE_MATCH_IPV6_SOURCE = 33, + ZT_NETWORK_RULE_MATCH_IPV6_DEST = 34, + ZT_NETWORK_RULE_MATCH_IP_TOS = 35, + ZT_NETWORK_RULE_MATCH_IP_PROTOCOL = 36, ZT_NETWORK_RULE_MATCH_ETHERTYPE = 37, - - /** - * Source Ethernet MAC address - */ - ZT_NETWORK_RULE_MATCH_MAC_SOURCE = 38, - - /** - * Destination Ethernet MAC address - */ - ZT_NETWORK_RULE_MATCH_MAC_DEST = 39, - - /** - * Source IPv4 address - */ - ZT_NETWORK_RULE_MATCH_IPV4_SOURCE = 40, - - /** - * Destination IPv4 address - */ - ZT_NETWORK_RULE_MATCH_IPV4_DEST = 41, - - /** - * Source IPv6 address - */ - ZT_NETWORK_RULE_MATCH_IPV6_SOURCE = 42, - - /** - * Destination IPv6 address - */ - ZT_NETWORK_RULE_MATCH_IPV6_DEST = 43, - - /** - * IP TOS (type of service) - */ - ZT_NETWORK_RULE_MATCH_IP_TOS = 44, - - /** - * IP protocol - */ - ZT_NETWORK_RULE_MATCH_IP_PROTOCOL = 45, - - /** - * ICMP type and possibly code (does not match if not ICMP) - */ - ZT_NETWORK_RULE_MATCH_ICMP = 46, - - /** - * IP source port range (start-end, inclusive) - */ - ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 47, - - /** - * IP destination port range (start-end, inclusive) - */ - ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 48, - - /** - * Packet characteristics (set of flags) - */ - ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 49, - - /** - * Frame size range (start-end, inclusive) - */ - ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 50, - - /** - * Random match with selectable probability - */ - ZT_NETWORK_RULE_MATCH_RANDOM = 51, - - /** - * Match if local and remote tags differ by no more than value, use 0 to check for equality - */ - ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 52, - - /** - * Match if local and remote tags ANDed together equal value. - */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 53, - - /** - * Match if local and remote tags ANDed together equal value. - */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 54, - - /** - * Match if local and remote tags XORed together equal value. - */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 55, - - /** - * Match if local and remote tags both equal a value - */ - ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 56, + ZT_NETWORK_RULE_MATCH_ICMP = 38, + ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 39, + ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 40, + ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 41, + ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 42, + ZT_NETWORK_RULE_MATCH_RANDOM = 43, + ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 44, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 45, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 46, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 47, + ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 48, /** * Maximum ID allowed for a MATCH entry in the rules table */ - ZT_NETWORK_RULE_MATCH__MAX_ID = 127 + ZT_NETWORK_RULE_MATCH__MAX_ID = 63 }; /** diff --git a/node/Capability.hpp b/node/Capability.hpp index 99980ce7..2c829ee5 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -166,7 +166,7 @@ public: // field followed by field data. The inclusion of the size will allow non-supported // rules to be ignored but still parsed. b.append((uint8_t)rules[i].t); - switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x7f)) { + switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x3f)) { //case ZT_NETWORK_RULE_ACTION_DROP: //case ZT_NETWORK_RULE_ACTION_ACCEPT: //case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: @@ -198,10 +198,6 @@ public: b.append((uint8_t)1); b.append((uint8_t)rules[i].v.vlanDei); break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - b.append((uint8_t)2); - b.append((uint16_t)rules[i].v.etherType); - break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: case ZT_NETWORK_RULE_MATCH_MAC_DEST: b.append((uint8_t)6); @@ -227,6 +223,10 @@ public: b.append((uint8_t)1); b.append((uint8_t)rules[i].v.ipProtocol); break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + b.append((uint8_t)2); + b.append((uint16_t)rules[i].v.etherType); + break; case ZT_NETWORK_RULE_MATCH_ICMP: b.append((uint8_t)3); b.append((uint8_t)rules[i].v.icmp.type); @@ -270,7 +270,7 @@ public: while ((ruleCount < maxRuleCount)&&(p < b.size())) { rules[ruleCount].t = (uint8_t)b[p++]; const unsigned int fieldLen = (unsigned int)b[p++]; - switch((ZT_VirtualNetworkRuleType)(rules[ruleCount].t & 0x7f)) { + switch((ZT_VirtualNetworkRuleType)(rules[ruleCount].t & 0x3f)) { default: break; case ZT_NETWORK_RULE_ACTION_TEE: @@ -293,9 +293,6 @@ public: case ZT_NETWORK_RULE_MATCH_VLAN_DEI: rules[ruleCount].v.vlanDei = (uint8_t)b[p]; break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - rules[ruleCount].v.etherType = b.template at(p); - break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: case ZT_NETWORK_RULE_MATCH_MAC_DEST: memcpy(rules[ruleCount].v.mac,b.field(p,6),6); @@ -316,6 +313,9 @@ public: case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: rules[ruleCount].v.ipProtocol = (uint8_t)b[p]; break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + rules[ruleCount].v.etherType = b.template at(p); + break; case ZT_NETWORK_RULE_MATCH_ICMP: rules[ruleCount].v.icmp.type = (uint8_t)b[p]; rules[ruleCount].v.icmp.code = (uint8_t)b[p+1]; diff --git a/node/Network.cpp b/node/Network.cpp index 00c201ba..177f1a6d 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -58,7 +58,6 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_VLAN_ID: return "MATCH_VLAN_ID"; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: return "MATCH_VLAN_PCP"; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: return "MATCH_VLAN_DEI"; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: return "MATCH_ETHERTYPE"; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: return "MATCH_MAC_SOURCE"; case ZT_NETWORK_RULE_MATCH_MAC_DEST: return "MATCH_MAC_DEST"; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: return "MATCH_IPV4_SOURCE"; @@ -67,6 +66,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_IPV6_DEST: return "MATCH_IPV6_DEST"; case ZT_NETWORK_RULE_MATCH_IP_TOS: return "MATCH_IP_TOS"; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: return "MATCH_IP_PROTOCOL"; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: return "MATCH_ETHERTYPE"; case ZT_NETWORK_RULE_MATCH_ICMP: return "MATCH_ICMP"; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: return "MATCH_IP_SOURCE_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; @@ -182,7 +182,7 @@ static _doZtFilterResult _doZtFilter( uint8_t thisSetMatches = 1; for(unsigned int rn=0;rn %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanDei,0,(unsigned int)thisRuleMatches); break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); - FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.etherType,etherType,(unsigned int)thisRuleMatches); - break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macSource.toInt(),(unsigned int)thisRuleMatches); @@ -380,6 +377,10 @@ static _doZtFilterResult _doZtFilter( FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.etherType,etherType,(unsigned int)thisRuleMatches); + break; case ZT_NETWORK_RULE_MATCH_ICMP: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { if (frameData[9] == 0x01) { @@ -560,8 +561,9 @@ static _doZtFilterResult _doZtFilter( break; } - // State of equals state AND result of last MATCH (possibly NOTed depending on bit 0x80) - thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); + if ((rules[rn].t & 0x40)) + thisSetMatches |= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); + else thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); } return DOZTFILTER_NO_MATCH; -- cgit v1.2.3 From e2509af163208f299735eefdb145289a3e90956e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 12 Oct 2016 12:30:32 -0700 Subject: Fix bug in default rules init in new networks. --- controller/EmbeddedNetworkController.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index bce8890c..1e0fa576 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -168,10 +168,10 @@ private: if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment - network["rules"] = { + network["rules"] = {{ { "not",false }, { "type","ACTION_ACCEPT" } - }; + }}; } network["objtype"] = "network"; } -- cgit v1.2.3 From 2d6a4e5974430ea01cf424aab8b372ebc872b25f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 13:52:45 -0700 Subject: cleanup --- controller/EmbeddedNetworkController.cpp | 22 ---------------------- controller/EmbeddedNetworkController.hpp | 12 ------------ node/Packet.cpp | 2 +- 3 files changed, 1 insertion(+), 35 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7559cf83..2c43d4b5 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -517,19 +517,6 @@ void EmbeddedNetworkController::threadMain() lastUpdatedNetworkMemberCache = OSUtils::now(); } - { // Every 25ms we push up to 50 network refreshes, which amounts to a max of about 300-500kb/sec - unsigned int count = 0; - Mutex::Lock _l(_refreshQueue_m); - while (_refreshQueue.size() > 0) { - _Refresh &r = _refreshQueue.front(); - //if (_node) - // _node->pushNetworkRefresh(r.dest,r.nwid,r.blacklistAddresses,r.blacklistThresholds,r.numBlacklistEntries); - _refreshQueue.pop_front(); - if (++count >= 50) - break; - } - } - Thread::sleep(25); } } @@ -1230,15 +1217,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _networkMemberCache[nwid][Address(address)] = member; } - { - Mutex::Lock _l(_refreshQueue_m); - _refreshQueue.push_back(_Refresh()); - _Refresh &r = _refreshQueue.back(); - r.dest = Address(address); - r.nwid = nwid; - r.numBlacklistEntries = 0; - } - // Add non-persisted fields member["clock"] = now; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 1e0fa576..99e1836d 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -204,18 +204,6 @@ private: std::map< std::pair,uint64_t > _lastRequestTime; Mutex _lastRequestTime_m; - // Queue of network member refreshes to be pushed - struct _Refresh - { - Address dest; - uint64_t nwid; - uint64_t blacklistAddresses[64]; - uint64_t blacklistThresholds[64]; - unsigned int numBlacklistEntries; - }; - std::list< _Refresh > _refreshQueue; - Mutex _refreshQueue_m; - Thread _daemon; volatile bool _daemonRun; }; diff --git a/node/Packet.cpp b/node/Packet.cpp index 20b80962..bdbc5fe4 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -40,7 +40,7 @@ const char *Packet::verbString(Verb v) case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; case VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS"; case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; - case VERB_NETWORK_CONFIG: return "NETWORK_CONFIG_REFRESH"; + case VERB_NETWORK_CONFIG: return "NETWORK_CONFIG"; case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; -- cgit v1.2.3 From 2cb760e0ace2693684e79f782f17b77bf47377bb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 14:14:46 -0700 Subject: Fix ICMP json. --- controller/EmbeddedNetworkController.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 2c43d4b5..d9ec76de 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -222,10 +222,10 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_ICMP: r["type"] = "MATCH_ICMP"; - r["type"] = (unsigned int)rule.v.icmp.type; + r["icmpType"] = (unsigned int)rule.v.icmp.type; if ((rule.v.icmp.flags & 0x01) != 0) - r["code"] = (unsigned int)rule.v.icmp.code; - else r["code"] = json(); + r["icmpCode"] = (unsigned int)rule.v.icmp.code; + else r["icmpCode"] = json(); break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; @@ -401,8 +401,8 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_ICMP") { rule.t |= ZT_NETWORK_RULE_MATCH_ICMP; - rule.v.icmp.type = (uint8_t)(_jI(r["type"],0ULL) & 0xffULL); - json &code = r["code"]; + rule.v.icmp.type = (uint8_t)(_jI(r["icmpType"],0ULL) & 0xffULL); + json &code = r["icmpCode"]; if (code.is_null()) { rule.v.icmp.code = 0; rule.v.icmp.flags = 0x00; -- cgit v1.2.3 From 8ffae313fd5042bb4330afd98019e4ee907714cb Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 3 Nov 2016 12:10:50 -0700 Subject: add new files & remove old ones from VS project. Now builds & runs on Windows again --- .gitignore | 2 +- controller/EmbeddedNetworkController.cpp | 3 +++ node/Membership.cpp | 28 ++++++++++++++++--------- node/Multicaster.cpp | 2 +- one.cpp | 6 +----- windows/ZeroTierOne.sln | 12 +++++++++-- windows/ZeroTierOne/ZeroTierOne.vcxproj | 11 ++++++++-- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 27 ++++++++++++++++-------- 8 files changed, 61 insertions(+), 30 deletions(-) (limited to 'controller') diff --git a/.gitignore b/.gitignore index b01abc35..d3ef1581 100755 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,7 @@ Thumbs.db /windows/TapDriver6/win7Release /windows/*.db /windows/*.opendb - +enc_temp_folder # *nix/Mac build droppings /build-* diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d9ec76de..8f5cefd1 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -21,7 +21,10 @@ #include #include #include + +#ifndef _WIN32 #include +#endif #include #include diff --git a/node/Membership.cpp b/node/Membership.cpp index c8fb8e4e..d7c7c0e6 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -232,7 +232,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme Membership::_RemoteTag *Membership::_newTag(const uint64_t id) { - _RemoteTag *t; + _RemoteTag *t = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -243,17 +243,21 @@ Membership::_RemoteTag *Membership::_newTag(const uint64_t id) minlr = _remoteTags[i]->lastReceived; } } - t->id = id; - t->lastReceived = 0; - t->revocationThreshold = 0; - t->tag = Tag(); + + if (t) { + t->id = id; + t->lastReceived = 0; + t->revocationThreshold = 0; + t->tag = Tag(); + } + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); return t; } Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) { - _RemoteCapability *c; + _RemoteCapability *c = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -264,10 +268,14 @@ Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) minlr = _remoteCaps[i]->lastReceived; } } - c->id = id; - c->lastReceived = 0; - c->revocationThreshold = 0; - c->cap = Capability(); + + if (c) { + c->id = id; + c->lastReceived = 0; + c->revocationThreshold = 0; + c->cap = Capability(); + } + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); return c; } diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 8743e8f8..17649c7b 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -343,7 +343,7 @@ void Multicaster::clean(uint64_t now) { Mutex::Lock _l(_gatherAuth_m); _GatherAuthKey *k = (_GatherAuthKey *)0; - uint64_t *ts = (uint64_t *)ts; + uint64_t *ts = NULL; Hashtable<_GatherAuthKey,uint64_t>::Iterator i(_gatherAuth); while (i.next(k,ts)) { if ((now - *ts) >= ZT_MULTICAST_CREDENTIAL_EXPIRATON) diff --git a/one.cpp b/one.cpp index 79e8caf8..55cc2e19 100644 --- a/one.cpp +++ b/one.cpp @@ -402,7 +402,7 @@ static int cli(int argc,char **argv) auto &addr = assignedAddresses[j]; if (addr.is_string()) { if (aa.length() > 0) aa.push_back(','); - aa.append(addr); + aa.append(addr.get()); } } } @@ -1194,9 +1194,5 @@ int main(int argc,char **argv) delete zt1Service; zt1Service = (OneService *)0; -#ifdef __UNIX_LIKE__ - OSUtils::rm(pidPath.c_str()); -#endif - return returnValue; } diff --git a/windows/ZeroTierOne.sln b/windows/ZeroTierOne.sln index 68596187..e9a1dfd2 100644 --- a/windows/ZeroTierOne.sln +++ b/windows/ZeroTierOne.sln @@ -1,6 +1,8 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2012 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZeroTierOne", "ZeroTierOne\ZeroTierOne.vcxproj", "{B00A4957-5977-4AC1-9EF4-571DC27EADA2}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TapDriver6", "TapDriver6\TapDriver6.vcxproj", "{43BA7584-D4DB-4F7C-90FC-E2B18A68A213}" @@ -157,7 +159,8 @@ Global {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|Win32.ActiveCfg = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|Win32.Build.0 = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|Win32.Deploy.0 = Debug|Win32 - {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x64.ActiveCfg = Debug|Win32 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x64.ActiveCfg = Debug|x64 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x64.Build.0 = Debug|x64 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x86.ActiveCfg = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x86.Build.0 = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x86.Deploy.0 = Debug|Win32 @@ -169,6 +172,7 @@ Global {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|Win32.Build.0 = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|Win32.Deploy.0 = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x64.ActiveCfg = Release|x64 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x64.Build.0 = Release|x64 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x86.ActiveCfg = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x86.Build.0 = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x86.Deploy.0 = Release|Win32 @@ -392,14 +396,18 @@ Global {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|Mixed Platforms.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|Win32.ActiveCfg = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x64.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x64.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x86.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x86.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Any CPU.ActiveCfg = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Any CPU.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Mixed Platforms.ActiveCfg = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Mixed Platforms.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Win32.ActiveCfg = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x64.ActiveCfg = Release|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x64.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x86.ActiveCfg = Release|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x86.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win8 Debug|Any CPU.ActiveCfg = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win8 Debug|Any CPU.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win8 Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 93da4b1b..b4bf308c 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -19,8 +19,8 @@ + - @@ -39,12 +39,13 @@ + - + @@ -54,10 +55,12 @@ + + @@ -251,6 +254,7 @@ NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + 4996 true @@ -267,6 +271,7 @@ NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) false + 4996 true @@ -290,6 +295,7 @@ AnySuitable Speed true + 4996 true @@ -315,6 +321,7 @@ AnySuitable Speed true + 4996 true diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index eeeb8d2f..337b8cbb 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -55,9 +55,6 @@ {f8a1c208-15b8-4d85-a4cb-11d2b82f2d1e} - - {da28e961-1761-41d8-9a59-65b00dfb1302} - {43f75f84-c70d-4d44-a0ef-28a7a399abd4} @@ -91,6 +88,9 @@ {409ec37e-ff36-4c13-b18d-52d6052e0ca2} + + {3cad34c8-c436-43ae-8323-57803637c832} + @@ -171,9 +171,6 @@ Source Files\ext\http-parser - - Source Files\ext\json-parser - Source Files @@ -192,9 +189,6 @@ Source Files\node - - Source Files\node - Source Files\node @@ -252,6 +246,21 @@ Source Files\osdep + + Source Files\node + + + Source Files\node + + + Source Files\node + + + Source Files\node + + + Source Files\controller + -- cgit v1.2.3 From 3c00cd0f88026ef1964ba6d36019b716d0bb6df6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 3 Nov 2016 14:17:46 -0700 Subject: Separate out JSON store from controller code. --- controller/JSONDB.cpp | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++ controller/JSONDB.hpp | 88 ++++++++++++++++++++++++++++++ objects.mk | 1 + 3 files changed, 235 insertions(+) create mode 100644 controller/JSONDB.cpp create mode 100644 controller/JSONDB.hpp (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp new file mode 100644 index 00000000..40bc21ac --- /dev/null +++ b/controller/JSONDB.cpp @@ -0,0 +1,146 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "JSONDB.hpp" + +namespace ZeroTier { + +static const nlohmann::json _EMPTY_JSON({{}}); + +bool JSONDB::put(const std::string &n,const nlohmann::json &obj) +{ + if (!_isValidObjectName(n)) + return false; + + std::string path(_genPath(n,false)); + if (!path.length()) + return false; + + std::string buf(obj.dump(2)); + if (!OSUtils::writeFile(path.c_str(),buf)) + return false; + + _E &e = _db[n]; + + e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); + e.lastCheck = OSUtils::now(); + e.obj = obj; + + return true; +} + +const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceCheck) +{ + if (!_isValidObjectName(n)) + return _EMPTY_JSON; + + const uint64_t now = OSUtils::now(); + std::string buf; + std::map::iterator e(_db.find(n)); + + if (e != _db.end()) { + if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck) + return e->second.obj; + + std::string path(_genPath(n,false)); + if (!path.length()) // sanity check + return _EMPTY_JSON; + + // We are somewhat tolerant to momentary disk failures here. This may + // occur over e.g. EC2's elastic filesystem (NFS). + const uint64_t lm = OSUtils::getLastModified(path.c_str()); + if ((lm)&&(e->second.lastModifiedOnDisk != lm)) { + if (OSUtils::readFile(path.c_str(),buf)) { + try { + e->second.lastModifiedOnDisk = lm; + e->second.lastCheck = now; + e->second.obj = nlohmann::json::parse(buf); + } catch ( ... ) { + e->second.obj = _EMPTY_JSON; + } + } + } + + return e->second.obj; + } else { + std::string path(_genPath(n,false)); + if (!path.length()) + return _EMPTY_JSON; + + if (!OSUtils::readFile(path.c_str(),buf)) + return _EMPTY_JSON; + + const uint64_t lm = OSUtils::getLastModified(path.c_str()); + if (!lm) + return _EMPTY_JSON; + + _E &e2 = _db[n]; + e2.lastModifiedOnDisk = lm; + e2.lastCheck = now; + try { + e2.obj = nlohmann::json::parse(buf); + } catch ( ... ) { + e2.obj = _EMPTY_JSON; + } + + return e2.obj; + } +} + +bool JSONDB::_isValidObjectName(const std::string &n) +{ + if (n.length() == 0) + return false; + const char *p = n.c_str(); + char c; + // For security reasons we should not allow dots, backslashes, or other path characters or potential path characters. + while ((c = *(p++))) { + if (!( ((c >= 'a')&&(c <= 'z')) || ((c >= 'A')&&(c <= 'Z')) || ((c >= '0')&&(c <= '9')) || (c == '/') || (c == '_') || (c == '~') || (c == '-') )) + return false; + } + return true; +} + +std::string JSONDB::_genPath(const std::string &n,bool create) +{ + std::vector pt(Utils::split(n.c_str(),"/","","")); + if (pt.size() == 0) + return std::string(); + if (pt.size() == 1) + return pt[0]; + + std::string p(_basePath); + if (create) OSUtils::mkdir(p.c_str()); + for(unsigned long i=0,j=pt.size()-1;i. + */ + +#ifndef ZT_JSONDB_HPP +#define ZT_JSONDB_HPP + +#include +#include +#include +#include +#include + +#include "../node/Constants.hpp" +#include "../node/Utils.hpp" +#include "../ext/json/json.hpp" +#include "../osdep/OSUtils.hpp" + +namespace ZeroTier { + +/** + * Hierarchical JSON store that persists into the filesystem + */ +class JSONDB +{ +public: + JSONDB(const std::string &basePath) : + _basePath(basePath) + { + this->_reloadAll(_basePath); + } + + bool put(const std::string &n,const nlohmann::json &obj); + + inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } + inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); } + + const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0); + + inline const nlohmann::json &get(const std::string &n1,const std::string &n2,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2),maxSinceCheck); } + inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3),maxSinceCheck); } + inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); } + inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); } + + template + inline void each(F func,unsigned long maxSinceCheck = 0) + { + const uint64_t now = OSUtils::now(); + for(std::map::const_iterator i(_db.begin());i!=_db.end();++i) { + if ((now - i->second.lastCheck) > (uint64_t)maxSinceCheck) + this->get(i->first); + func(i->first,i->second.obj); + } + } + +private: + bool _isValidObjectName(const std::string &n); + std::string _genPath(const std::string &n,bool create); + void _reloadAll(const std::string &path); + + struct _E + { + uint64_t lastModifiedOnDisk; + uint64_t lastCheck; + nlohmann::json obj; + }; + + std::string _basePath; + std::map _db; +}; + +} // namespace ZeroTier + +#endif diff --git a/objects.mk b/objects.mk index 5738e769..078a92a7 100644 --- a/objects.mk +++ b/objects.mk @@ -1,5 +1,6 @@ OBJS=\ controller/EmbeddedNetworkController.o \ + controller/JSONDB.o \ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ -- cgit v1.2.3 From b03c7b2f30b18cc2d243bd226612d911f158bdc4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 4 Nov 2016 15:18:31 -0700 Subject: Refactor controller to use split-out DB for better performance and less ugly. --- controller/EmbeddedNetworkController.cpp | 265 ++++++++++++++++--------------- controller/EmbeddedNetworkController.hpp | 14 +- controller/JSONDB.cpp | 58 ++++--- controller/JSONDB.hpp | 36 +++-- osdep/OSUtils.cpp | 16 +- osdep/OSUtils.hpp | 6 +- 6 files changed, 223 insertions(+), 172 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8f5cefd1..21cd78e4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -64,6 +64,7 @@ using json = nlohmann::json; namespace ZeroTier { // JSON blob I/O +/* static json _readJson(const std::string &path) { std::string buf; @@ -78,6 +79,7 @@ static bool _writeJson(const std::string &path,const json &obj) { return OSUtils::writeFile(path.c_str(),obj.dump(2)); } +*/ // Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible static uint64_t _jI(const json &jv,const uint64_t dfl) @@ -475,55 +477,17 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : - _node(node), - _path(dbPath), - _daemonRun(true) + _db(dbPath), + _node(node) { OSUtils::mkdir(dbPath); OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions - _daemon = Thread::start(this); } EmbeddedNetworkController::~EmbeddedNetworkController() { } -void EmbeddedNetworkController::threadMain() - throw() -{ - uint64_t lastUpdatedNetworkMemberCache = 0; - while (_daemonRun) { - // Every 60 seconds we rescan the filesystem for network members and rebuild our cache - if ((OSUtils::now() - lastUpdatedNetworkMemberCache) >= 60000) { - const std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); - for(auto n=networks.begin();n!=networks.end();++n) { - if (n->length() == 16) { - const std::vector members(OSUtils::listSubdirectories((*n + ZT_PATH_SEPARATOR_S + "member").c_str())); - std::map newCache; - for(auto m=members.begin();m!=members.end();++m) { - if (m->length() == ZT_ADDRESS_LENGTH_HEX) { - const Address maddr(*m); - try { - const json mj(_readJson((_path + ZT_PATH_SEPARATOR_S + "network" + ZT_PATH_SEPARATOR_S + *n + ZT_PATH_SEPARATOR_S + "member" + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"))); - if ((mj.is_object())&&(mj.size() > 0)) { - newCache[maddr] = mj; - } - } catch ( ... ) {} - } - } - { - Mutex::Lock _l(_networkMemberCache_m); - _networkMemberCache[Utils::hexStrToU64(n->c_str())] = newCache; - } - } - } - lastUpdatedNetworkMemberCache = OSUtils::now(); - } - - Thread::sleep(25); - } -} - NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) { if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { @@ -541,12 +505,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( lrt = now; } - json network(_readJson(_networkJP(nwid,false))); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + json network; + json member; + { + Mutex::Lock _l(_db_m); + network = _db.get("network",nwids,0); + member = _db.get("network",nwids,"member",identity.address().toString(),0); + } if (!network.size()) return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; - - const std::string memberJP(_memberJP(nwid,identity.address(),true)); - json member(_readJson(memberJP)); _initMember(member); { @@ -673,7 +642,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // If they are not authorized, STOP! if (!authorizedBy) { - _writeJson(memberJP,member); + Mutex::Lock _l(_db_m); + _db.put("network",nwids,"member",identity.address().toString(),member); return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } @@ -966,7 +936,10 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( return NETCONF_QUERY_INTERNAL_SERVER_ERROR; } - _writeJson(memberJP,member); + { + Mutex::Lock _l(_db_m); + _db.put("network",nwids,"member",identity.address().toString(),member); + } return NetworkController::NETCONF_QUERY_OK; } @@ -985,7 +958,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - json network(_readJson(_networkJP(nwid,false))); + json network; + { + Mutex::Lock _l(_db_m); + network = _db.get("network",nwids,0); + } if (!network.size()) return 404; @@ -996,7 +973,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( if (path.size() >= 4) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - json member(_readJson(_memberJP(nwid,Address(address),false))); + json member; + { + Mutex::Lock _l(_db_m); + member = _db.get("network",nwids,"member",Address(address).toString(),0); + } if (!member.size()) return 404; @@ -1007,19 +988,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( return 200; } else { + Mutex::Lock _l(_db_m); + responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - responseBody.append(_jS(member["revision"],"0")); - } + std::string pfx(std::string("network/") + nwids + "member/"); + _db.filter(pfx,120000,[&responseBody](const std::string &n,const json &member) { + if (member.size() > 0) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(_jS(member["id"],"")); + responseBody.append("\":"); + responseBody.append(_jS(member["revision"],"0")); } - } + return true; // never delete + }); responseBody.push_back('}'); responseContentType = "application/json"; @@ -1056,7 +1037,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } } else if (path.size() == 1) { - responseBody = "["; + /* std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); for(auto i(networks.begin());i!=networks.end();++i) { if (i->length() == 16) { @@ -1065,6 +1046,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( responseBody.append("\""); } } + */ responseBody.push_back(']'); responseContentType = "application/json"; return 200; @@ -1122,7 +1104,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); if (path.size() >= 3) { - json network(_readJson(_networkJP(nwid,false))); + json network; + { + Mutex::Lock _l(_db_m); + network = _db.get("network",nwids,0); + } if (!network.size()) return 404; @@ -1131,7 +1117,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); - json member(_readJson(_memberJP(nwid,Address(address),true))); + json member; + { + Mutex::Lock _l(_db_m); + member = _db.get("network",nwids,"member",Address(address).toString(),0); + } + if (!member.size()) + return 404; _initMember(member); try { @@ -1154,7 +1146,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } if (b.count("ipAssignments")) { - auto ipa = b["ipAssignments"]; + json &ipa = b["ipAssignments"]; if (ipa.is_array()) { json mipa(json::array()); for(unsigned long i=0;i mtags; for(unsigned long i=0;i= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - json network(_readJson(_networkJP(nwid,false))); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + json network; + { + Mutex::Lock _l(_db_m); + network = _db.get("network",nwids,0); + } if (!network.size()) return 404; @@ -1526,22 +1532,23 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - json member(_readJson(_memberJP(nwid,Address(address),false))); + Mutex::Lock _l(_db_m); + + json member = _db.get("network",nwids,"member",Address(address).toString(),0); if (!member.size()) return 404; - - OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str()); + _db.erase("network",nwids,"member",Address(address).toString()); responseBody = member.dump(2); responseContentType = "application/json"; return 200; } } else { - OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); - { - Mutex::Lock _l(_networkMemberCache_m); - _networkMemberCache.erase(nwid); - } + Mutex::Lock _l(_db_m); + std::string pfx("network/"); pfx.append(nwids); + _db.filter(pfx,120000,[](const std::string &n,const json &obj) { + return false; // delete + }); responseBody = network.dump(2); responseContentType = "application/json"; return 200; @@ -1618,42 +1625,46 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) { - Mutex::Lock _mcl(_networkMemberCache_m); - std::map< Address,nlohmann::json > &memberCacheEntry = _networkMemberCache[nwid]; - nmi.totalMemberCount = memberCacheEntry.size(); - for(std::map< Address,nlohmann::json >::iterator nm(memberCacheEntry.begin());nm!=memberCacheEntry.end();++nm) { - if (_jB(nm->second["authorized"],false)) { - ++nmi.authorizedMemberCount; - - if (nm->second.count("recentLog")) { - json &mlog = nm->second["recentLog"]; - if ((mlog.is_array())&&(mlog.size() > 0)) { - json &mlog1 = mlog[0]; - if (mlog1.is_object()) { - if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) - ++nmi.activeMemberCount; + char pfx[256]; + Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid); + + Mutex::Lock _l(_db_m); + _db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) { + try { + if (_jB(member["authorized"],false)) { + ++nmi.authorizedMemberCount; + + if (member.count("recentLog")) { + const json &mlog = member["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + const json &mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) + ++nmi.activeMemberCount; + } } } - } - if (_jB(nm->second["activeBridge"],false)) { - nmi.activeBridges.insert(nm->first); - } + if (_jB(member["activeBridge"],false)) { + nmi.activeBridges.insert(_jS(member["id"],"0000000000")); + } - if (nm->second.count("ipAssignments")) { - json &mips = nm->second["ipAssignments"]; - if (mips.is_array()) { - for(unsigned long i=0;isecond["lastDeauthorizedTime"],0ULL)); - } - } + } catch ( ... ) {} + return true; + }); } } // namespace ZeroTier diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 99e1836d..f364c078 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -40,6 +40,8 @@ #include "../ext/json/json.hpp" +#include "JSONDB.hpp" + namespace ZeroTier { class Node; @@ -50,10 +52,6 @@ public: EmbeddedNetworkController(Node *node,const char *dbPath); virtual ~EmbeddedNetworkController(); - // Thread main method -- do not call directly - void threadMain() - throw(); - virtual NetworkController::ResultCode doNetworkConfigRequest( const InetAddress &fromAddr, const Identity &signingId, @@ -88,6 +86,7 @@ private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); // Network base path and network JSON path + /* inline std::string _networkBP(const uint64_t nwid,bool create) { char tmp[64]; @@ -124,6 +123,10 @@ private: // In-memory cache of network members std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache; Mutex _networkMemberCache_m; + */ + + JSONDB _db; + Mutex _db_m; // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places // This does lock _networkMemberCache_m @@ -203,9 +206,6 @@ private: // Last request time by address, for rate limitation std::map< std::pair,uint64_t > _lastRequestTime; Mutex _lastRequestTime_m; - - Thread _daemon; - volatile bool _daemonRun; }; } // namespace ZeroTier diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 40bc21ac..bb0b98ed 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -27,7 +27,7 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) if (!_isValidObjectName(n)) return false; - std::string path(_genPath(n,false)); + std::string path(_genPath(n,true)); if (!path.length()) return false; @@ -36,10 +36,9 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) return false; _E &e = _db[n]; - + e.obj = obj; e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); e.lastCheck = OSUtils::now(); - e.obj = obj; return true; } @@ -56,7 +55,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe if (e != _db.end()) { if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck) return e->second.obj; - std::string path(_genPath(n,false)); if (!path.length()) // sanity check return _EMPTY_JSON; @@ -64,15 +62,13 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe // We are somewhat tolerant to momentary disk failures here. This may // occur over e.g. EC2's elastic filesystem (NFS). const uint64_t lm = OSUtils::getLastModified(path.c_str()); - if ((lm)&&(e->second.lastModifiedOnDisk != lm)) { + if (e->second.lastModifiedOnDisk != lm) { if (OSUtils::readFile(path.c_str(),buf)) { try { - e->second.lastModifiedOnDisk = lm; - e->second.lastCheck = now; e->second.obj = nlohmann::json::parse(buf); - } catch ( ... ) { - e->second.obj = _EMPTY_JSON; - } + e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP + e->second.lastCheck = now; + } catch ( ... ) {} // parse errors result in "holding pattern" behavior } } @@ -86,22 +82,50 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe return _EMPTY_JSON; const uint64_t lm = OSUtils::getLastModified(path.c_str()); - if (!lm) - return _EMPTY_JSON; - _E &e2 = _db[n]; - e2.lastModifiedOnDisk = lm; - e2.lastCheck = now; try { e2.obj = nlohmann::json::parse(buf); } catch ( ... ) { e2.obj = _EMPTY_JSON; } + e2.lastModifiedOnDisk = lm; + e2.lastCheck = now; return e2.obj; } } +void JSONDB::erase(const std::string &n) +{ + if (!_isValidObjectName(n)) + return; + + std::string path(_genPath(n,true)); + if (!path.length()) + return; + + OSUtils::rm(path.c_str()); + _db.erase(n); +} + +void JSONDB::_reload(const std::string &p) +{ + std::map l(OSUtils::listDirectoryFull(p.c_str())); + for(std::map::iterator li(l.begin());li!=l.end();++li) { + if (li->second == 'f') { + // assume p starts with _basePath, which it always does -- will throw otherwise + std::string n(p.substr(_basePath.length())); + while ((n.length() > 0)&&(n[0] == ZT_PATH_SEPARATOR)) n = n.substr(1); + if (ZT_PATH_SEPARATOR != '/') std::replace(n.begin(),n.end(),ZT_PATH_SEPARATOR,'/'); + if ((n.length() > 0)&&(n[n.length() - 1] != '/')) n.push_back('/'); + n.append(li->first); + this->get(n,0); // causes load and cache or update + } else if (li->second == 'd') { + this->_reload(p + ZT_PATH_SEPARATOR + li->first); + } + } +} + bool JSONDB::_isValidObjectName(const std::string &n) { if (n.length() == 0) @@ -139,8 +163,4 @@ std::string JSONDB::_genPath(const std::string &n,bool create) return p; } -void JSONDB::_reloadAll(const std::string &path) -{ -} - } // namespace ZeroTier diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index b189c530..e33cd768 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -41,13 +41,21 @@ public: JSONDB(const std::string &basePath) : _basePath(basePath) { - this->_reloadAll(_basePath); + _reload(_basePath); + } + + inline void reload() + { + _db.clear(); + _reload(_basePath); } bool put(const std::string &n,const nlohmann::json &obj); inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); } + inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); } + inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); } const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0); @@ -56,27 +64,37 @@ public: inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); } inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); } + void erase(const std::string &n); + + inline void erase(const std::string &n1,const std::string &n2) { this->erase(n1 + "/" + n2); } + inline void erase(const std::string &n1,const std::string &n2,const std::string &n3) { this->erase(n1 + "/" + n2 + "/" + n3); } + inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4); } + inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5); } + template - inline void each(F func,unsigned long maxSinceCheck = 0) + inline void filter(const std::string &prefix,unsigned long maxSinceCheck,F func) { - const uint64_t now = OSUtils::now(); - for(std::map::const_iterator i(_db.begin());i!=_db.end();++i) { - if ((now - i->second.lastCheck) > (uint64_t)maxSinceCheck) - this->get(i->first); - func(i->first,i->second.obj); + for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { + if (i->first.substr(0,prefix.length()) == prefix) { + if (!func(i->first,get(i->second.obj,maxSinceCheck))) { + std::map::iterator i2(i); ++i2; + this->erase(i->first); + i = i2; + } else ++i; + } else break; } } private: + void _reload(const std::string &p); bool _isValidObjectName(const std::string &n); std::string _genPath(const std::string &n,bool create); - void _reloadAll(const std::string &path); struct _E { + nlohmann::json obj; uint64_t lastModifiedOnDisk; uint64_t lastCheck; - nlohmann::json obj; }; std::string _basePath; diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index c652e272..4a81625b 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -107,17 +107,18 @@ std::vector OSUtils::listDirectory(const char *path) return r; } -std::vector OSUtils::listSubdirectories(const char *path) +std::map OSUtils::listDirectoryFull(const char *path) { - std::vector r; + std::map r; #ifdef __WINDOWS__ HANDLE hFind; WIN32_FIND_DATAA ffd; if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { do { - if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))&&((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)) - r.push_back(std::string(ffd.cFileName)); + if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))) { + r[ffd.cFileName] = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) ? 'd' : 'f'; + } } while (FindNextFileA(hFind,&ffd)); FindClose(hFind); } @@ -132,8 +133,9 @@ std::vector OSUtils::listSubdirectories(const char *path) if (readdir_r(d,&de,&dptr)) break; if (dptr) { - if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_DIR)) - r.push_back(std::string(dptr->d_name)); + if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))) { + r[dptr->d_name] = (dptr->d_type == DT_DIR) ? 'd' : 'f'; + } } else break; } closedir(d); @@ -178,7 +180,7 @@ bool OSUtils::rmDashRf(const char *path) std::string p(path); p.push_back(ZT_PATH_SEPARATOR); p.append(dptr->d_name); - if (unlink(p.c_str()) != 0) { + if (unlink(p.c_str()) != 0) { // unlink first will remove symlinks instead of recursing them if (!rmDashRf(p.c_str())) return false; } diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 4f74344f..c481780b 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -110,12 +110,12 @@ public: static std::vector listDirectory(const char *path); /** - * List a directory's subdirectories + * List all contents in a directory * * @param path Path to list - * @return Names of subdirectories (without path prepended) + * @return Names of things and types, currently just 'f' and 'd' */ - static std::vector listSubdirectories(const char *path); + static std::map listDirectoryFull(const char *path); /** * Delete a directory and all its files and subdirectories recursively -- cgit v1.2.3 From 7729cbe31368abb2a737bfc5a74e81c4fb4c6717 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 4 Nov 2016 15:34:49 -0700 Subject: Fix ambiguous error on some compilers. --- controller/JSONDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index bb0b98ed..59110f25 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -20,7 +20,7 @@ namespace ZeroTier { -static const nlohmann::json _EMPTY_JSON({{}}); +static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); bool JSONDB::put(const std::string &n,const nlohmann::json &obj) { -- cgit v1.2.3 From 330a07a554e0d872f61750c39da06a67c7c1a784 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 4 Nov 2016 15:48:23 -0700 Subject: cleanup --- controller/EmbeddedNetworkController.hpp | 49 ++------------------------------ 1 file changed, 3 insertions(+), 46 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index f364c078..dee76271 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -85,49 +85,6 @@ public: private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); - // Network base path and network JSON path - /* - inline std::string _networkBP(const uint64_t nwid,bool create) - { - char tmp[64]; - std::string p(_path + ZT_PATH_SEPARATOR_S + "network"); - if (create) OSUtils::mkdir(p.c_str()); - p.push_back(ZT_PATH_SEPARATOR); - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nwid); - p.append(tmp); - if (create) OSUtils::mkdir(p.c_str()); - return p; - } - inline std::string _networkJP(const uint64_t nwid,bool create) - { - return (_networkBP(nwid,create) + ZT_PATH_SEPARATOR + "config.json"); - } - - // Member base path and member JSON path - inline std::string _memberBP(const uint64_t nwid,const Address &member,bool create) - { - std::string p(_networkBP(nwid,create)); - p.push_back(ZT_PATH_SEPARATOR); - p.append("member"); - if (create) OSUtils::mkdir(p.c_str()); - p.push_back(ZT_PATH_SEPARATOR); - p.append(member.toString()); - if (create) OSUtils::mkdir(p.c_str()); - return p; - } - inline std::string _memberJP(const uint64_t nwid,const Address &member,bool create) - { - return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); - } - - // In-memory cache of network members - std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache; - Mutex _networkMemberCache_m; - */ - - JSONDB _db; - Mutex _db_m; - // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places // This does lock _networkMemberCache_m struct _NetworkMemberInfo @@ -190,11 +147,12 @@ private: member["clock"] = now; } - // These are const after construction + JSONDB _db; + Mutex _db_m; + Node *const _node; std::string _path; - // Circuit tests outstanding struct _CircuitTestEntry { ZT_CircuitTest *test; @@ -203,7 +161,6 @@ private: std::map< uint64_t,_CircuitTestEntry > _circuitTests; Mutex _circuitTests_m; - // Last request time by address, for rate limitation std::map< std::pair,uint64_t > _lastRequestTime; Mutex _lastRequestTime_m; }; -- cgit v1.2.3 From cae9041c2a656431cbcd69128a7f0dba5a856b04 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 4 Nov 2016 15:52:01 -0700 Subject: . --- controller/EmbeddedNetworkController.hpp | 1 + 1 file changed, 1 insertion(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index dee76271..59404f06 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -130,6 +130,7 @@ private: // If unspecified, rules are set to allow anything and behave like a flat L2 segment network["rules"] = {{ { "not",false }, + { "or", false }, { "type","ACTION_ACCEPT" } }}; } -- cgit v1.2.3 From 0d108d37f64b7fe6d7f42632b67cab6e1a200ba5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 4 Nov 2016 16:12:44 -0700 Subject: . --- controller/EmbeddedNetworkController.cpp | 18 ------------------ controller/JSONDB.hpp | 5 ++++- 2 files changed, 4 insertions(+), 19 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 21cd78e4..8f86cc46 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -63,24 +63,6 @@ using json = nlohmann::json; namespace ZeroTier { -// JSON blob I/O -/* -static json _readJson(const std::string &path) -{ - std::string buf; - if (OSUtils::readFile(path.c_str(),buf)) { - try { - return json::parse(buf); - } catch ( ... ) {} - } - return json::object(); -} -static bool _writeJson(const std::string &path,const json &obj) -{ - return OSUtils::writeFile(path.c_str(),obj.dump(2)); -} -*/ - // Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible static uint64_t _jI(const json &jv,const uint64_t dfl) { diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index e33cd768..61735655 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -19,6 +19,9 @@ #ifndef ZT_JSONDB_HPP #define ZT_JSONDB_HPP +#include +#include + #include #include #include @@ -75,7 +78,7 @@ public: inline void filter(const std::string &prefix,unsigned long maxSinceCheck,F func) { for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { - if (i->first.substr(0,prefix.length()) == prefix) { + if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { if (!func(i->first,get(i->second.obj,maxSinceCheck))) { std::map::iterator i2(i); ++i2; this->erase(i->first); -- cgit v1.2.3 From 08ff666e999376a903b7e7fdc0ba47de2a929bcd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 4 Nov 2016 16:14:58 -0700 Subject: . --- controller/EmbeddedNetworkController.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8f86cc46..3533213e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1364,7 +1364,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (ipp.is_array()) { json nipp = json::array(); for(unsigned long i=0;i Date: Fri, 4 Nov 2016 16:23:41 -0700 Subject: Fix network list API call. --- controller/EmbeddedNetworkController.cpp | 28 +++++++++++++++++----------- controller/JSONDB.cpp | 4 +++- 2 files changed, 20 insertions(+), 12 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3533213e..9c7611a4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1019,16 +1019,22 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } } else if (path.size() == 1) { - /* - std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); - for(auto i(networks.begin());i!=networks.end();++i) { - if (i->length() == 16) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\""); - } + std::set networkIds; + { + Mutex::Lock _l(_db_m); + _db.filter("network/",120000,[&networkIds](const std::string &n,const json &obj) { + if (n.length() == (16 + 8)) + networkIds.insert(n.substr(8)); + return true; // do not delete + }); + } + + responseBody.push_back('['); + for(std::set::iterator i(networkIds.begin());i!=networkIds.end();++i) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\""); } - */ responseBody.push_back(']'); responseContentType = "application/json"; return 200; @@ -1147,7 +1153,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (tags.is_array()) { std::map mtags; for(unsigned long i=0;i 0)&&(n[n.length() - 1] != '/')) n.push_back('/'); n.append(li->first); - this->get(n,0); // causes load and cache or update + if ((n.length() > 5)&&(n.substr(n.length() - 5) == ".json")) { + this->get(n.substr(0,n.length() - 5),0); // causes load and cache or update + } } else if (li->second == 'd') { this->_reload(p + ZT_PATH_SEPARATOR + li->first); } -- cgit v1.2.3 From a454a37a6e171f2a9101680de4c2dfda2c2246f5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 7 Nov 2016 13:27:17 -0800 Subject: Self test JSONDB. --- controller/JSONDB.cpp | 2 -- selftest.cpp | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 506fc425..6adc8a66 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -147,8 +147,6 @@ std::string JSONDB::_genPath(const std::string &n,bool create) std::vector pt(Utils::split(n.c_str(),"/","","")); if (pt.size() == 0) return std::string(); - if (pt.size() == 1) - return pt[0]; std::string p(_basePath); if (create) OSUtils::mkdir(p.c_str()); diff --git a/selftest.cpp b/selftest.cpp index aecccd21..5fcc2673 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -53,6 +53,8 @@ #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" +#include "controller/JSONDB.hpp" + #ifdef __WINDOWS__ #include #endif @@ -807,6 +809,24 @@ static int testOther() } std::cout << "PASS (junk value to prevent optimization-out of test: " << foo << ")" << std::endl; + std::cout << "[other] Testing controller/JSONDB..."; std::cout.flush(); + { + JSONDB db1("jsondb-test"); + std::map db1data; + for(unsigned int i=0;i<256;++i) { + std::string n; + for(unsigned int j=0,k=rand() % 4;j<=k;++j) { + if (j > 0) n.push_back('/'); + char foo[24]; + Utils::snprintf(foo,sizeof(foo),"%lx",rand()); + n.append(foo); + } + db1data[n] = {{"i",i}}; + db1.put(n,db1data[n]); + } + } + std::cout << "PASS" << std::endl; + return 0; } -- cgit v1.2.3 From 5f63d5039bcca80241b12655c250a4238b936a74 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 7 Nov 2016 14:01:23 -0800 Subject: Bug fixes, self test of JSONDB disabled by default. --- controller/JSONDB.hpp | 6 ++++++ selftest.cpp | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 61735655..a9a5e6ac 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -88,6 +88,9 @@ public: } } + inline bool operator==(const JSONDB &db) const { return ((_basePath == db._basePath)&&(_db == db._db)); } + inline bool operator!=(const JSONDB &db) const { return (!(*this == db)); } + private: void _reload(const std::string &p); bool _isValidObjectName(const std::string &n); @@ -98,6 +101,9 @@ private: nlohmann::json obj; uint64_t lastModifiedOnDisk; uint64_t lastCheck; + + inline bool operator==(const _E &e) const { return (obj == e.obj); } + inline bool operator!=(const _E &e) const { return (obj != e.obj); } }; std::string _basePath; diff --git a/selftest.cpp b/selftest.cpp index 5fcc2673..7ca4ac3b 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -809,10 +809,11 @@ static int testOther() } std::cout << "PASS (junk value to prevent optimization-out of test: " << foo << ")" << std::endl; + /* std::cout << "[other] Testing controller/JSONDB..."; std::cout.flush(); { - JSONDB db1("jsondb-test"); std::map db1data; + JSONDB db1("jsondb-test"); for(unsigned int i=0;i<256;++i) { std::string n; for(unsigned int j=0,k=rand() % 4;j<=k;++j) { @@ -824,8 +825,26 @@ static int testOther() db1data[n] = {{"i",i}}; db1.put(n,db1data[n]); } + for(std::map::iterator i(db1data.begin());i!=db1data.end();++i) { + i->second["foo"] = "bar"; + db1.put(i->first,i->second); + } + JSONDB db2("jsondb-test"); + if (db1 != db2) { + std::cout << " FAILED (db1!=db2 #1)" << std::endl; + return -1; + } + for(std::map::iterator i(db1data.begin());i!=db1data.end();++i) { + db1.erase(i->first); + } + db2.reload(); + if (db1 != db2) { + std::cout << " FAILED (db1!=db2 #2)" << std::endl; + return -1; + } } - std::cout << "PASS" << std::endl; + std::cout << " PASS" << std::endl; + */ return 0; } -- cgit v1.2.3 From 4868d21526d7cbced5786dc9359a6ded4200e284 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 7 Nov 2016 23:49:03 +0000 Subject: Bug fixes in controller refactor. --- controller/EmbeddedNetworkController.cpp | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 9c7611a4..d60bde98 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1073,10 +1073,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( responseContentType = "application/json"; return 400; } - } catch (std::exception &exc) { - responseBody = std::string("{ \"message\": \"body JSON is invalid: ") + exc.what() + "\" }"; - responseContentType = "application/json"; - return 400; } catch ( ... ) { responseBody = "{ \"message\": \"body JSON is invalid\" }"; responseContentType = "application/json"; @@ -1092,13 +1088,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); if (path.size() >= 3) { - json network; - { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,0); - } - if (!network.size()) - return 404; if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { uint64_t address = Utils::hexStrToU64(path[3].c_str()); @@ -1110,8 +1099,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Mutex::Lock _l(_db_m); member = _db.get("network",nwids,"member",Address(address).toString(),0); } - if (!member.size()) - return 404; _initMember(member); try { @@ -1283,8 +1270,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } network = _db.get("network",nwids,0); - if (!network.size()) - return 404; } _initNetwork(network); -- cgit v1.2.3 From 360c84e0351728b0c0479ceddba924997acb46e4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 8 Nov 2016 00:05:18 +0000 Subject: Minor fixes. --- controller/EmbeddedNetworkController.cpp | 6 ++++-- controller/JSONDB.hpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d60bde98..85717e28 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1508,20 +1508,22 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( Mutex::Lock _l(_db_m); json member = _db.get("network",nwids,"member",Address(address).toString(),0); - if (!member.size()) - return 404; _db.erase("network",nwids,"member",Address(address).toString()); + if (!member.size()) + return 404; responseBody = member.dump(2); responseContentType = "application/json"; return 200; } } else { Mutex::Lock _l(_db_m); + std::string pfx("network/"); pfx.append(nwids); _db.filter(pfx,120000,[](const std::string &n,const json &obj) { return false; // delete }); + responseBody = network.dump(2); responseContentType = "application/json"; return 200; diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index a9a5e6ac..bd1ae5a5 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -79,7 +79,7 @@ public: { for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { - if (!func(i->first,get(i->second.obj,maxSinceCheck))) { + if (!func(i->first,get(i->first,maxSinceCheck))) { std::map::iterator i2(i); ++i2; this->erase(i->first); i = i2; -- cgit v1.2.3 From 4524899e4df849ab661a28676e5d0c44d239aa9f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 8 Nov 2016 12:41:27 -0800 Subject: Update LM time on members on request. --- controller/EmbeddedNetworkController.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 85717e28..c7114758 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -622,6 +622,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["recentLog"] = recentLog; } + // Update last modified time + member["lastModified"] = now; + // If they are not authorized, STOP! if (!authorizedBy) { Mutex::Lock _l(_db_m); -- cgit v1.2.3 From 3d948a930e935f126ab661c63e698283ff937380 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 8 Nov 2016 14:24:30 -0800 Subject: Send a blanket rule to old versions. New versions will still bidirecitonally enforce on the inbound side. --- controller/EmbeddedNetworkController.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c7114758..2871df9b 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -673,12 +673,20 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( json &memberCapabilities = member["capabilities"]; json &memberTags = member["tags"]; - if (rules.is_array()) { - for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) - break; - if (_parseRule(rules[i],nc.rules[nc.ruleCount])) - ++nc.ruleCount; + if (metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,0) <= 0) { + // Old versions with no rules engine support get an allow everything rule. + // Since rules are enforced bidirectionally, newer versions *will* still + // enforce rules on the inbound side. + nc.ruleCount = 1; + nc.rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; + } else { + if (rules.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + break; + if (_parseRule(rules[i],nc.rules[nc.ruleCount])) + ++nc.ruleCount; + } } } -- cgit v1.2.3 From 1ebfca666d19dbc7def22c4a0f4e8071a3977357 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 9 Nov 2016 12:34:20 -0800 Subject: Memo-ize some computed stuff to control CPU utilization. --- controller/EmbeddedNetworkController.cpp | 80 ++++++++++++++++++++------------ controller/EmbeddedNetworkController.hpp | 3 ++ 2 files changed, 53 insertions(+), 30 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 2871df9b..f91ba533 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1535,6 +1535,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( return false; // delete }); + Mutex::Lock _l2(_nmiCache_m); + _nmiCache.erase(nwid); + responseBody = network.dump(2); responseContentType = "application/json"; return 200; @@ -1614,43 +1617,60 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid char pfx[256]; Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid); - Mutex::Lock _l(_db_m); - _db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) { - try { - if (_jB(member["authorized"],false)) { - ++nmi.authorizedMemberCount; - - if (member.count("recentLog")) { - const json &mlog = member["recentLog"]; - if ((mlog.is_array())&&(mlog.size() > 0)) { - const json &mlog1 = mlog[0]; - if (mlog1.is_object()) { - if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) - ++nmi.activeMemberCount; + { + Mutex::Lock _l(_nmiCache_m); + std::map::iterator c(_nmiCache.find(nwid)); + if ((c != _nmiCache.end())&&((now - c->second.nmiTimestamp) < 1000)) { // a short duration cache but limits CPU use on big networks + nmi = c->second; + return; + } + } + + { + Mutex::Lock _l(_db_m); + _db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) { + try { + if (_jB(member["authorized"],false)) { + ++nmi.authorizedMemberCount; + + if (member.count("recentLog")) { + const json &mlog = member["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + const json &mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) + ++nmi.activeMemberCount; + } } } - } - if (_jB(member["activeBridge"],false)) { - nmi.activeBridges.insert(_jS(member["id"],"0000000000")); - } + if (_jB(member["activeBridge"],false)) { + nmi.activeBridges.insert(_jS(member["id"],"0000000000")); + } - if (member.count("ipAssignments")) { - const json &mips = member["ipAssignments"]; - if (mips.is_array()) { - for(unsigned long i=0;i _nmiCache; + Mutex _nmiCache_m; void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); // These init objects with default and static/informational fields -- cgit v1.2.3 From eea712a1ae80994c32f250537ce14e52071312fb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 9 Nov 2016 13:26:14 -0800 Subject: Field in wrong place fixed. --- controller/EmbeddedNetworkController.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 2a66c530..53d3be0f 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -115,7 +115,6 @@ private: if (!member.count("creationTime")) member["creationTime"] = OSUtils::now(); if (!member.count("noAutoAssignIps")) member["noAutoAssignIps"] = false; if (!member.count("revision")) member["revision"] = 0ULL; - if (!member.count("enableBroadcast")) member["enableBroadcast"] = true; member["objtype"] = "member"; } inline void _initNetwork(nlohmann::json &network) @@ -124,6 +123,7 @@ private: if (!network.count("creationTime")) network["creationTime"] = OSUtils::now(); if (!network.count("name")) network["name"] = ""; if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; + if (!network.count("enableBroadcast")) network["enableBroadcast"] = true; if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); -- cgit v1.2.3 From 5ebf5077f56442908d4a5ced8d969df9e7de8c4f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 9 Nov 2016 17:11:10 -0800 Subject: Log last meta-data in controller, and ease up just a bit on keepalives. --- controller/EmbeddedNetworkController.cpp | 2 +- node/Constants.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index f91ba533..624c3145 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -622,8 +622,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["recentLog"] = recentLog; } - // Update last modified time member["lastModified"] = now; + member["lastRequestMetaData"] = metaData.data(); // If they are not authorized, STOP! if (!authorizedBy) { diff --git a/node/Constants.hpp b/node/Constants.hpp index b7042d5d..6400e289 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -256,12 +256,12 @@ /** * How frequently to send heartbeats over in-use paths */ -#define ZT_PATH_HEARTBEAT_PERIOD 10000 +#define ZT_PATH_HEARTBEAT_PERIOD 14000 /** * Paths are considered inactive if they have not received traffic in this long */ -#define ZT_PATH_ALIVE_TIMEOUT 25000 +#define ZT_PATH_ALIVE_TIMEOUT 45000 /** * Minimum time between attempts to check dead paths to see if they can be re-awakened -- cgit v1.2.3 From 226123ca08ffbb5f4e4f0699b92fb9db08576a66 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 10 Nov 2016 11:54:47 -0800 Subject: Refactor controller to permit sending of pushes as well as just replies to config requests. --- controller/EmbeddedNetworkController.cpp | 77 +++++++++++++++-------- controller/EmbeddedNetworkController.hpp | 14 +++-- node/IncomingPacket.cpp | 84 +------------------------ node/Network.cpp | 105 +++++++++++++------------------ node/Network.hpp | 10 ++- node/NetworkController.hpp | 69 +++++++++++++------- node/Node.cpp | 85 +++++++++++++++++++++++++ node/Node.hpp | 8 ++- 8 files changed, 252 insertions(+), 200 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 624c3145..91b59215 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -470,11 +470,21 @@ EmbeddedNetworkController::~EmbeddedNetworkController() { } -NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) +void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender) { - if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { - return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } + this->_sender = sender; + this->_signingId = signingId; +} + +void EmbeddedNetworkController::request( + uint64_t nwid, + const InetAddress &fromAddr, + uint64_t requestPacketId, + const Identity &identity, + const Dictionary &metaData) +{ + if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) + return; const uint64_t now = OSUtils::now(); @@ -483,7 +493,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( Mutex::Lock _l(_lastRequestTime_m); uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) - return NetworkController::NETCONF_QUERY_IGNORE; + return; lrt = now; } @@ -496,8 +506,13 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( network = _db.get("network",nwids,0); member = _db.get("network",nwids,"member",identity.address().toString(),0); } - if (!network.size()) - return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; + + if (!network.size()) { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); + return; + } + + json origMember(member); // for detecting modification later _initMember(member); { @@ -507,10 +522,13 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // a "collision" from being able to auth onto our network in place of an already // known member. try { - if (Identity(haveIdStr.c_str()) != identity) - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + if (Identity(haveIdStr.c_str()) != identity) { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; + } } catch ( ... ) { - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; } } else { // If we do not yet know this member's identity, learn it. @@ -521,7 +539,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // These are always the same, but make sure they are set member["id"] = identity.address().toString(); member["address"] = member["id"]; - member["nwid"] = network["id"]; + member["nwid"] = nwids; // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; @@ -597,7 +615,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } // Log this request - { + if (requestPacketId) { // only log if this is a request, not for generated pushes json rlEntry = json::object(); rlEntry["ts"] = now; rlEntry["authorized"] = (authorizedBy) ? true : false; @@ -620,22 +638,27 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } member["recentLog"] = recentLog; - } - member["lastModified"] = now; - member["lastRequestMetaData"] = metaData.data(); + // Also only do this on real requests + member["lastRequestMetaData"] = metaData.data(); + } // If they are not authorized, STOP! if (!authorizedBy) { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",identity.address().toString(),member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + if (origMember != member) { + member["lastModified"] = now; + Mutex::Lock _l(_db_m); + _db.put("network",nwids,"member",identity.address().toString(),member); + } + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; } // ------------------------------------------------------------------------- // If we made it this far, they are authorized. // ------------------------------------------------------------------------- + NetworkConfig nc; _NetworkMemberInfo nmi; _getNetworkMemberInfo(now,nwid,nmi); @@ -661,8 +684,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); - for(std::set

::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) + for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) { nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + } json &v4AssignMode = network["v4AssignMode"]; json &v6AssignMode = network["v6AssignMode"]; @@ -714,7 +738,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(signingId,identity.address())) + if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) ++nc.capabilityCount; if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) break; @@ -733,7 +757,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) break; nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(signingId)) + if (nc.tags[nc.tagCount].sign(_signingId)) ++nc.tagCount; } } @@ -923,17 +947,20 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); - if (com.sign(signingId)) { + if (com.sign(_signingId)) { nc.com = com; } else { - return NETCONF_QUERY_INTERNAL_SERVER_ERROR; + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR); + return; } - { + if (member != origMember) { + member["lastModified"] = now; Mutex::Lock _l(_db_m); _db.put("network",nwids,"member",identity.address().toString(),member); } - return NetworkController::NETCONF_QUERY_OK; + + _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 53d3be0f..79b919b9 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -52,13 +52,14 @@ public: EmbeddedNetworkController(Node *node,const char *dbPath); virtual ~EmbeddedNetworkController(); - virtual NetworkController::ResultCode doNetworkConfigRequest( + virtual void init(const Identity &signingId,Sender *sender); + + virtual void request( + uint64_t nwid, const InetAddress &fromAddr, - const Identity &signingId, + uint64_t requestPacketId, const Identity &identity, - uint64_t nwid, - const Dictionary &metaData, - NetworkConfig &nc); + const Dictionary &metaData); unsigned int handleControlPlaneHttpGET( const std::vector &path, @@ -157,6 +158,9 @@ private: Node *const _node; std::string _path; + NetworkController::Sender *_sender; + Identity _signingId; + struct _CircuitTestEntry { ZT_CircuitTest *test; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 5afacd0e..bde5df71 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -865,92 +865,12 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); const unsigned int hopCount = hops(); const uint64_t requestPacketId = packetId(); - bool trustEstablished = false; if (RR->localNetworkController) { const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); const Dictionary metaData(metaDataBytes,metaDataLength); - - NetworkConfig *netconf = new NetworkConfig(); - try { - switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _path->address(),RR->identity,peer->identity(),nwid,metaData,*netconf)) { - - case NetworkController::NETCONF_QUERY_OK: { - trustEstablished = true; - Dictionary *dconf = new Dictionary(); - try { - if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { - uint64_t configUpdateId = RR->node->prng(); - if (!configUpdateId) ++configUpdateId; - const unsigned int totalSize = dconf->sizeBytes(); - unsigned int chunkIndex = 0; - while (chunkIndex < totalSize) { - const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256))); - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - - const unsigned int sigStart = outp.size(); - outp.append(nwid); - outp.append((uint16_t)chunkLen); - outp.append((const void *)(dconf->data() + chunkIndex),chunkLen); - - outp.append((uint8_t)0); // no flags - outp.append((uint64_t)configUpdateId); - outp.append((uint32_t)totalSize); - outp.append((uint32_t)chunkIndex); - - C25519::Signature sig(RR->identity.sign(reinterpret_cast(outp.data()) + sigStart,outp.size() - sigStart)); - outp.append((uint8_t)1); - outp.append((uint16_t)ZT_C25519_SIGNATURE_LEN); - outp.append(sig.data,ZT_C25519_SIGNATURE_LEN); - - outp.compress(); - RR->sw->send(outp,true); - chunkIndex += chunkLen; - } - } - delete dconf; - } catch ( ... ) { - delete dconf; - throw; - } - } break; - - case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); - outp.append(nwid); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); - } break; - - case NetworkController::NETCONF_QUERY_ACCESS_DENIED: { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); - outp.append(nwid); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); - } break; - - case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR: - break; - case NetworkController::NETCONF_QUERY_IGNORE: - break; - default: - TRACE("NETWORK_CONFIG_REQUEST failed: invalid return value from NetworkController::doNetworkConfigRequest()"); - break; - } - delete netconf; - } catch ( ... ) { - delete netconf; - throw; - } + RR->localNetworkController->request(nwid,(hopCount > 0) ? InetAddress() : _path->address(),requestPacketId,peer->identity(),metaData); } else { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); @@ -961,7 +881,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,trustEstablished); + peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); diff --git a/node/Network.cpp b/node/Network.cpp index c0e4b105..1f8e7ebf 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -599,7 +599,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : if (conf.length()) { dconf->load(conf.c_str()); if (nconf->fromDictionary(*dconf)) { - this->_setConfiguration(*nconf,false); + this->setConfiguration(*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; } @@ -1015,7 +1015,7 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) } if (nc) { - this->_setConfiguration(*nc,true); + this->setConfiguration(*nc,true); delete nc; return configUpdateId; } else { @@ -1025,6 +1025,46 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) return 0; } +int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) +{ + // _lock is NOT locked when this is called + try { + if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) + return 0; + if (_config == nconf) + return 1; // OK config, but duplicate of what we already have + + ZT_VirtualNetworkConfig ctmp; + bool oldPortInitialized; + { + Mutex::Lock _l(_lock); + _config = nconf; + _lastConfigUpdate = RR->node->now(); + _netconfFailure = NETCONF_FAILURE_NONE; + oldPortInitialized = _portInitialized; + _portInitialized = true; + _externalConfig(&ctmp); + } + _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + + if (saveToDisk) { + Dictionary *d = new Dictionary(); + try { + char n[64]; + Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); + if (nconf.toDictionary(*d,false)) + RR->node->dataStorePut(n,(const void *)d->data(),d->sizeBytes(),true); + } catch ( ... ) {} + delete d; + } + + return 2; // OK and configuration has changed + } catch ( ... ) { + TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); + } + return 0; +} + void Network::requestConfiguration() { const Address ctrl(controller()); @@ -1046,26 +1086,7 @@ void Network::requestConfiguration() if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { - NetworkConfig *nconf = new NetworkConfig(); - try { - switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,*nconf)) { - case NetworkController::NETCONF_QUERY_OK: - this->_setConfiguration(*nconf,true); - break; - case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: - this->setNotFound(); - break; - case NetworkController::NETCONF_QUERY_ACCESS_DENIED: - this->setAccessDenied(); - break; - default: - this->setNotFound(); - break; - } - } catch ( ... ) { - this->setNotFound(); - } - delete nconf; + RR->localNetworkController->request(_id,InetAddress(),0xffffffffffffffffULL,RR->identity,rmd); } else { this->setNotFound(); } @@ -1257,46 +1278,6 @@ ZT_VirtualNetworkStatus Network::_status() const } } -int Network::_setConfiguration(const NetworkConfig &nconf,bool saveToDisk) -{ - // _lock is NOT locked when this is called - try { - if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) - return 0; - if (_config == nconf) - return 1; // OK config, but duplicate of what we already have - - ZT_VirtualNetworkConfig ctmp; - bool oldPortInitialized; - { - Mutex::Lock _l(_lock); - _config = nconf; - _lastConfigUpdate = RR->node->now(); - _netconfFailure = NETCONF_FAILURE_NONE; - oldPortInitialized = _portInitialized; - _portInitialized = true; - _externalConfig(&ctmp); - } - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); - - if (saveToDisk) { - Dictionary *d = new Dictionary(); - try { - char n[64]; - Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - if (nconf.toDictionary(*d,false)) - RR->node->dataStorePut(n,(const void *)d->data(),d->sizeBytes(),true); - } catch ( ... ) {} - delete d; - } - - return 2; // OK and configuration has changed - } catch ( ... ) { - TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); - } - return 0; -} - void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const { // assumes _lock is locked diff --git a/node/Network.hpp b/node/Network.hpp index 527d3048..1627be58 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -187,6 +187,15 @@ public: */ uint64_t handleConfigChunk(const Packet &chunk,unsigned int ptr); + /** + * Set network configuration + * + * @param nconf Network configuration + * @param saveToDisk Save to disk? Used during loading, should usually be true otherwise. + * @return 0 == bad, 1 == accepted but duplicate/unchanged, 2 == accepted and new + */ + int setConfiguration(const NetworkConfig &nconf,bool saveToDisk); + /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this */ @@ -328,7 +337,6 @@ public: inline void **userPtr() throw() { return &_uPtr; } private: - int _setConfiguration(const NetworkConfig &nconf,bool saveToDisk); ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); diff --git a/node/NetworkController.hpp b/node/NetworkController.hpp index db95dd14..fc5db4af 100644 --- a/node/NetworkController.hpp +++ b/node/NetworkController.hpp @@ -27,7 +27,6 @@ namespace ZeroTier { -class RuntimeEnvironment; class Identity; class Address; struct InetAddress; @@ -38,45 +37,69 @@ struct InetAddress; class NetworkController { public: + enum ErrorCode + { + NC_ERROR_NONE = 0, + NC_ERROR_OBJECT_NOT_FOUND = 1, + NC_ERROR_ACCESS_DENIED = 2, + NC_ERROR_INTERNAL_SERVER_ERROR = 3 + }; + /** - * Return value of doNetworkConfigRequest + * Interface for sender used to send pushes and replies */ - enum ResultCode + class Sender { - NETCONF_QUERY_OK = 0, - NETCONF_QUERY_OBJECT_NOT_FOUND = 1, - NETCONF_QUERY_ACCESS_DENIED = 2, - NETCONF_QUERY_INTERNAL_SERVER_ERROR = 3, - NETCONF_QUERY_IGNORE = 4 + public: + /** + * Send a configuration to a remote peer + * + * @param nwid Network ID + * @param requestPacketId Request packet ID to send OK(NETWORK_CONFIG_REQUEST) or 0 to send NETWORK_CONFIG (push) + * @param destination Destination peer Address + * @param nc Network configuration to send + * @param sendLegacyFormatConfig If true, send an old-format network config + */ + virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) = 0; + + /** + * Send a network configuration request error + * + * @param nwid Network ID + * @param requestPacketId Request packet ID or 0 if none + * @param destination Destination peer Address + * @param errorCode Error code + */ + virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) = 0; }; NetworkController() {} virtual ~NetworkController() {} /** - * Handle a network config request, sending replies if necessary - * - * This call is permitted to block, and may be called concurrently from more - * than one thread. Implementations must use locks if needed. + * Called when this is added to a Node to initialize and supply info * - * On internal server errors, the 'error' field in result can be filled in - * to indicate the error. + * @param signingId Identity for signing of network configurations, certs, etc. + * @param sender Sender implementation for sending replies or config pushes + */ + virtual void init(const Identity &signingId,Sender *sender) = 0; + + /** + * Handle a network configuration request * - * @param fromAddr Originating wire address or null address if packet is not direct (or from self) - * @param signingId Identity that should be used to sign results -- must include private key - * @param identity Originating peer ZeroTier identity * @param nwid 64-bit network ID + * @param fromAddr Originating wire address or null address if packet is not direct (or from self) + * @param requestPacketId Packet ID of request packet or 0 if not initiated by remote request + * @param identity ZeroTier identity of originating peer * @param metaData Meta-data bundled with request (if any) - * @param nc NetworkConfig to fill with results * @return Returns NETCONF_QUERY_OK if result 'nc' is valid, or an error code on error */ - virtual NetworkController::ResultCode doNetworkConfigRequest( + virtual void request( + uint64_t nwid, const InetAddress &fromAddr, - const Identity &signingId, + uint64_t requestPacketId, const Identity &identity, - uint64_t nwid, - const Dictionary &metaData, - NetworkConfig &nc) = 0; + const Dictionary &metaData) = 0; }; } // namespace ZeroTier diff --git a/node/Node.cpp b/node/Node.cpp index 9314478f..69808bcf 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -490,6 +490,7 @@ void Node::clearLocalInterfaceAddresses() void Node::setNetconfMaster(void *networkControllerInstance) { RR->localNetworkController = reinterpret_cast(networkControllerInstance); + RR->localNetworkController->init(RR->identity,this); } ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) @@ -718,6 +719,90 @@ void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_ RR->topology->setTrustedPaths(reinterpret_cast(networks),ids,count); } +void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) +{ + if (destination == RR->identity.address()) { + SharedPtr n(network(nwid)); + if (!n) return; + n->setConfiguration(nc,true); + } else { + Dictionary *dconf = new Dictionary(); + try { + if (nc.toDictionary(*dconf,sendLegacyFormatConfig)) { + uint64_t configUpdateId = prng(); + if (!configUpdateId) ++configUpdateId; + + const unsigned int totalSize = dconf->sizeBytes(); + unsigned int chunkIndex = 0; + while (chunkIndex < totalSize) { + const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256))); + Packet outp(destination,RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + + const unsigned int sigStart = outp.size(); + outp.append(nwid); + outp.append((uint16_t)chunkLen); + outp.append((const void *)(dconf->data() + chunkIndex),chunkLen); + + outp.append((uint8_t)0); // no flags + outp.append((uint64_t)configUpdateId); + outp.append((uint32_t)totalSize); + outp.append((uint32_t)chunkIndex); + + C25519::Signature sig(RR->identity.sign(reinterpret_cast(outp.data()) + sigStart,outp.size() - sigStart)); + outp.append((uint8_t)1); + outp.append((uint16_t)ZT_C25519_SIGNATURE_LEN); + outp.append(sig.data,ZT_C25519_SIGNATURE_LEN); + + outp.compress(); + RR->sw->send(outp,true); + chunkIndex += chunkLen; + } + } + delete dconf; + } catch ( ... ) { + delete dconf; + throw; + } + } +} + +void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) +{ + if (destination == RR->identity.address()) { + SharedPtr n(network(nwid)); + if (!n) return; + switch(errorCode) { + case NetworkController::NC_ERROR_OBJECT_NOT_FOUND: + case NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR: + n->setNotFound(); + break; + case NetworkController::NC_ERROR_ACCESS_DENIED: + n->setAccessDenied(); + break; + + default: break; + } + } else if (requestPacketId) { + Packet outp(destination,RR->identity.address(),Packet::VERB_ERROR); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + switch(errorCode) { + //case NetworkController::NC_ERROR_OBJECT_NOT_FOUND: + //case NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR: + default: + outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); + break; + case NetworkController::NC_ERROR_ACCESS_DENIED: + outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); + break; + } + outp.append(nwid); + RR->sw->send(outp,true); + } +} + } // namespace ZeroTier /****************************************************************************/ diff --git a/node/Node.hpp b/node/Node.hpp index 11462531..e616da3d 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -36,6 +36,7 @@ #include "Network.hpp" #include "Path.hpp" #include "Salsa20.hpp" +#include "NetworkController.hpp" #undef TRACE #ifdef ZT_TRACE @@ -55,7 +56,7 @@ namespace ZeroTier { * * The pointer returned by ZT_Node_new() is an instance of this class. */ -class Node +class Node : public NetworkController::Sender { public: Node( @@ -69,7 +70,7 @@ public: ZT_PathCheckFunction pathCheckFunction, ZT_EventCallback eventCallback); - ~Node(); + virtual ~Node(); // Public API Functions ---------------------------------------------------- @@ -282,6 +283,9 @@ public: return false; } + virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig); + virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode); + private: inline SharedPtr _network(uint64_t nwid) const { -- cgit v1.2.3 From 298e4a9f14e9697b2d780f791f9ec6ada4f974a6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 10 Nov 2016 12:33:09 -0800 Subject: Also avoid sending tags and caps to old members since there is no point. --- controller/EmbeddedNetworkController.cpp | 80 ++++++++++++++++---------------- 1 file changed, 40 insertions(+), 40 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 91b59215..ed8ffa74 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -712,53 +712,53 @@ void EmbeddedNetworkController::request( ++nc.ruleCount; } } - } - if ((memberCapabilities.is_array())&&(memberCapabilities.size() > 0)&&(capabilities.is_array())) { - std::map< uint64_t,json * > capsById; - for(unsigned long i=0;i 0)&&(capabilities.is_array())) { + std::map< uint64_t,json * > capsById; + for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { - ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; - unsigned int caprc = 0; - json &caprj = (*cap)["rules"]; - if ((caprj.is_array())&&(caprj.size() > 0)) { - for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) - break; - if (_parseRule(caprj[j],capr[caprc])) - ++caprc; + for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { + ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; + unsigned int caprc = 0; + json &caprj = (*cap)["rules"]; + if ((caprj.is_array())&&(caprj.size() > 0)) { + for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) + break; + if (_parseRule(caprj[j],capr[caprc])) + ++caprc; + } } + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); + if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) + ++nc.capabilityCount; + if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + break; } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) - ++nc.capabilityCount; - if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) - break; } } - } - if (memberTags.is_array()) { - std::map< uint32_t,uint32_t > tagsById; - for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { - if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) - break; - nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(_signingId)) - ++nc.tagCount; + if (memberTags.is_array()) { + std::map< uint32_t,uint32_t > tagsById; + for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { + if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + break; + nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); + if (nc.tags[nc.tagCount].sign(_signingId)) + ++nc.tagCount; + } } } -- cgit v1.2.3 From f0fcd222a12372de63a2c1b02e6c7304a5332a35 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 10 Nov 2016 12:54:43 -0800 Subject: Actually push updates when things change. --- controller/EmbeddedNetworkController.cpp | 60 ++++++++++++++++++++++++-------- controller/EmbeddedNetworkController.hpp | 2 ++ 2 files changed, 48 insertions(+), 14 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index ed8ffa74..0fba8749 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -488,7 +488,6 @@ void EmbeddedNetworkController::request( const uint64_t now = OSUtils::now(); - // Check rate limit circuit breaker to prevent flooding { Mutex::Lock _l(_lastRequestTime_m); uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; @@ -1137,12 +1136,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Mutex::Lock _l(_db_m); member = _db.get("network",nwids,"member",Address(address).toString(),0); } + json origMember(member); // for detecting changes _initMember(member); try { if (b.count("activeBridge")) member["activeBridge"] = _jB(b["activeBridge"],false); if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = _jB(b["noAutoAssignIps"],false); - if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = _jS(b["identity"],""); // allow identity to be populated only if not already known if (b.count("authorized")) { const bool newAuth = _jB(b["authorized"],false); @@ -1214,13 +1213,16 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["id"] = addrs; member["address"] = addrs; // legacy member["nwid"] = nwids; - member["lastModified"] = now; - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",Address(address).toString(),member); + if (member != origMember) { + member["lastModified"] = now; + json &revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + { + Mutex::Lock _l(_db_m); + _db.put("network",nwids,"member",Address(address).toString(),member); + } + _pushMemberUpdate(now,nwid,member); } // Add non-persisted fields @@ -1309,6 +1311,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network = _db.get("network",nwids,0); } + json origNetwork(network); // for detecting changes _initNetwork(network); try { @@ -1489,13 +1492,20 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy - json &rev = network["revision"]; - network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL); - network["lastModified"] = now; - { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,network); + if (network != origNetwork) { + json &revj = network["revision"]; + network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + network["lastModified"] = now; + { + Mutex::Lock _l(_db_m); + _db.put("network",nwids,network); + } + std::string pfx("network/"); pfx.append(nwids); pfx.append("/member/"); + _db.filter(pfx,120000,[this,&now,&nwid](const std::string &n,const json &obj) { + _pushMemberUpdate(now,nwid,obj); + return true; // do not delete + }); } _NetworkMemberInfo nmi; @@ -1700,4 +1710,26 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid } } +void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member) +{ + try { + const std::string idstr = member["identity"]; + const std::string mdstr = member["lastRequestMetaData"]; + if ((idstr.length() > 0)&&(mdstr.length() > 0)) { + const Identity id(idstr); + bool online; + { + Mutex::Lock _l(_lastRequestTime_m); + std::map,uint64_t>::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); + online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); + } + Dictionary *metaData = new Dictionary(mdstr.c_str()); + try { + this->request(nwid,InetAddress(),0,id,*metaData); + } catch ( ... ) {} + delete metaData; + } + } catch ( ... ) {} +} + } // namespace ZeroTier diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 79b919b9..620ef3e3 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -103,6 +103,8 @@ private: Mutex _nmiCache_m; void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); + void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); + // These init objects with default and static/informational fields inline void _initMember(nlohmann::json &member) { -- cgit v1.2.3 From 1b10d3413a13ae3b3a4503e806f96130c4c50fff Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 10 Nov 2016 13:08:43 -0800 Subject: Use circuit breaker only for requests. --- controller/EmbeddedNetworkController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0fba8749..85072eef 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -488,7 +488,7 @@ void EmbeddedNetworkController::request( const uint64_t now = OSUtils::now(); - { + if (requestPacketId) { Mutex::Lock _l(_lastRequestTime_m); uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) -- cgit v1.2.3 From e26bee45fb2ed78bc1984886da0ffc48d8f5a102 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 10 Nov 2016 13:57:01 -0800 Subject: Multithreading in network controller. Threads are only started if controller is used. --- controller/EmbeddedNetworkController.cpp | 1433 +++++++++++++++--------------- controller/EmbeddedNetworkController.hpp | 27 + osdep/BlockingQueue.hpp | 64 ++ 3 files changed, 832 insertions(+), 692 deletions(-) create mode 100644 osdep/BlockingQueue.hpp (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 85072eef..7f885b4e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -459,6 +459,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : + _threadsStarted(false), _db(dbPath), _node(node) { @@ -468,6 +469,13 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa EmbeddedNetworkController::~EmbeddedNetworkController() { + Mutex::Lock _l(_threads_m); + if (_threadsStarted) { + for(int i=0;i<(ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT*2);++i) + _queue.post((_RQEntry *)0); + for(int i=0;i> 24))||(!_sender)) return; - const uint64_t now = OSUtils::now(); - - if (requestPacketId) { - Mutex::Lock _l(_lastRequestTime_m); - uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; - if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) - return; - lrt = now; - } - - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network; - json member; { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,0); - member = _db.get("network",nwids,"member",identity.address().toString(),0); + Mutex::Lock _l(_threads_m); + if (!_threadsStarted) { + for(int i=0;incSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); - return; - } + _RQEntry *qe = new _RQEntry; + qe->nwid = nwid; + qe->requestPacketId = requestPacketId; + qe->fromAddr = fromAddr; + qe->identity = identity; + qe->metaData = metaData; + _queue.post(qe); +} - json origMember(member); // for detecting modification later - _initMember(member); +unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if ((path.size() > 0)&&(path[0] == "network")) { - { - std::string haveIdStr(_jS(member["identity"],"")); - if (haveIdStr.length() > 0) { - // If we already know this member's identity perform a full compare. This prevents - // a "collision" from being able to auth onto our network in place of an already - // known member. - try { - if (Identity(haveIdStr.c_str()) != identity) { - _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); - return; - } - } catch ( ... ) { - _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); - return; + if ((path.size() >= 2)&&(path[1].length() == 16)) { + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + + json network; + { + Mutex::Lock _l(_db_m); + network = _db.get("network",nwids,0); } - } else { - // If we do not yet know this member's identity, learn it. - member["identity"] = identity.toString(false); - } - } + if (!network.size()) + return 404; - // These are always the same, but make sure they are set - member["id"] = identity.address().toString(); - member["address"] = member["id"]; - member["nwid"] = nwids; + if (path.size() >= 3) { - // Determine whether and how member is authorized - const char *authorizedBy = (const char *)0; - if (_jB(member["authorized"],false)) { - authorizedBy = "memberIsAuthorized"; - } else if (!_jB(network["private"],true)) { - authorizedBy = "networkIsPublic"; - if (!member.count("authorized")) { - member["authorized"] = true; - json ah; - ah["a"] = true; - ah["by"] = authorizedBy; - ah["ts"] = now; - ah["ct"] = json(); - ah["c"] = json(); - member["authHistory"].push_back(ah); - member["lastModified"] = now; - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - } - } else { - char presentedAuth[512]; - if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) { - presentedAuth[511] = (char)0; // sanity check + if (path[2] == "member") { - // Check for bearer token presented by member - if ((strlen(presentedAuth) > 6)&&(!strncmp(presentedAuth,"token:",6))) { - const char *const presentedToken = presentedAuth + 6; + if (path.size() >= 4) { + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - json &authTokens = network["authTokens"]; - if (authTokens.is_array()) { - for(unsigned long i=0;i now))&&(tstr == presentedToken)) { - bool usable = (maxUses == 0); - if (!usable) { - uint64_t useCount = 0; - json &ahist = member["authHistory"]; - if (ahist.is_array()) { - for(unsigned long j=0;j 0) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(_jS(member["id"],"")); + responseBody.append("\":"); + responseBody.append(_jS(member["revision"],"0")); } - } + return true; // never delete + }); + responseBody.push_back('}'); + responseContentType = "application/json"; + + return 200; } - } - } - } - } - // Log this request - if (requestPacketId) { // only log if this is a request, not for generated pushes - json rlEntry = json::object(); - rlEntry["ts"] = now; - rlEntry["authorized"] = (authorizedBy) ? true : false; - rlEntry["authorizedBy"] = (authorizedBy) ? authorizedBy : ""; - rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - if (fromAddr) - rlEntry["fromAddr"] = fromAddr.toString(); + } else if ((path[2] == "test")&&(path.size() >= 4)) { - json recentLog = json::array(); - recentLog.push_back(rlEntry); - json &oldLog = member["recentLog"]; - if (oldLog.is_array()) { - for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) - break; - } - } - member["recentLog"] = recentLog; + Mutex::Lock _l(_circuitTests_m); + std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); + if ((cte != _circuitTests.end())&&(cte->second.test)) { - // Also only do this on real requests - member["lastRequestMetaData"] = metaData.data(); - } + responseBody = "["; + responseBody.append(cte->second.jsonResults); + responseBody.push_back(']'); + responseContentType = "application/json"; - // If they are not authorized, STOP! - if (!authorizedBy) { - if (origMember != member) { - member["lastModified"] = now; - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",identity.address().toString(),member); - } - _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); - return; - } + return 200; - // ------------------------------------------------------------------------- - // If we made it this far, they are authorized. - // ------------------------------------------------------------------------- + } // else 404 - NetworkConfig nc; - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); + } // else 404 - // Compute credential TTL. This is the "moving window" for COM agreement and - // the global TTL for Capability and Tag objects. (The same value is used - // for both.) This is computed by reference to the last time we deauthorized - // a member, since within the time period since this event any temporal - // differences are not particularly relevant. - uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; - if (now > nmi.mostRecentDeauthTime) - credentialtmd += (now - nmi.mostRecentDeauthTime); - if (credentialtmd > ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) - credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; + } else { - nc.networkId = nwid; - nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; - nc.timestamp = now; - nc.credentialTimeMaxDelta = credentialtmd; - nc.revision = _jI(network["revision"],0ULL); - nc.issuedTo = identity.address(); - if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (_jB(network["allowPassiveBridging"],false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); - nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); + const uint64_t now = OSUtils::now(); + _NetworkMemberInfo nmi; + _getNetworkMemberInfo(now,nwid,nmi); + _addNetworkNonPersistedFields(network,now,nmi); + responseBody = network.dump(2); + responseContentType = "application/json"; + return 200; - for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) { - nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); - } + } + } else if (path.size() == 1) { - json &v4AssignMode = network["v4AssignMode"]; - json &v6AssignMode = network["v6AssignMode"]; - json &ipAssignmentPools = network["ipAssignmentPools"]; - json &routes = network["routes"]; - json &rules = network["rules"]; - json &capabilities = network["capabilities"]; - json &memberCapabilities = member["capabilities"]; - json &memberTags = member["tags"]; - - if (metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,0) <= 0) { - // Old versions with no rules engine support get an allow everything rule. - // Since rules are enforced bidirectionally, newer versions *will* still - // enforce rules on the inbound side. - nc.ruleCount = 1; - nc.rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; - } else { - if (rules.is_array()) { - for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) - break; - if (_parseRule(rules[i],nc.rules[nc.ruleCount])) - ++nc.ruleCount; - } - } - - if ((memberCapabilities.is_array())&&(memberCapabilities.size() > 0)&&(capabilities.is_array())) { - std::map< uint64_t,json * > capsById; - for(unsigned long i=0;i networkIds; + { + Mutex::Lock _l(_db_m); + _db.filter("network/",120000,[&networkIds](const std::string &n,const json &obj) { + if (n.length() == (16 + 8)) + networkIds.insert(n.substr(8)); + return true; // do not delete + }); } - for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { - ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; - unsigned int caprc = 0; - json &caprj = (*cap)["rules"]; - if ((caprj.is_array())&&(caprj.size() > 0)) { - for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) - break; - if (_parseRule(caprj[j],capr[caprc])) - ++caprc; - } - } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) - ++nc.capabilityCount; - if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) - break; - } + responseBody.push_back('['); + for(std::set::iterator i(networkIds.begin());i!=networkIds.end();++i) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\""); } - } + responseBody.push_back(']'); + responseContentType = "application/json"; + return 200; - if (memberTags.is_array()) { - std::map< uint32_t,uint32_t > tagsById; - for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { - if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) - break; - nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(_signingId)) - ++nc.tagCount; - } - } - } + } // else 404 - if (routes.is_array()) { - for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) - break; - json &route = routes[i]; - json &target = route["target"]; - json &via = route["via"]; - if (target.is_string()) { - const InetAddress t(target.get()); - InetAddress v; - if (via.is_string()) v.fromString(via.get()); - if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); - *(reinterpret_cast(&(r->target))) = t; - if (v.ss_family == t.ss_family) - *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; - } - } - } - } + } else { - const bool noAutoAssignIps = _jB(member["noAutoAssignIps"],false); + char tmp[4096]; + Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + responseBody = tmp; + responseContentType = "application/json"; + return 200; - if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { - if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; - } - if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; - } } - bool haveManagedIpv4AutoAssignment = false; - bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count - json ipAssignments = member["ipAssignments"]; // we want to make a copy - if (ipAssignments.is_array()) { - for(unsigned long i=0;i(&(nc.routes[rk].target))->containsAddress(ip)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); - } +unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if (path.empty()) + return 404; - if (routedNetmaskBits > 0) { - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - ip.setPort(routedNetmaskBits); - nc.staticIps[nc.staticIpCount++] = ip; - } - if (ip.ss_family == AF_INET) - haveManagedIpv4AutoAssignment = true; - else if (ip.ss_family == AF_INET6) - haveManagedIpv6AutoAssignment = true; - } + json b; + try { + b = json::parse(body); + if (!b.is_object()) { + responseBody = "{ \"message\": \"body is not a JSON object\" }"; + responseContentType = "application/json"; + return 400; } - } else { - ipAssignments = json::array(); + } catch ( ... ) { + responseBody = "{ \"message\": \"body JSON is invalid\" }"; + responseContentType = "application/json"; + return 400; } + const uint64_t now = OSUtils::now(); - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!noAutoAssignIps) ) { - for(unsigned long p=0;((p s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { - // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. - xx[0] = Utils::hton(x[0]); - xx[1] = Utils::hton(x[1] + identity.address().toInt()); - } else { - // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway - Utils::getSecureRandom((void *)xx,16); - if ((e[0] > s[0])) - xx[0] %= (e[0] - s[0]); - else xx[0] = 0; - if ((e[1] > s[1])) - xx[1] %= (e[1] - s[1]); - else xx[1] = 0; - xx[0] = Utils::hton(x[0] + xx[0]); - xx[1] = Utils::hton(x[1] + xx[1]); - } + if ((path.size() >= 2)&&(path[1].length() == 16)) { + uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - InetAddress ip6((const void *)xx,16,0); + if (path.size() >= 3) { - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); - } + if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { + uint64_t address = Utils::hexStrToU64(path[3].c_str()); + char addrs[24]; + Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); - // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip6))) { - ipAssignments.push_back(ip6.toIpString()); - member["ipAssignments"] = ipAssignments; - ip6.setPort((unsigned int)routedNetmaskBits); - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) - nc.staticIps[nc.staticIpCount++] = ip6; - haveManagedIpv6AutoAssignment = true; - break; - } + json member; + { + Mutex::Lock _l(_db_m); + member = _db.get("network",nwids,"member",Address(address).toString(),0); } - } - } - } - } - - if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(_jB(v4AssignMode["zt"],false))) && (!haveManagedIpv4AutoAssignment) && (!noAutoAssignIps) ) { - for(unsigned long p=0;((p(&ipRangeStartIA)->sin_addr.s_addr)); - uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); - if ((ipRangeEnd < ipRangeStart)||(ipRangeStart == 0)) - continue; - uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; - - // Start with the LSB of the member's address - uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); - - for(uint32_t k=ipRangeStart,trialCount=0;((k<=ipRangeEnd)&&(trialCount < 1000));++k,++trialCount) { - uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; - ++ipTrialCounter; - if ((ip & 0x000000ff) == 0x000000ff) - continue; // don't allow addresses that end in .255 - - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = -1; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); - if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { - routedNetmaskBits = targetBits; - break; - } - } - } - - // If it's routed, then try to claim and assign it and if successful end loop - const InetAddress ip4(Utils::hton(ip),0); - if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { - ipAssignments.push_back(ip4.toIpString()); - member["ipAssignments"] = ipAssignments; - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); - v4ip->sin_family = AF_INET; - v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); - v4ip->sin_addr.s_addr = Utils::hton(ip); - } - haveManagedIpv4AutoAssignment = true; - break; - } - } - } - } - } - } - - CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); - if (com.sign(_signingId)) { - nc.com = com; - } else { - _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR); - return; - } - - if (member != origMember) { - member["lastModified"] = now; - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",identity.address().toString(),member); - } - - _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); -} - -unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if ((path.size() > 0)&&(path[0] == "network")) { - - if ((path.size() >= 2)&&(path[1].length() == 16)) { - const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - json network; - { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,0); - } - if (!network.size()) - return 404; - - if (path.size() >= 3) { - - if (path[2] == "member") { - - if (path.size() >= 4) { - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - json member; - { - Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString(),0); - } - if (!member.size()) - return 404; - - _addMemberNonPersistedFields(member,OSUtils::now()); - responseBody = member.dump(2); - responseContentType = "application/json"; - - return 200; - } else { - - Mutex::Lock _l(_db_m); - - responseBody = "{"; - std::string pfx(std::string("network/") + nwids + "member/"); - _db.filter(pfx,120000,[&responseBody](const std::string &n,const json &member) { - if (member.size() > 0) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(_jS(member["id"],"")); - responseBody.append("\":"); - responseBody.append(_jS(member["revision"],"0")); - } - return true; // never delete - }); - responseBody.push_back('}'); - responseContentType = "application/json"; - - return 200; - } - - } else if ((path[2] == "test")&&(path.size() >= 4)) { - - Mutex::Lock _l(_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); - if ((cte != _circuitTests.end())&&(cte->second.test)) { - - responseBody = "["; - responseBody.append(cte->second.jsonResults); - responseBody.push_back(']'); - responseContentType = "application/json"; - - return 200; - - } // else 404 - - } // else 404 - - } else { - - const uint64_t now = OSUtils::now(); - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); - _addNetworkNonPersistedFields(network,now,nmi); - responseBody = network.dump(2); - responseContentType = "application/json"; - return 200; - - } - } else if (path.size() == 1) { - - std::set networkIds; - { - Mutex::Lock _l(_db_m); - _db.filter("network/",120000,[&networkIds](const std::string &n,const json &obj) { - if (n.length() == (16 + 8)) - networkIds.insert(n.substr(8)); - return true; // do not delete - }); - } - - responseBody.push_back('['); - for(std::set::iterator i(networkIds.begin());i!=networkIds.end();++i) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\""); - } - responseBody.push_back(']'); - responseContentType = "application/json"; - return 200; - - } // else 404 - - } else { - - char tmp[4096]; - Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); - responseBody = tmp; - responseContentType = "application/json"; - return 200; - - } - - return 404; -} - -unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if (path.empty()) - return 404; - - json b; - try { - b = json::parse(body); - if (!b.is_object()) { - responseBody = "{ \"message\": \"body is not a JSON object\" }"; - responseContentType = "application/json"; - return 400; - } - } catch ( ... ) { - responseBody = "{ \"message\": \"body JSON is invalid\" }"; - responseContentType = "application/json"; - return 400; - } - const uint64_t now = OSUtils::now(); - - if (path[0] == "network") { - - if ((path.size() >= 2)&&(path[1].length() == 16)) { - uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - if (path.size() >= 3) { - - if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - uint64_t address = Utils::hexStrToU64(path[3].c_str()); - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); - - json member; - { - Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString(),0); - } - json origMember(member); // for detecting changes - _initMember(member); + json origMember(member); // for detecting changes + _initMember(member); try { if (b.count("activeBridge")) member["activeBridge"] = _jB(b["activeBridge"],false); @@ -1546,107 +1096,606 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( Mutex::Lock _l(_db_m); network = _db.get("network",nwids,0); } - if (!network.size()) - return 404; + if (!network.size()) + return 404; + + if (path.size() >= 3) { + if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + Mutex::Lock _l(_db_m); + + json member = _db.get("network",nwids,"member",Address(address).toString(),0); + _db.erase("network",nwids,"member",Address(address).toString()); + + if (!member.size()) + return 404; + responseBody = member.dump(2); + responseContentType = "application/json"; + return 200; + } + } else { + Mutex::Lock _l(_db_m); + + std::string pfx("network/"); pfx.append(nwids); + _db.filter(pfx,120000,[](const std::string &n,const json &obj) { + return false; // delete + }); + + Mutex::Lock _l2(_nmiCache_m); + _nmiCache.erase(nwid); + + responseBody = network.dump(2); + responseContentType = "application/json"; + return 200; + } + } // else 404 + + } // else 404 + + return 404; +} + +void EmbeddedNetworkController::threadMain() + throw() +{ + for(;;) { + _RQEntry *const qe = _queue.get(); + if (!qe) break; // enqueue a NULL to terminate threads + try { + _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); + } catch ( ... ) {} + delete qe; + } +} + +void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) +{ + char tmp[65535]; + EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); + + if (!test) + return; + if (!report) + return; + + Mutex::Lock _l(self->_circuitTests_m); + std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); + + if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? + self->_node->circuitTestEnd(test); + ::free((void *)test); + return; + } + + Utils::snprintf(tmp,sizeof(tmp), + "%s{\n" + "\t\"timestamp\": %llu," ZT_EOL_S + "\t\"testId\": \"%.16llx\"," ZT_EOL_S + "\t\"upstream\": \"%.10llx\"," ZT_EOL_S + "\t\"current\": \"%.10llx\"," ZT_EOL_S + "\t\"receivedTimestamp\": %llu," ZT_EOL_S + "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S + "\t\"flags\": %llu," ZT_EOL_S + "\t\"sourcePacketHopCount\": %u," ZT_EOL_S + "\t\"errorCode\": %u," ZT_EOL_S + "\t\"vendor\": %d," ZT_EOL_S + "\t\"protocolVersion\": %u," ZT_EOL_S + "\t\"majorVersion\": %u," ZT_EOL_S + "\t\"minorVersion\": %u," ZT_EOL_S + "\t\"revision\": %u," ZT_EOL_S + "\t\"platform\": %d," ZT_EOL_S + "\t\"architecture\": %d," ZT_EOL_S + "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S + "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S + "}", + ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), + (unsigned long long)report->timestamp, + (unsigned long long)test->testId, + (unsigned long long)report->upstream, + (unsigned long long)report->current, + (unsigned long long)OSUtils::now(), + (unsigned long long)report->sourcePacketId, + (unsigned long long)report->flags, + report->sourcePacketHopCount, + report->errorCode, + (int)report->vendor, + report->protocolVersion, + report->majorVersion, + report->minorVersion, + report->revision, + (int)report->platform, + (int)report->architecture, + reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), + reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); + + cte->second.jsonResults.append(tmp); +} + +void EmbeddedNetworkController::_request( + uint64_t nwid, + const InetAddress &fromAddr, + uint64_t requestPacketId, + const Identity &identity, + const Dictionary &metaData) +{ + if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) + return; + + const uint64_t now = OSUtils::now(); + + if (requestPacketId) { + Mutex::Lock _l(_lastRequestTime_m); + uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; + if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + return; + lrt = now; + } + + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + json network; + json member; + { + Mutex::Lock _l(_db_m); + network = _db.get("network",nwids,0); + member = _db.get("network",nwids,"member",identity.address().toString(),0); + } + + if (!network.size()) { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); + return; + } + + json origMember(member); // for detecting modification later + _initMember(member); + + { + std::string haveIdStr(_jS(member["identity"],"")); + if (haveIdStr.length() > 0) { + // If we already know this member's identity perform a full compare. This prevents + // a "collision" from being able to auth onto our network in place of an already + // known member. + try { + if (Identity(haveIdStr.c_str()) != identity) { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; + } + } catch ( ... ) { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; + } + } else { + // If we do not yet know this member's identity, learn it. + member["identity"] = identity.toString(false); + } + } + + // These are always the same, but make sure they are set + member["id"] = identity.address().toString(); + member["address"] = member["id"]; + member["nwid"] = nwids; + + // Determine whether and how member is authorized + const char *authorizedBy = (const char *)0; + if (_jB(member["authorized"],false)) { + authorizedBy = "memberIsAuthorized"; + } else if (!_jB(network["private"],true)) { + authorizedBy = "networkIsPublic"; + if (!member.count("authorized")) { + member["authorized"] = true; + json ah; + ah["a"] = true; + ah["by"] = authorizedBy; + ah["ts"] = now; + ah["ct"] = json(); + ah["c"] = json(); + member["authHistory"].push_back(ah); + member["lastModified"] = now; + json &revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + } + } else { + char presentedAuth[512]; + if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) { + presentedAuth[511] = (char)0; // sanity check + + // Check for bearer token presented by member + if ((strlen(presentedAuth) > 6)&&(!strncmp(presentedAuth,"token:",6))) { + const char *const presentedToken = presentedAuth + 6; + + json &authTokens = network["authTokens"]; + if (authTokens.is_array()) { + for(unsigned long i=0;i now))&&(tstr == presentedToken)) { + bool usable = (maxUses == 0); + if (!usable) { + uint64_t useCount = 0; + json &ahist = member["authHistory"]; + if (ahist.is_array()) { + for(unsigned long j=0;j= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) + break; + } + } + member["recentLog"] = recentLog; + + // Also only do this on real requests + member["lastRequestMetaData"] = metaData.data(); + } + + // If they are not authorized, STOP! + if (!authorizedBy) { + if (origMember != member) { + member["lastModified"] = now; + Mutex::Lock _l(_db_m); + _db.put("network",nwids,"member",identity.address().toString(),member); + } + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; + } + + // ------------------------------------------------------------------------- + // If we made it this far, they are authorized. + // ------------------------------------------------------------------------- + + NetworkConfig nc; + _NetworkMemberInfo nmi; + _getNetworkMemberInfo(now,nwid,nmi); + + // Compute credential TTL. This is the "moving window" for COM agreement and + // the global TTL for Capability and Tag objects. (The same value is used + // for both.) This is computed by reference to the last time we deauthorized + // a member, since within the time period since this event any temporal + // differences are not particularly relevant. + uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; + if (now > nmi.mostRecentDeauthTime) + credentialtmd += (now - nmi.mostRecentDeauthTime); + if (credentialtmd > ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) + credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; + + nc.networkId = nwid; + nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc.timestamp = now; + nc.credentialTimeMaxDelta = credentialtmd; + nc.revision = _jI(network["revision"],0ULL); + nc.issuedTo = identity.address(); + if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (_jB(network["allowPassiveBridging"],false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); + nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); + + for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) { + nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + } + + json &v4AssignMode = network["v4AssignMode"]; + json &v6AssignMode = network["v6AssignMode"]; + json &ipAssignmentPools = network["ipAssignmentPools"]; + json &routes = network["routes"]; + json &rules = network["rules"]; + json &capabilities = network["capabilities"]; + json &memberCapabilities = member["capabilities"]; + json &memberTags = member["tags"]; + + if (metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,0) <= 0) { + // Old versions with no rules engine support get an allow everything rule. + // Since rules are enforced bidirectionally, newer versions *will* still + // enforce rules on the inbound side. + nc.ruleCount = 1; + nc.rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; + } else { + if (rules.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + break; + if (_parseRule(rules[i],nc.rules[nc.ruleCount])) + ++nc.ruleCount; + } + } + + if ((memberCapabilities.is_array())&&(memberCapabilities.size() > 0)&&(capabilities.is_array())) { + std::map< uint64_t,json * > capsById; + for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { + ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; + unsigned int caprc = 0; + json &caprj = (*cap)["rules"]; + if ((caprj.is_array())&&(caprj.size() > 0)) { + for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) + break; + if (_parseRule(caprj[j],capr[caprc])) + ++caprc; + } + } + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); + if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) + ++nc.capabilityCount; + if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + break; + } + } + } + + if (memberTags.is_array()) { + std::map< uint32_t,uint32_t > tagsById; + for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { + if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + break; + nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); + if (nc.tags[nc.tagCount].sign(_signingId)) + ++nc.tagCount; + } + } + } + + if (routes.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) + break; + json &route = routes[i]; + json &target = route["target"]; + json &via = route["via"]; + if (target.is_string()) { + const InetAddress t(target.get()); + InetAddress v; + if (via.is_string()) v.fromString(via.get()); + if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { + ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + *(reinterpret_cast(&(r->target))) = t; + if (v.ss_family == t.ss_family) + *(reinterpret_cast(&(r->via))) = v; + ++nc.routeCount; + } + } + } + } - if (path.size() >= 3) { - if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + const bool noAutoAssignIps = _jB(member["noAutoAssignIps"],false); - Mutex::Lock _l(_db_m); + if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { + if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); + nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + } + if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); + nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + } + } - json member = _db.get("network",nwids,"member",Address(address).toString(),0); - _db.erase("network",nwids,"member",Address(address).toString()); + bool haveManagedIpv4AutoAssignment = false; + bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count + json ipAssignments = member["ipAssignments"]; // we want to make a copy + if (ipAssignments.is_array()) { + for(unsigned long i=0;i(&(nc.routes[rk].target))->containsAddress(ip)) ) + routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + } + + if (routedNetmaskBits > 0) { + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + ip.setPort(routedNetmaskBits); + nc.staticIps[nc.staticIpCount++] = ip; } - } else { - Mutex::Lock _l(_db_m); + if (ip.ss_family == AF_INET) + haveManagedIpv4AutoAssignment = true; + else if (ip.ss_family == AF_INET6) + haveManagedIpv6AutoAssignment = true; + } + } + } else { + ipAssignments = json::array(); + } - std::string pfx("network/"); pfx.append(nwids); - _db.filter(pfx,120000,[](const std::string &n,const json &obj) { - return false; // delete - }); + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!noAutoAssignIps) ) { + for(unsigned long p=0;((p s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { + // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. + xx[0] = Utils::hton(x[0]); + xx[1] = Utils::hton(x[1] + identity.address().toInt()); + } else { + // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway + Utils::getSecureRandom((void *)xx,16); + if ((e[0] > s[0])) + xx[0] %= (e[0] - s[0]); + else xx[0] = 0; + if ((e[1] > s[1])) + xx[1] %= (e[1] - s[1]); + else xx[1] = 0; + xx[0] = Utils::hton(x[0] + xx[0]); + xx[1] = Utils::hton(x[1] + xx[1]); + } - responseBody = network.dump(2); - responseContentType = "application/json"; - return 200; + InetAddress ip6((const void *)xx,16,0); + + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = 0; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) + routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + } + + // If it's routed, then try to claim and assign it and if successful end loop + if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip6))) { + ipAssignments.push_back(ip6.toIpString()); + member["ipAssignments"] = ipAssignments; + ip6.setPort((unsigned int)routedNetmaskBits); + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) + nc.staticIps[nc.staticIpCount++] = ip6; + haveManagedIpv6AutoAssignment = true; + break; + } + } + } } - } // else 404 + } + } - } // else 404 + if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(_jB(v4AssignMode["zt"],false))) && (!haveManagedIpv4AutoAssignment) && (!noAutoAssignIps) ) { + for(unsigned long p=0;((p(&ipRangeStartIA)->sin_addr.s_addr)); + uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); + if ((ipRangeEnd < ipRangeStart)||(ipRangeStart == 0)) + continue; + uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; - return 404; -} + // Start with the LSB of the member's address + uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); -void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) -{ - char tmp[65535]; - EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); + for(uint32_t k=ipRangeStart,trialCount=0;((k<=ipRangeEnd)&&(trialCount < 1000));++k,++trialCount) { + uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; + ++ipTrialCounter; + if ((ip & 0x000000ff) == 0x000000ff) + continue; // don't allow addresses that end in .255 - if (!test) - return; - if (!report) - return; + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = -1; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); + int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); + if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { + routedNetmaskBits = targetBits; + break; + } + } + } - Mutex::Lock _l(self->_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); + // If it's routed, then try to claim and assign it and if successful end loop + const InetAddress ip4(Utils::hton(ip),0); + if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { + ipAssignments.push_back(ip4.toIpString()); + member["ipAssignments"] = ipAssignments; + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); + v4ip->sin_family = AF_INET; + v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); + v4ip->sin_addr.s_addr = Utils::hton(ip); + } + haveManagedIpv4AutoAssignment = true; + break; + } + } + } + } + } + } - if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? - self->_node->circuitTestEnd(test); - ::free((void *)test); + CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); + if (com.sign(_signingId)) { + nc.com = com; + } else { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR); return; } - Utils::snprintf(tmp,sizeof(tmp), - "%s{\n" - "\t\"timestamp\": %llu," ZT_EOL_S - "\t\"testId\": \"%.16llx\"," ZT_EOL_S - "\t\"upstream\": \"%.10llx\"," ZT_EOL_S - "\t\"current\": \"%.10llx\"," ZT_EOL_S - "\t\"receivedTimestamp\": %llu," ZT_EOL_S - "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S - "\t\"flags\": %llu," ZT_EOL_S - "\t\"sourcePacketHopCount\": %u," ZT_EOL_S - "\t\"errorCode\": %u," ZT_EOL_S - "\t\"vendor\": %d," ZT_EOL_S - "\t\"protocolVersion\": %u," ZT_EOL_S - "\t\"majorVersion\": %u," ZT_EOL_S - "\t\"minorVersion\": %u," ZT_EOL_S - "\t\"revision\": %u," ZT_EOL_S - "\t\"platform\": %d," ZT_EOL_S - "\t\"architecture\": %d," ZT_EOL_S - "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S - "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S - "}", - ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), - (unsigned long long)report->timestamp, - (unsigned long long)test->testId, - (unsigned long long)report->upstream, - (unsigned long long)report->current, - (unsigned long long)OSUtils::now(), - (unsigned long long)report->sourcePacketId, - (unsigned long long)report->flags, - report->sourcePacketHopCount, - report->errorCode, - (int)report->vendor, - report->protocolVersion, - report->majorVersion, - report->minorVersion, - report->revision, - (int)report->platform, - (int)report->architecture, - reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), - reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); + if (member != origMember) { + member["lastModified"] = now; + Mutex::Lock _l(_db_m); + _db.put("network",nwids,"member",identity.address().toString(),member); + } - cte->second.jsonResults.append(tmp); + _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 620ef3e3..0169b1d3 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -37,11 +37,15 @@ #include "../osdep/OSUtils.hpp" #include "../osdep/Thread.hpp" +#include "../osdep/BlockingQueue.hpp" #include "../ext/json/json.hpp" #include "JSONDB.hpp" +// Number of background threads to start -- not actually started until needed +#define ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT 2 + namespace ZeroTier { class Node; @@ -83,8 +87,31 @@ public: std::string &responseBody, std::string &responseContentType); + void threadMain() + throw(); + private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); + void _request( + uint64_t nwid, + const InetAddress &fromAddr, + uint64_t requestPacketId, + const Identity &identity, + const Dictionary &metaData); + + struct _RQEntry + { + uint64_t nwid; + uint64_t requestPacketId; + InetAddress fromAddr; + Identity identity; + Dictionary metaData; + }; + BlockingQueue<_RQEntry *> _queue; + + Thread _threads[ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT]; + bool _threadsStarted; + Mutex _threads_m; // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places // This does lock _networkMemberCache_m diff --git a/osdep/BlockingQueue.hpp b/osdep/BlockingQueue.hpp new file mode 100644 index 00000000..6172f4da --- /dev/null +++ b/osdep/BlockingQueue.hpp @@ -0,0 +1,64 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_BLOCKINGQUEUE_HPP +#define ZT_BLOCKINGQUEUE_HPP + +#include +#include +#include + +namespace ZeroTier { + +/** + * Simple C++11 thread-safe queue + * + * Do not use in node/ since we have not gone C++11 there yet. + */ +template +class BlockingQueue +{ +public: + BlockingQueue(void) {} + + inline void post(T t) + { + std::lock_guard lock(m); + q.push(t); + c.notify_one(); + } + + inline T get(void) + { + std::unique_lock lock(m); + while(q.empty()) + c.wait(lock); + T val = q.front(); + q.pop(); + return val; + } + +private: + std::queue q; + mutable std::mutex m; + std::condition_variable c; +}; + +} // namespace ZeroTier + +#endif -- cgit v1.2.3 From 15c6e2ec70b4c43e04e1d79d9743c535c6a530a0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 15 Nov 2016 14:06:25 -0800 Subject: Fix member deauthorization time threshold bug. --- controller/EmbeddedNetworkController.cpp | 50 +++++++++++++++++--------------- controller/EmbeddedNetworkController.hpp | 2 ++ 2 files changed, 28 insertions(+), 24 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7f885b4e..b2ca732a 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -697,6 +697,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( const bool newAuth = _jB(b["authorized"],false); if (newAuth != _jB(member["authorized"],false)) { member["authorized"] = newAuth; + member[((newAuth) ? "lastAuthorizedTime" : "lastDeauthorizedTime")] = now; + json ah; ah["a"] = newAuth; ah["by"] = "api"; @@ -1278,23 +1280,14 @@ void EmbeddedNetworkController::_request( // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; + bool autoAuthorized = false; + json autoAuthCredentialType,autoAuthCredential; if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; } else if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; - if (!member.count("authorized")) { - member["authorized"] = true; - json ah; - ah["a"] = true; - ah["by"] = authorizedBy; - ah["ts"] = now; - ah["ct"] = json(); - ah["c"] = json(); - member["authHistory"].push_back(ah); - member["lastModified"] = now; - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - } + if (!member.count("authorized")) + autoAuthorized = true; } else { char presentedAuth[512]; if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) { @@ -1329,17 +1322,9 @@ void EmbeddedNetworkController::_request( } if (usable) { authorizedBy = "token"; - member["authorized"] = true; - json ah; - ah["a"] = true; - ah["by"] = authorizedBy; - ah["ts"] = now; - ah["ct"] = "token"; - ah["c"] = tstr; - member["authHistory"].push_back(ah); - member["lastModified"] = now; - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + autoAuthorized = true; + autoAuthCredentialType = "token"; + autoAuthCredential = tstr; } } } @@ -1349,6 +1334,23 @@ void EmbeddedNetworkController::_request( } } + // If we auto-authorized, update member record + if ((autoAuthorized)&&(authorizedBy)) { + member["authorized"] = true; + member["lastAuthorizedTime"] = now; + + json ah; + ah["a"] = true; + ah["by"] = authorizedBy; + ah["ts"] = now; + ah["ct"] = autoAuthCredentialType; + ah["c"] = autoAuthCredential; + member["authHistory"].push_back(ah); + + json &revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + } + // Log this request if (requestPacketId) { // only log if this is a request, not for generated pushes json rlEntry = json::object(); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 0169b1d3..cde6522d 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -145,6 +145,8 @@ private: if (!member.count("creationTime")) member["creationTime"] = OSUtils::now(); if (!member.count("noAutoAssignIps")) member["noAutoAssignIps"] = false; if (!member.count("revision")) member["revision"] = 0ULL; + if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL; + if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL; member["objtype"] = "member"; } inline void _initNetwork(nlohmann::json &network) -- cgit v1.2.3 From 07b2a3818ca389f45bff33606f729baf0260fdd9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 15 Nov 2016 14:26:05 -0800 Subject: Fix TTL scaling in cert. --- controller/EmbeddedNetworkController.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b2ca732a..b78f847e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1399,16 +1399,18 @@ void EmbeddedNetworkController::_request( _NetworkMemberInfo nmi; _getNetworkMemberInfo(now,nwid,nmi); - // Compute credential TTL. This is the "moving window" for COM agreement and - // the global TTL for Capability and Tag objects. (The same value is used - // for both.) This is computed by reference to the last time we deauthorized - // a member, since within the time period since this event any temporal - // differences are not particularly relevant. - uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; - if (now > nmi.mostRecentDeauthTime) - credentialtmd += (now - nmi.mostRecentDeauthTime); - if (credentialtmd > ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) - credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; + uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; + if (now > nmi.mostRecentDeauthTime) { + // If we recently de-authorized a member, shrink credential TTL/max delta to + // be below the threshold required to exclude it. Cap this to a min/max to + // prevent jitter or absurdly large values. + const uint64_t deauthWindow = now - nmi.mostRecentDeauthTime; + if (deauthWindow < ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA) { + credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; + } else if (deauthWindow < (ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA + 5000ULL)) { + credentialtmd = deauthWindow - 5000ULL; + } + } nc.networkId = nwid; nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; -- cgit v1.2.3 From 25f9c294dc677576ded51025b2c7e6397bdc11c0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 13:01:45 -0800 Subject: Small bug fix and warning removal. --- controller/EmbeddedNetworkController.cpp | 12 +++++++----- node/InetAddress.hpp | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b78f847e..74937dd8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1776,11 +1776,13 @@ void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,con std::map,uint64_t>::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); } - Dictionary *metaData = new Dictionary(mdstr.c_str()); - try { - this->request(nwid,InetAddress(),0,id,*metaData); - } catch ( ... ) {} - delete metaData; + if (online) { + Dictionary *metaData = new Dictionary(mdstr.c_str()); + try { + this->request(nwid,InetAddress(),0,id,*metaData); + } catch ( ... ) {} + delete metaData; + } } } catch ( ... ) {} } diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 1dff710d..c37fa621 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -427,7 +427,7 @@ struct InetAddress : public sockaddr_storage } else { unsigned long tmp = reinterpret_cast(this)->sin6_port; const uint8_t *a = reinterpret_cast(this); - for(long i=0;i(&tmp)[i % sizeof(tmp)] ^= a[i]; return tmp; } -- cgit v1.2.3 From ccdd4ffda70a35219fae1cdce1306f3d4ebd8dc9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 15:49:28 -0800 Subject: Move split() to OSUtils since it is not used in core. --- controller/EmbeddedNetworkController.cpp | 2 +- controller/JSONDB.cpp | 2 +- node/Utils.cpp | 44 -------------------------------- node/Utils.hpp | 11 -------- one.cpp | 4 +-- osdep/OSUtils.cpp | 44 ++++++++++++++++++++++++++++++++ osdep/OSUtils.hpp | 11 ++++++++ service/ControlPlane.cpp | 4 +-- 8 files changed, 61 insertions(+), 61 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 74937dd8..974936e4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -889,7 +889,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &v6m = b["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (v6m.is_string()) { // backward compatibility - std::vector v6ms(Utils::split(_jS(v6m,"").c_str(),",","","")); + std::vector v6ms(OSUtils::split(_jS(v6m,"").c_str(),",","","")); std::sort(v6ms.begin(),v6ms.end()); v6ms.erase(std::unique(v6ms.begin(),v6ms.end()),v6ms.end()); nv6m["rfc4193"] = false; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 6adc8a66..a9e9d05b 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -144,7 +144,7 @@ bool JSONDB::_isValidObjectName(const std::string &n) std::string JSONDB::_genPath(const std::string &n,bool create) { - std::vector pt(Utils::split(n.c_str(),"/","","")); + std::vector pt(OSUtils::split(n.c_str(),"/","","")); if (pt.size() == 0) return std::string(); diff --git a/node/Utils.cpp b/node/Utils.cpp index 9d67fc22..215e3b3e 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -218,50 +218,6 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) s20.encrypt12(buf,buf,bytes); } -std::vector Utils::split(const char *s,const char *const sep,const char *esc,const char *quot) -{ - std::vector fields; - std::string buf; - - if (!esc) - esc = ""; - if (!quot) - quot = ""; - - bool escapeState = false; - char quoteState = 0; - while (*s) { - if (escapeState) { - escapeState = false; - buf.push_back(*s); - } else if (quoteState) { - if (*s == quoteState) { - quoteState = 0; - fields.push_back(buf); - buf.clear(); - } else buf.push_back(*s); - } else { - const char *quotTmp; - if (strchr(esc,*s)) - escapeState = true; - else if ((buf.size() <= 0)&&((quotTmp = strchr(quot,*s)))) - quoteState = *quotTmp; - else if (strchr(sep,*s)) { - if (buf.size() > 0) { - fields.push_back(buf); - buf.clear(); - } // else skip runs of seperators - } else buf.push_back(*s); - } - ++s; - } - - if (buf.size()) - fields.push_back(buf); - - return fields; -} - bool Utils::scopy(char *dest,unsigned int len,const char *src) { if (!len) diff --git a/node/Utils.hpp b/node/Utils.hpp index cfe56501..48c43da3 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -111,17 +111,6 @@ public: */ static void getSecureRandom(void *buf,unsigned int bytes); - /** - * Split a string by delimiter, with optional escape and quote characters - * - * @param s String to split - * @param sep One or more separators - * @param esc Zero or more escape characters - * @param quot Zero or more quote characters - * @return Vector of tokens - */ - static std::vector split(const char *s,const char *const sep,const char *esc,const char *quot); - /** * Tokenize a string (alias for strtok_r or strtok_s depending on platform) * diff --git a/one.cpp b/one.cpp index 51cda0c7..3dad2ed9 100644 --- a/one.cpp +++ b/one.cpp @@ -696,7 +696,7 @@ static int idtool(int argc,char **argv) CertificateOfMembership com; for(int a=3;a params(Utils::split(argv[a],",","","")); + std::vector params(OSUtils::split(argv[a],",","","")); if (params.size() == 3) { uint64_t qId = Utils::hexStrToU64(params[0].c_str()); uint64_t qValue = Utils::hexStrToU64(params[1].c_str()); @@ -1084,7 +1084,7 @@ int main(int argc,char **argv) fprintf(stderr,"%s: no home path specified and no platform default available" ZT_EOL_S,argv[0]); return 1; } else { - std::vector hpsp(Utils::split(homeDir.c_str(),ZT_PATH_SEPARATOR_S,"","")); + std::vector hpsp(OSUtils::split(homeDir.c_str(),ZT_PATH_SEPARATOR_S,"","")); std::string ptmp; if (homeDir[0] == ZT_PATH_SEPARATOR) ptmp.push_back(ZT_PATH_SEPARATOR); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 4a81625b..65704fa3 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -315,6 +315,50 @@ bool OSUtils::writeFile(const char *path,const void *buf,unsigned int len) return false; } +std::vector OSUtils::split(const char *s,const char *const sep,const char *esc,const char *quot) +{ + std::vector fields; + std::string buf; + + if (!esc) + esc = ""; + if (!quot) + quot = ""; + + bool escapeState = false; + char quoteState = 0; + while (*s) { + if (escapeState) { + escapeState = false; + buf.push_back(*s); + } else if (quoteState) { + if (*s == quoteState) { + quoteState = 0; + fields.push_back(buf); + buf.clear(); + } else buf.push_back(*s); + } else { + const char *quotTmp; + if (strchr(esc,*s)) + escapeState = true; + else if ((buf.size() <= 0)&&((quotTmp = strchr(quot,*s)))) + quoteState = *quotTmp; + else if (strchr(sep,*s)) { + if (buf.size() > 0) { + fields.push_back(buf); + buf.clear(); + } // else skip runs of seperators + } else buf.push_back(*s); + } + ++s; + } + + if (buf.size()) + fields.push_back(buf); + + return fields; +} + std::string OSUtils::platformDefaultHomePath() { #ifdef __UNIX_LIKE__ diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index c481780b..c73f0bd3 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -236,6 +236,17 @@ public: */ static bool writeFile(const char *path,const void *buf,unsigned int len); + /** + * Split a string by delimiter, with optional escape and quote characters + * + * @param s String to split + * @param sep One or more separators + * @param esc Zero or more escape characters + * @param quot Zero or more quote characters + * @return Vector of tokens + */ + static std::vector split(const char *s,const char *const sep,const char *esc,const char *quot); + /** * Write a block of data to disk, replacing any current file contents * diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 150bba1b..07304fc9 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -266,7 +266,7 @@ unsigned int ControlPlane::handleRequest( { char json[8194]; unsigned int scode = 404; - std::vector ps(Utils::split(path.c_str(),"/","","")); + std::vector ps(OSUtils::split(path.c_str(),"/","","")); std::map urlArgs; Mutex::Lock _l(_lock); @@ -279,7 +279,7 @@ unsigned int ControlPlane::handleRequest( if (qpos != std::string::npos) { std::string args(ps[ps.size() - 1].substr(qpos + 1)); ps[ps.size() - 1] = ps[ps.size() - 1].substr(0,qpos); - std::vector asplit(Utils::split(args.c_str(),"&","","")); + std::vector asplit(OSUtils::split(args.c_str(),"&","","")); for(std::vector::iterator a(asplit.begin());a!=asplit.end();++a) { std::size_t eqpos = a->find('='); if (eqpos == std::string::npos) -- cgit v1.2.3 From a54c2b438c0abc574039055ecfc70dd829e2cade Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 15 Dec 2016 15:08:47 -0800 Subject: Basic support for streaming of changes via stdout from controller. --- controller/EmbeddedNetworkController.cpp | 4 ++-- controller/EmbeddedNetworkController.hpp | 7 ++++++- controller/JSONDB.cpp | 19 +++++++++++++++---- controller/JSONDB.hpp | 5 ++++- service/OneService.cpp | 2 +- 5 files changed, 28 insertions(+), 9 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 974936e4..df20d4ce 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -458,9 +458,9 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return false; } -EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : +EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath,FILE *feed) : _threadsStarted(false), - _db(dbPath), + _db(dbPath,feed), _node(node) { OSUtils::mkdir(dbPath); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index cde6522d..37c067ff 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -53,7 +53,12 @@ class Node; class EmbeddedNetworkController : public NetworkController { public: - EmbeddedNetworkController(Node *node,const char *dbPath); + /** + * @param node Parent node + * @param dbPath Path to store data + * @param feed FILE to send feed of all data and changes to (zero-delimited JSON objects) or NULL for none + */ + EmbeddedNetworkController(Node *node,const char *dbPath,FILE *feed); virtual ~EmbeddedNetworkController(); virtual void init(const Identity &signingId,Sender *sender); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index a9e9d05b..dfea9dd1 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -27,14 +27,17 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) if (!_isValidObjectName(n)) return false; - std::string path(_genPath(n,true)); + const std::string path(_genPath(n,true)); if (!path.length()) return false; - std::string buf(obj.dump(2)); + const std::string buf(obj.dump(2)); if (!OSUtils::writeFile(path.c_str(),buf)) return false; + if (_feed) + fwrite(buf.c_str(),buf.length()+1,1,_feed); + _E &e = _db[n]; e.obj = obj; e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); @@ -55,7 +58,8 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe if (e != _db.end()) { if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck) return e->second.obj; - std::string path(_genPath(n,false)); + + const std::string path(_genPath(n,false)); if (!path.length()) // sanity check return _EMPTY_JSON; @@ -68,13 +72,16 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe e->second.obj = nlohmann::json::parse(buf); e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP e->second.lastCheck = now; + + if (_feed) + fwrite(buf.c_str(),buf.length()+1,1,_feed); // it changed, so send to feed (also sends all objects on startup, which we want for Central) } catch ( ... ) {} // parse errors result in "holding pattern" behavior } } return e->second.obj; } else { - std::string path(_genPath(n,false)); + const std::string path(_genPath(n,false)); if (!path.length()) return _EMPTY_JSON; @@ -87,10 +94,14 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe e2.obj = nlohmann::json::parse(buf); } catch ( ... ) { e2.obj = _EMPTY_JSON; + buf = "{}"; } e2.lastModifiedOnDisk = lm; e2.lastCheck = now; + if (_feed) + fwrite(buf.c_str(),buf.length()+1,1,_feed); + return e2.obj; } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index bd1ae5a5..40113655 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -19,6 +19,7 @@ #ifndef ZT_JSONDB_HPP #define ZT_JSONDB_HPP +#include #include #include @@ -41,7 +42,8 @@ namespace ZeroTier { class JSONDB { public: - JSONDB(const std::string &basePath) : + JSONDB(const std::string &basePath,FILE *feed) : + _feed(feed), _basePath(basePath) { _reload(_basePath); @@ -106,6 +108,7 @@ private: inline bool operator!=(const _E &e) const { return (obj != e.obj); } }; + FILE *_feed; std::string _basePath; std::map _db; }; diff --git a/service/OneService.cpp b/service/OneService.cpp index 2e256307..3a7edacb 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -875,7 +875,7 @@ public: for(int i=0;i<3;++i) _portsBE[i] = Utils::hton((uint16_t)_ports[i]); - _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str()); + _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(FILE *)0); _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER -- cgit v1.2.3 From fe530548bbc8d4d0e274f814718fad579a012812 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 22 Dec 2016 16:57:45 -0800 Subject: Fix MATCH_RANDOM in controller. --- controller/EmbeddedNetworkController.cpp | 1 + node/Network.cpp | 1 + 2 files changed, 2 insertions(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index df20d4ce..01a7152c 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -428,6 +428,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_RANDOM") { rule.t |= ZT_NETWORK_RULE_MATCH_RANDOM; rule.v.randomProbability = (uint32_t)(_jI(r["probability"],0ULL) & 0xffffffffULL); + return true; } else if (t == "MATCH_TAGS_DIFFERENCE") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); diff --git a/node/Network.cpp b/node/Network.cpp index e8b103ba..2488bea2 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -111,6 +111,7 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b ); if (msg) printf(" + (%s)" ZT_EOL_S,msg); + fflush(stdout); } #else #define FILTER_TRACE(f,...) {} -- cgit v1.2.3 From 0d066e3b08b723dc5fe3630fbf6a6d2a5a3f0baf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 22 Dec 2016 18:26:43 -0800 Subject: Fix JSON parse bug in REDIRECT target. --- controller/EmbeddedNetworkController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 01a7152c..ee243777 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -310,7 +310,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); return true; } else if (t == "ACTION_DEBUG_LOG") { -- cgit v1.2.3 From bf2b9e3692c1a20dd7af3d8bfca4b6c591cd2214 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 22 Dec 2016 18:52:34 -0800 Subject: Auto-authorize new members on public networks properly. --- controller/EmbeddedNetworkController.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index ee243777..bed76df7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1287,7 +1287,8 @@ void EmbeddedNetworkController::_request( authorizedBy = "memberIsAuthorized"; } else if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; - if (!member.count("authorized")) + json &ahist = member["authHistory"]; + if ((!ahist.is_array())||(ahist.size() == 0)) autoAuthorized = true; } else { char presentedAuth[512]; -- cgit v1.2.3 From a064e19b8a78d3809e9f80fba010e0f53197c1a2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 10 Jan 2017 13:51:10 -0800 Subject: Refactor some JSON stuff for performance, and fix a build error. --- controller/EmbeddedNetworkController.cpp | 261 +++++++++++++------------------ controller/JSONDB.cpp | 6 +- one.cpp | 3 - osdep/OSUtils.cpp | 58 +++++++ osdep/OSUtils.hpp | 8 + service/ControlPlane.cpp | 2 +- service/OneService.cpp | 68 ++------ 7 files changed, 188 insertions(+), 218 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index bed76df7..3c13e7ce 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -63,53 +63,6 @@ using json = nlohmann::json; namespace ZeroTier { -// Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible -static uint64_t _jI(const json &jv,const uint64_t dfl) -{ - if (jv.is_number()) { - return (uint64_t)jv; - } else if (jv.is_string()) { - std::string s = jv; - return Utils::strToU64(s.c_str()); - } else if (jv.is_boolean()) { - return ((bool)jv ? 1ULL : 0ULL); - } - return dfl; -} -static bool _jB(const json &jv,const bool dfl) -{ - if (jv.is_boolean()) { - return (bool)jv; - } else if (jv.is_number()) { - return ((uint64_t)jv > 0ULL); - } else if (jv.is_string()) { - std::string s = jv; - if (s.length() > 0) { - switch(s[0]) { - case 't': - case 'T': - case '1': - return true; - } - } - return false; - } - return dfl; -} -static std::string _jS(const json &jv,const char *dfl) -{ - if (jv.is_string()) { - return jv; - } else if (jv.is_number()) { - char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); - return tmp; - } else if (jv.is_boolean()) { - return ((bool)jv ? std::string("1") : std::string("0")); - } - return std::string((dfl) ? dfl : ""); -} - static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; @@ -281,13 +234,13 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) if (!r.is_object()) return false; - const std::string t(_jS(r["type"],"")); + const std::string t(OSUtils::jsonString(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); - if (_jB(r["not"],false)) + if (OSUtils::jsonBool(r["not"],false)) rule.t = 0x80; else rule.t = 0x00; - if (_jB(r["or"],false)) + if (OSUtils::jsonBool(r["or"],false)) rule.t |= 0x40; if (t == "ACTION_DROP") { @@ -298,115 +251,115 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; - rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); - rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); + rule.v.fwd.address = Utils::hexStrToU64(OSUtils::jsonString(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(OSUtils::jsonInt(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(OSUtils::jsonInt(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "ACTION_WATCH") { rule.t |= ZT_NETWORK_RULE_ACTION_WATCH; - rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; - rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); - rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); + rule.v.fwd.address = Utils::hexStrToU64(OSUtils::jsonString(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(OSUtils::jsonInt(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(OSUtils::jsonInt(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; - rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.address = Utils::hexStrToU64(OSUtils::jsonString(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(OSUtils::jsonInt(r["flags"],0ULL) & 0xffffffffULL); return true; } else if (t == "ACTION_DEBUG_LOG") { rule.t |= ZT_NETWORK_RULE_ACTION_DEBUG_LOG; return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(OSUtils::jsonString(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(OSUtils::jsonString(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_VLAN_ID") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; - rule.v.vlanId = (uint16_t)(_jI(r["vlanId"],0ULL) & 0xffffULL); + rule.v.vlanId = (uint16_t)(OSUtils::jsonInt(r["vlanId"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_VLAN_PCP") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; - rule.v.vlanPcp = (uint8_t)(_jI(r["vlanPcp"],0ULL) & 0xffULL); + rule.v.vlanPcp = (uint8_t)(OSUtils::jsonInt(r["vlanPcp"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_VLAN_DEI") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; - rule.v.vlanDei = (uint8_t)(_jI(r["vlanDei"],0ULL) & 0xffULL); + rule.v.vlanDei = (uint8_t)(OSUtils::jsonInt(r["vlanDei"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_MAC_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; - const std::string mac(_jS(r["mac"],"0")); + const std::string mac(OSUtils::jsonString(r["mac"],"0")); Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); return true; } else if (t == "MATCH_MAC_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; - const std::string mac(_jS(r["mac"],"0")); + const std::string mac(OSUtils::jsonString(r["mac"],"0")); Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); return true; } else if (t == "MATCH_IPV4_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; - InetAddress ip(_jS(r["ip"],"0.0.0.0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0")); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV4_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; - InetAddress ip(_jS(r["ip"],"0.0.0.0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0")); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV6_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; - InetAddress ip(_jS(r["ip"],"::0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"::0")); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IPV6_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; - InetAddress ip(_jS(r["ip"],"::0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"::0")); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IP_TOS") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; - rule.v.ipTos = (uint8_t)(_jI(r["ipTos"],0ULL) & 0xffULL); + rule.v.ipTos = (uint8_t)(OSUtils::jsonInt(r["ipTos"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_IP_PROTOCOL") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; - rule.v.ipProtocol = (uint8_t)(_jI(r["ipProtocol"],0ULL) & 0xffULL); + rule.v.ipProtocol = (uint8_t)(OSUtils::jsonInt(r["ipProtocol"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_ETHERTYPE") { rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; - rule.v.etherType = (uint16_t)(_jI(r["etherType"],0ULL) & 0xffffULL); + rule.v.etherType = (uint16_t)(OSUtils::jsonInt(r["etherType"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_ICMP") { rule.t |= ZT_NETWORK_RULE_MATCH_ICMP; - rule.v.icmp.type = (uint8_t)(_jI(r["icmpType"],0ULL) & 0xffULL); + rule.v.icmp.type = (uint8_t)(OSUtils::jsonInt(r["icmpType"],0ULL) & 0xffULL); json &code = r["icmpCode"]; if (code.is_null()) { rule.v.icmp.code = 0; rule.v.icmp.flags = 0x00; } else { - rule.v.icmp.code = (uint8_t)(_jI(code,0ULL) & 0xffULL); + rule.v.icmp.code = (uint8_t)(OSUtils::jsonInt(code,0ULL) & 0xffULL); rule.v.icmp.flags = 0x01; } return true; } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; - rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); + rule.v.port[0] = (uint16_t)(OSUtils::jsonInt(r["start"],0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(OSUtils::jsonInt(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_IP_DEST_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; - rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); + rule.v.port[0] = (uint16_t)(OSUtils::jsonInt(r["start"],0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(OSUtils::jsonInt(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; @@ -422,37 +375,37 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_FRAME_SIZE_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); + rule.v.frameSize[0] = (uint16_t)(OSUtils::jsonInt(r["start"],0ULL) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(OSUtils::jsonInt(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; } else if (t == "MATCH_RANDOM") { rule.t |= ZT_NETWORK_RULE_MATCH_RANDOM; - rule.v.randomProbability = (uint32_t)(_jI(r["probability"],0ULL) & 0xffffffffULL); + rule.v.randomProbability = (uint32_t)(OSUtils::jsonInt(r["probability"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_DIFFERENCE") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; - rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_EQUAL") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_EQUAL; - rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); return true; } @@ -552,7 +505,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( return 404; _addMemberNonPersistedFields(member,OSUtils::now()); - responseBody = member.dump(2); + responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; return 200; @@ -565,9 +518,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( _db.filter(pfx,120000,[&responseBody](const std::string &n,const json &member) { if (member.size() > 0) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(_jS(member["id"],"")); + responseBody.append(OSUtils::jsonString(member["id"],"")); responseBody.append("\":"); - responseBody.append(_jS(member["revision"],"0")); + responseBody.append(OSUtils::jsonString(member["revision"],"0")); } return true; // never delete }); @@ -600,7 +553,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( _NetworkMemberInfo nmi; _getNetworkMemberInfo(now,nwid,nmi); _addNetworkNonPersistedFields(network,now,nmi); - responseBody = network.dump(2); + responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; return 200; @@ -655,7 +608,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json b; try { - b = json::parse(body); + b = OSUtils::jsonParse(body); if (!b.is_object()) { responseBody = "{ \"message\": \"body is not a JSON object\" }"; responseContentType = "application/json"; @@ -691,12 +644,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _initMember(member); try { - if (b.count("activeBridge")) member["activeBridge"] = _jB(b["activeBridge"],false); - if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = _jB(b["noAutoAssignIps"],false); + if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"],false); + if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"],false); if (b.count("authorized")) { - const bool newAuth = _jB(b["authorized"],false); - if (newAuth != _jB(member["authorized"],false)) { + const bool newAuth = OSUtils::jsonBool(b["authorized"],false); + if (newAuth != OSUtils::jsonBool(member["authorized"],false)) { member["authorized"] = newAuth; member[((newAuth) ? "lastAuthorizedTime" : "lastDeauthorizedTime")] = now; @@ -732,7 +685,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i::iterator t(mtags.begin());t!=mtags.end();++t) { @@ -750,7 +703,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (capabilities.is_array()) { json mcaps = json::array(); for(unsigned long i=0;ireportAtEveryHop = (_jB(b["reportAtEveryHop"],true) ? 1 : 0); + test->reportAtEveryHop = (OSUtils::jsonBool(b["reportAtEveryHop"],true) ? 1 : 0); if (!test->hopCount) { ::free((void *)test); @@ -868,19 +821,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _initNetwork(network); try { - if (b.count("name")) network["name"] = _jS(b["name"],""); - if (b.count("private")) network["private"] = _jB(b["private"],true); - if (b.count("enableBroadcast")) network["enableBroadcast"] = _jB(b["enableBroadcast"],false); - if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = _jB(b["allowPassiveBridging"],false); - if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); + if (b.count("name")) network["name"] = OSUtils::jsonString(b["name"],""); + if (b.count("private")) network["private"] = OSUtils::jsonBool(b["private"],true); + if (b.count("enableBroadcast")) network["enableBroadcast"] = OSUtils::jsonBool(b["enableBroadcast"],false); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = OSUtils::jsonBool(b["allowPassiveBridging"],false); + if (b.count("multicastLimit")) network["multicastLimit"] = OSUtils::jsonInt(b["multicastLimit"],32ULL); if (b.count("v4AssignMode")) { json nv4m; json &v4m = b["v4AssignMode"]; if (v4m.is_string()) { // backward compatibility - nv4m["zt"] = (_jS(v4m,"") == "zt"); + nv4m["zt"] = (OSUtils::jsonString(v4m,"") == "zt"); } else if (v4m.is_object()) { - nv4m["zt"] = _jB(v4m["zt"],false); + nv4m["zt"] = OSUtils::jsonBool(v4m["zt"],false); } else nv4m["zt"] = false; network["v4AssignMode"] = nv4m; } @@ -890,7 +843,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &v6m = b["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (v6m.is_string()) { // backward compatibility - std::vector v6ms(OSUtils::split(_jS(v6m,"").c_str(),",","","")); + std::vector v6ms(OSUtils::split(OSUtils::jsonString(v6m,"").c_str(),",","","")); std::sort(v6ms.begin(),v6ms.end()); v6ms.erase(std::unique(v6ms.begin(),v6ms.end()),v6ms.end()); nv6m["rfc4193"] = false; @@ -905,9 +858,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( nv6m["6plane"] = true; } } else if (v6m.is_object()) { - if (v6m.count("rfc4193")) nv6m["rfc4193"] = _jB(v6m["rfc4193"],false); - if (v6m.count("zt")) nv6m["zt"] = _jB(v6m["zt"],false); - if (v6m.count("6plane")) nv6m["6plane"] = _jB(v6m["6plane"],false); + if (v6m.count("rfc4193")) nv6m["rfc4193"] = OSUtils::jsonBool(v6m["rfc4193"],false); + if (v6m.count("zt")) nv6m["zt"] = OSUtils::jsonBool(v6m["zt"],false); + if (v6m.count("6plane")) nv6m["6plane"] = OSUtils::jsonBool(v6m["6plane"],false); } else { nv6m["rfc4193"] = false; nv6m["zt"] = false; @@ -951,8 +904,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i 0) { json t = json::object(); t["token"] = tstr; - t["expires"] = _jI(token["expires"],0ULL); - t["maxUsesPerMember"] = _jI(token["maxUsesPerMember"],0ULL); + t["expires"] = OSUtils::jsonInt(token["expires"],0ULL); + t["maxUsesPerMember"] = OSUtils::jsonInt(token["maxUsesPerMember"],0ULL); nat.push_back(t); } } @@ -1010,7 +963,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &cap = capabilities[i]; if (cap.is_object()) { json ncap = json::object(); - const uint64_t capId = _jI(cap["id"],0ULL); + const uint64_t capId = OSUtils::jsonInt(cap["id"],0ULL); ncap["id"] = capId; json &rules = cap["rules"]; @@ -1065,7 +1018,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _getNetworkMemberInfo(now,nwid,nmi); _addNetworkNonPersistedFields(network,now,nmi); - responseBody = network.dump(2); + responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; return 200; } // else 404 @@ -1113,7 +1066,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( if (!member.size()) return 404; - responseBody = member.dump(2); + responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; return 200; } @@ -1128,7 +1081,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( Mutex::Lock _l2(_nmiCache_m); _nmiCache.erase(nwid); - responseBody = network.dump(2); + responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; return 200; } @@ -1254,7 +1207,7 @@ void EmbeddedNetworkController::_request( _initMember(member); { - std::string haveIdStr(_jS(member["identity"],"")); + std::string haveIdStr(OSUtils::jsonString(member["identity"],"")); if (haveIdStr.length() > 0) { // If we already know this member's identity perform a full compare. This prevents // a "collision" from being able to auth onto our network in place of an already @@ -1283,9 +1236,9 @@ void EmbeddedNetworkController::_request( const char *authorizedBy = (const char *)0; bool autoAuthorized = false; json autoAuthCredentialType,autoAuthCredential; - if (_jB(member["authorized"],false)) { + if (OSUtils::jsonBool(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; - } else if (!_jB(network["private"],true)) { + } else if (!OSUtils::jsonBool(network["private"],true)) { authorizedBy = "networkIsPublic"; json &ahist = member["authHistory"]; if ((!ahist.is_array())||(ahist.size() == 0)) @@ -1304,9 +1257,9 @@ void EmbeddedNetworkController::_request( for(unsigned long i=0;i now))&&(tstr == presentedToken)) { bool usable = (maxUses == 0); @@ -1316,7 +1269,7 @@ void EmbeddedNetworkController::_request( if (ahist.is_array()) { for(unsigned long j=0;j::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) { nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); @@ -1459,11 +1412,11 @@ void EmbeddedNetworkController::_request( for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; @@ -1491,7 +1444,7 @@ void EmbeddedNetworkController::_request( for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) @@ -1525,14 +1478,14 @@ void EmbeddedNetworkController::_request( } } - const bool noAutoAssignIps = _jB(member["noAutoAssignIps"],false); + const bool noAutoAssignIps = OSUtils::jsonBool(member["noAutoAssignIps"],false); if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { - if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if ((OSUtils::jsonBool(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } - if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if ((OSUtils::jsonBool(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } @@ -1572,12 +1525,12 @@ void EmbeddedNetworkController::_request( ipAssignments = json::array(); } - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!noAutoAssignIps) ) { + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(OSUtils::jsonBool(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!noAutoAssignIps) ) { for(unsigned long p=0;((p(&ipRangeStartIA)->sin_addr.s_addr)); uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); @@ -1722,7 +1675,7 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid Mutex::Lock _l(_db_m); _db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) { try { - if (_jB(member["authorized"],false)) { + if (OSUtils::jsonBool(member["authorized"],false)) { ++nmi.authorizedMemberCount; if (member.count("recentLog")) { @@ -1730,28 +1683,28 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid if ((mlog.is_array())&&(mlog.size() > 0)) { const json &mlog1 = mlog[0]; if (mlog1.is_object()) { - if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) + if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) ++nmi.activeMemberCount; } } } - if (_jB(member["activeBridge"],false)) { - nmi.activeBridges.insert(_jS(member["id"],"0000000000")); + if (OSUtils::jsonBool(member["activeBridge"],false)) { + nmi.activeBridges.insert(OSUtils::jsonString(member["id"],"0000000000")); } if (member.count("ipAssignments")) { const json &mips = member["ipAssignments"]; if (mips.is_array()) { for(unsigned long i=0;isecond.lastModifiedOnDisk != lm) { if (OSUtils::readFile(path.c_str(),buf)) { try { - e->second.obj = nlohmann::json::parse(buf); + e->second.obj = OSUtils::jsonParse(buf); e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP e->second.lastCheck = now; @@ -91,7 +91,7 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe const uint64_t lm = OSUtils::getLastModified(path.c_str()); _E &e2 = _db[n]; try { - e2.obj = nlohmann::json::parse(buf); + e2.obj = OSUtils::jsonParse(buf); } catch ( ... ) { e2.obj = _EMPTY_JSON; buf = "{}"; diff --git a/one.cpp b/one.cpp index c4fc30e2..1cda3fb1 100644 --- a/one.cpp +++ b/one.cpp @@ -897,9 +897,6 @@ static void printHelp(const char *cn,FILE *out) fprintf(out, COPYRIGHT_NOTICE ZT_EOL_S LICENSE_GRANT ZT_EOL_S); - std::string updateUrl(OneService::autoUpdateUrl()); - if (updateUrl.length()) - fprintf(out,"Automatic updates enabled:" ZT_EOL_S" %s" ZT_EOL_S" (all updates are securely authenticated by 256-bit ECDSA signature)" ZT_EOL_S"" ZT_EOL_S,updateUrl.c_str()); fprintf(out,"Usage: %s [-switches] [home directory]" ZT_EOL_S"" ZT_EOL_S,cn); fprintf(out,"Available switches:" ZT_EOL_S); fprintf(out," -h - Display this help" ZT_EOL_S); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 65704fa3..a6757bcc 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -395,6 +395,64 @@ std::string OSUtils::platformDefaultHomePath() #endif // __UNIX_LIKE__ or not... } +// Inline these massive JSON operations in one place only to reduce binary footprint and compile time +nlohmann::json OSUtils::jsonParse(const std::string &buf) { return nlohmann::json::parse(buf); } +std::string OSUtils::jsonDump(const nlohmann::json &j) { return j.dump(2); } + +uint64_t OSUtils::jsonInt(const nlohmann::json &jv,const uint64_t dfl) +{ + try { + if (jv.is_number()) { + return (uint64_t)jv; + } else if (jv.is_string()) { + std::string s = jv; + return Utils::strToU64(s.c_str()); + } else if (jv.is_boolean()) { + return ((bool)jv ? 1ULL : 0ULL); + } + } catch ( ... ) {} + return dfl; +} + +bool OSUtils::jsonBool(const nlohmann::json &jv,const bool dfl) +{ + try { + if (jv.is_boolean()) { + return (bool)jv; + } else if (jv.is_number()) { + return ((uint64_t)jv > 0ULL); + } else if (jv.is_string()) { + std::string s = jv; + if (s.length() > 0) { + switch(s[0]) { + case 't': + case 'T': + case '1': + return true; + } + } + return false; + } + } catch ( ... ) {} + return dfl; +} + +std::string OSUtils::jsonString(const nlohmann::json &jv,const char *dfl) +{ + try { + if (jv.is_string()) { + return jv; + } else if (jv.is_number()) { + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); + return tmp; + } else if (jv.is_boolean()) { + return ((bool)jv ? std::string("1") : std::string("0")); + } + } catch ( ... ) {} + return std::string((dfl) ? dfl : ""); +} + // Used to convert HTTP header names to ASCII lower case const unsigned char OSUtils::TOLOWER_TABLE[256] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ' ', '!', '"', '#', '$', '%', '&', 0x27, '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index c73f0bd3..7aa83f17 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -45,6 +45,8 @@ #include #endif +#include "../ext/json/json.hpp" + namespace ZeroTier { /** @@ -267,6 +269,12 @@ public: */ static std::string platformDefaultHomePath(); + static nlohmann::json jsonParse(const std::string &buf); + static std::string jsonDump(const nlohmann::json &j); + static uint64_t jsonInt(const nlohmann::json &jv,const uint64_t dfl); + static bool jsonBool(const nlohmann::json &jv,const bool dfl); + static std::string jsonString(const nlohmann::json &jv,const char *dfl); + private: static const unsigned char TOLOWER_TABLE[256]; }; diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 3bdfdf1b..86158a91 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -515,7 +515,7 @@ unsigned int ControlPlane::handleRequest( _svc->getNetworkSettings(nws->networks[i].nwid,localSettings); try { - nlohmann::json j(nlohmann::json::parse(body)); + nlohmann::json j(OSUtils::jsonParse(body)); if (j.is_object()) { nlohmann::json &allowManaged = j["allowManaged"]; if (allowManaged.is_boolean()) localSettings.allowManaged = (bool)allowManaged; diff --git a/service/OneService.cpp b/service/OneService.cpp index cd267982..9a1084d0 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -148,52 +148,6 @@ namespace ZeroTier { namespace { -static uint64_t _jI(const json &jv,const uint64_t dfl) -{ - if (jv.is_number()) { - return (uint64_t)jv; - } else if (jv.is_string()) { - std::string s = jv; - return Utils::strToU64(s.c_str()); - } else if (jv.is_boolean()) { - return ((bool)jv ? 1ULL : 0ULL); - } - return dfl; -} -static bool _jB(const json &jv,const bool dfl) -{ - if (jv.is_boolean()) { - return (bool)jv; - } else if (jv.is_number()) { - return ((uint64_t)jv > 0ULL); - } else if (jv.is_string()) { - std::string s = jv; - if (s.length() > 0) { - switch(s[0]) { - case 't': - case 'T': - case '1': - return true; - } - } - return false; - } - return dfl; -} -static std::string _jS(const json &jv,const char *dfl) -{ - if (jv.is_string()) { - return jv; - } else if (jv.is_number()) { - char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); - return tmp; - } else if (jv.is_boolean()) { - return ((bool)jv ? std::string("1") : std::string("0")); - } - return std::string((dfl) ? dfl : ""); -} - #if 0 #ifdef ZT_AUTO_UPDATE @@ -747,7 +701,7 @@ public: std::string lcbuf; if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "local.conf").c_str(),lcbuf)) { try { - _localConfig = json::parse(lcbuf); + _localConfig = OSUtils::jsonParse(lcbuf); if (!_localConfig.is_object()) { fprintf(stderr,"WARNING: unable to parse local.conf (root element is not a JSON object)" ZT_EOL_S); } @@ -760,11 +714,11 @@ public: json &physical = _localConfig["physical"]; if (physical.is_object()) { for(json::iterator phy(physical.begin());phy!=physical.end();++phy) { - InetAddress net(_jS(phy.key(),"")); + InetAddress net(OSUtils::jsonString(phy.key(),"")); if (net) { if (phy.value().is_object()) { uint64_t tpid; - if ((tpid = _jI(phy.value()["trustedPathId"],0ULL)) != 0ULL) { + if ((tpid = OSUtils::jsonInt(phy.value()["trustedPathId"],0ULL)) != 0ULL) { if ( ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (net.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (net.netmaskBits() > 0) ) { trustedPathIds[trustedPathCount] = tpid; trustedPathNetworks[trustedPathCount] = net; @@ -1157,7 +1111,7 @@ public: if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) { const Address ztaddr(nstr.c_str()); if (ztaddr) { - const std::string rstr(_jS(v.value()["role"],"")); + const std::string rstr(OSUtils::jsonString(v.value()["role"],"")); _node->setRole(ztaddr.toInt(),((rstr == "upstream")||(rstr == "UPSTREAM")) ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF); const uint64_t ztaddr2 = ztaddr.toInt(); @@ -1169,7 +1123,7 @@ public: json &tryAddrs = v.value()["try"]; if (tryAddrs.is_array()) { for(unsigned long i=0;i 0)) { if (phy.value().is_object()) { - if (_jB(phy.value()["blacklist"],false)) { + if (OSUtils::jsonBool(phy.value()["blacklist"],false)) { if (net.ss_family == AF_INET) _globalV4Blacklist.push_back(net); else if (net.ss_family == AF_INET6) @@ -1219,7 +1173,7 @@ public: _interfacePrefixBlacklist.clear(); json &settings = _localConfig["settings"]; if (settings.is_object()) { - const std::string rp(_jS(settings["relayPolicy"],"")); + const std::string rp(OSUtils::jsonString(settings["relayPolicy"],"")); if ((rp == "always")||(rp == "ALWAYS")) _node->setRelayPolicy(ZT_RELAY_POLICY_ALWAYS); else if ((rp == "never")||(rp == "NEVER")) @@ -1229,7 +1183,7 @@ public: json &ignoreIfs = settings["interfacePrefixBlacklist"]; if (ignoreIfs.is_array()) { for(unsigned long i=0;i 0) _interfacePrefixBlacklist.push_back(tmp); } @@ -1238,7 +1192,7 @@ public: json &amf = settings["allowManagementFrom"]; if (amf.is_array()) { for(unsigned long i=0;i Date: Fri, 13 Jan 2017 15:19:59 -0800 Subject: Windows update build in Advanced Installer, and warning removal. --- .gitignore | 1 + controller/JSONDB.cpp | 2 +- ext/installfiles/windows/ZeroTier One.aip | 83 ++++++++++++++++++++++--------- 3 files changed, 62 insertions(+), 24 deletions(-) (limited to 'controller') diff --git a/.gitignore b/.gitignore index eb07dd4d..8d404eef 100755 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ zt1-src.tar.gz *.tmp .depend node_modules +zt1_update_* debian/files debian/zerotier-one debian/zerotier-one*.debhelper diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 4747e470..044f791c 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -161,7 +161,7 @@ std::string JSONDB::_genPath(const std::string &n,bool create) std::string p(_basePath); if (create) OSUtils::mkdir(p.c_str()); - for(unsigned long i=0,j=pt.size()-1;i + @@ -26,19 +27,19 @@ - + - + - - + + - - + + @@ -60,6 +61,7 @@ + @@ -73,7 +75,7 @@ - + @@ -92,6 +94,7 @@ + @@ -130,6 +133,10 @@ + + + + @@ -146,14 +153,19 @@ + + + + - + + @@ -194,17 +206,27 @@ - + + + + + + + + + + + @@ -214,10 +236,7 @@ - - - @@ -229,8 +248,6 @@ - - @@ -242,6 +259,7 @@ + @@ -258,7 +276,6 @@ - @@ -266,17 +283,29 @@ - - + + - - - + + + + + + + + + + + + + + + @@ -290,12 +319,18 @@ - + - + + + + + + + @@ -321,6 +356,8 @@ + + @@ -334,7 +371,7 @@ - + -- cgit v1.2.3 From e9007b1f569ce15780ed34b837db987eec457d2f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 19 Jan 2017 10:44:26 -0800 Subject: NodeJS migration script for old Sqlite controller.db to new controller data format. --- controller/EmbeddedNetworkController.hpp | 1 + controller/README.md | 10 +- controller/migrate-sqlite/migrate.js | 320 +++++++++++++++++++++++++++++++ controller/migrate-sqlite/package.json | 15 ++ osdep/OSUtils.cpp | 2 +- service/OneService.cpp | 10 +- 6 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 controller/migrate-sqlite/migrate.js create mode 100644 controller/migrate-sqlite/package.json (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 37c067ff..bd3a6666 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -165,6 +165,7 @@ private: if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); + if (!network.count("routes")) network["routes"] = nlohmann::json::array(); if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment diff --git a/controller/README.md b/controller/README.md index 805641d9..edbdf804 100644 --- a/controller/README.md +++ b/controller/README.md @@ -15,15 +15,9 @@ The migration tool is written in nodeJS and can be used like this: cd migrate-sqlite npm install - node migrate-sqlite.js + node migrate.js -You may need to `sudo node ...` if the ZeroTier working directory is owned by root. - -This code will dump the contents of any `controller.db` in the ZeroTier working directory and recreate its data in the form of JSON objects under `controller.d`. The old `controller.db` will then be renamed to `controller.db.migrated` and the controller will start normally. - -After migrating make sure that the contents of `controller.d` are owned and writable by the user that will be running the ZeroTier controller process! (Usually this is root but controllers that don't also join networks are sometimes run as unprivileged users.) - -If you don't have nodeJS on the machine running ZeroTier it is perfectly fine to just copy `controller.db` to a directory on another machine and run the migration tool there. After running your migration the contents of `controller.d` can be tar'd up and copied back over to the controller. Just remember to rename or remove `controller.db` on the controller machine afterwords so the controller will start. +Very old versions of nodeJS may have issues. We tested it with version 7. ### Scalability and Reliability diff --git a/controller/migrate-sqlite/migrate.js b/controller/migrate-sqlite/migrate.js new file mode 100644 index 00000000..ac9678a7 --- /dev/null +++ b/controller/migrate-sqlite/migrate.js @@ -0,0 +1,320 @@ +'use strict'; + +var sqlite3 = require('sqlite3').verbose(); +var fs = require('fs'); +var async = require('async'); + +function blobToIPv4(b) +{ + if (!b) + return null; + if (b.length !== 16) + return null; + return b.readUInt8(12).toString()+'.'+b.readUInt8(13).toString()+'.'+b.readUInt8(14).toString()+'.'+b.readUInt8(15).toString(); +} +function blobToIPv6(b) +{ + if (!b) + return null; + if (b.length !== 16) + return null; + var s = ''; + for(var i=0;i<16;++i) { + var x = b.readUInt8(i).toString(16); + if (x.length === 1) + s += '0'; + s += x; + if ((((i+1) & 1) === 0)&&(i !== 15)) + s += ':'; + } + return s; +} + +if (process.argv.length !== 4) { + console.log('ZeroTier Old Sqlite3 Controller DB Migration Utility'); + console.log('(c)2017 ZeroTier, Inc. [GPL3]'); + console.log(''); + console.log('Usage: node migrate.js '); + console.log(''); + console.log('The first argument must be the path to the old Sqlite3 controller.db'); + console.log('file. The second must be the path to the EMPTY controller.d database'); + console.log('directory for a new (1.1.17 or newer) controller. If this path does'); + console.log('not exist it will be created.'); + console.log(''); + console.log('WARNING: this will ONLY work correctly on a 1.1.14 controller database.'); + console.log('If your controller is old you should first upgrade to 1.1.14 and run the'); + console.log('controller so that it will brings its Sqlite3 database up to the latest'); + console.log('version before running this migration.'); + console.log(''); + process.exit(1); +} + +var oldDbPath = process.argv[2]; +var newDbPath = process.argv[3]; + +console.log('Starting migrate of "'+oldDbPath+'" to "'+newDbPath+'"...'); +console.log(''); + +var old = new sqlite3.Database(oldDbPath); + +var networks = {}; + +var nodeIdentities = {}; +var networkCount = 0; +var memberCount = 0; +var routeCount = 0; +var ipAssignmentPoolCount = 0; +var ipAssignmentCount = 0; +var ruleCount = 0; +var oldSchemaVersion = -1; + +async.series([function(nextStep) { + + old.each('SELECT v from Config WHERE k = \'schemaVersion\'',function(err,row) { + oldSchemaVersion = parseInt(row.v)||-1; + },nextStep); + +},function(nextStep) { + + if (oldSchemaVersion !== 4) { + console.log('FATAL: this MUST be run on a 1.1.14 controller.db! Upgrade your old'); + console.log('controller to 1.1.14 first and run it once to bring its DB up to date.'); + return process.exit(1); + } + + console.log('Reading networks...'); + old.each('SELECT * FROM Network',function(err,row) { + if ((typeof row.id === 'string')&&(row.id.length === 16)) { + var flags = parseInt(row.flags)||0; + networks[row.id] = { + id: row.id, + nwid: row.id, + objtype: 'network', + authTokens: [], + capabilities: [], + creationTime: parseInt(row.creationTime)||0, + enableBroadcast: !!row.enableBroadcast, + ipAssignmentPools: [], + lastModified: Date.now(), + multicastLimit: row.multicastLimit||32, + name: row.name||'', + private: !!row.private, + revision: parseInt(row.revision)||1, + rules: [{ 'type': 'ACTION_ACCEPT' }], // populated later if there are defined rules, otherwise default is allow all + routes: [], + v4AssignMode: { + 'zt': ((flags & 1) !== 0) + }, + v6AssignMode: { + '6plane': ((flags & 4) !== 0), + 'rfc4193': ((flags & 2) !== 0), + 'zt': ((flags & 8) !== 0) + }, + _members: {} // temporary + }; + ++networkCount; + //console.log(networks[row.id]); + } + },nextStep); + +},function(nextStep) { + + console.log(' '+networkCount+' networks.'); + console.log('Reading network route definitions...'); + old.each('SELECT * from Route WHERE ipVersion = 4 OR ipVersion = 6',function(err,row) { + var network = networks[row.networkId]; + if (network) { + var rt = { + target: (((row.ipVersion == 4) ? blobToIPv4(row.target) : blobToIPv6(row.target))+'/'+row.targetNetmaskBits), + via: ((row.via) ? ((row.ipVersion == 4) ? blobToIPv4(row.via) : blobToIPv6(row.via)) : null) + }; + network.routes.push(rt); + ++routeCount; + } + },nextStep); + +},function(nextStep) { + + console.log(' '+routeCount+' routes in '+networkCount+' networks.'); + console.log('Reading IP assignment pools...'); + old.each('SELECT * FROM IpAssignmentPool WHERE ipVersion = 4 OR ipVersion = 6',function(err,row) { + var network = networks[row.networkId]; + if (network) { + var p = { + ipRangeStart: ((row.ipVersion == 4) ? blobToIPv4(row.ipRangeStart) : blobToIPv6(row.ipRangeStart)), + ipRangeEnd: ((row.ipVersion == 4) ? blobToIPv4(row.ipRangeEnd) : blobToIPv6(row.ipRangeEnd)) + }; + network.ipAssignmentPools.push(p); + ++ipAssignmentPoolCount; + } + },nextStep); + +},function(nextStep) { + + console.log(' '+ipAssignmentPoolCount+' IP assignment pools in '+networkCount+' networks.'); + console.log('Reading known node identities...'); + old.each('SELECT * FROM Node',function(err,row) { + nodeIdentities[row.id] = row.identity; + },nextStep); + +},function(nextStep) { + + console.log(' '+Object.keys(nodeIdentities).length+' known identities.'); + console.log('Reading network members...'); + old.each('SELECT * FROM Member',function(err,row) { + var network = networks[row.networkId]; + if (network) { + network._members[row.nodeId] = { + id: row.nodeId, + address: row.nodeId, + objtype: 'member', + authorized: !!row.authorized, + activeBridge: !!row.activeBridge, + authHistory: [], + capabilities: [], + creationTime: 0, + identity: nodeIdentities[row.nodeId]||null, + ipAssignments: [], + lastAuthorizedTime: (row.authorized) ? Date.now() : 0, + lastDeauthorizedTime: (row.authorized) ? 0 : Date.now(), + lastModified: Date.now(), + lastRequestMetaData: '', + noAutoAssignIps: false, + nwid: row.networkId, + revision: parseInt(row.memberRevision)||1, + tags: [], + recentLog: [] + }; + ++memberCount; + //console.log(network._members[row.nodeId]); + } + },nextStep); + +},function(nextStep) { + + console.log(' '+memberCount+' members of '+networkCount+' networks.'); + console.log('Reading static IP assignments...'); + old.each('SELECT * FROM IpAssignment WHERE ipVersion = 4 OR ipVersion = 6',function(err,row) { + var network = networks[row.networkId]; + if (network) { + var member = network._members[row.nodeId]; + if ((member)&&((member.authorized)||(!network['private']))) { // don't mirror assignments to unauthorized members to avoid conflicts + if (row.ipVersion == 4) { + member.ipAssignments.push(blobToIPv4(row.ip)); + ++ipAssignmentCount; + } else if (row.ipVersion == 6) { + member.ipAssignments.push(blobToIPv6(row.ip)); + ++ipAssignmentCount; + } + } + } + },nextStep); + +},function(nextStep) { + + // Old versions only supported Ethertype whitelisting, so that's + // all we mirror forward. The other fields were always unused. + + console.log(' '+ipAssignmentCount+' IP assignments for '+memberCount+' authorized members of '+networkCount+' networks.'); + console.log('Reading allowed Ethernet types (old basic rules)...'); + var etherTypesByNetwork = {}; + old.each('SELECT DISTINCT networkId,ruleNo,etherType FROM Rule WHERE "action" = \'accept\'',function(err,row) { + if (row.networkId in networks) { + var et = parseInt(row.etherType)||0; + var ets = etherTypesByNetwork[row.networkId]; + if (!ets) + etherTypesByNetwork[row.networkId] = [ et ]; + else ets.push(et); + } + },function(err) { + if (err) return nextStep(err); + for(var nwid in etherTypesByNetwork) { + var ets = etherTypesByNetwork[nwid].sort(); + var network = networks[nwid]; + if (network) { + var rules = []; + if (ets.indexOf(0) >= 0) { + // If 0 is in the list, all Ethernet types are allowed so we accept all. + rules.push({ 'type': 'ACTION_ACCEPT' }); + } else { + // Otherwise we whitelist. + for(var i=0;i 0) { + try { + fs.mkdirSync(nwBase+network.id); + } catch (e) {} + var mbase = nwBase+network.id+'/member'; + try { + fs.mkdirSync(mbase,0o700); + } catch (e) {} + mbase = mbase + '/'; + + for(var mi=0;mi", + "license": "GPL-3.0", + "dependencies": { + "async": "^2.1.4", + "sqlite3": "^3.1.8" + } +} diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 08d67bbe..170e1376 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -398,7 +398,7 @@ std::string OSUtils::platformDefaultHomePath() // Inline these massive JSON operations in one place only to reduce binary footprint and compile time nlohmann::json OSUtils::jsonParse(const std::string &buf) { return nlohmann::json::parse(buf); } -std::string OSUtils::jsonDump(const nlohmann::json &j) { return j.dump(2); } +std::string OSUtils::jsonDump(const nlohmann::json &j) { return j.dump(1); } uint64_t OSUtils::jsonInt(const nlohmann::json &jv,const uint64_t dfl) { diff --git a/service/OneService.cpp b/service/OneService.cpp index b273aad4..0fd8496d 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -625,7 +625,15 @@ public: for(int i=0;i<3;++i) _portsBE[i] = Utils::hton((uint16_t)_ports[i]); - _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(FILE *)0); + // Check for legacy controller.db and terminate if present to prevent nasty surprises for DIY controller folks + if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S "controller.db").c_str())) { + Mutex::Lock _l(_termReason_m); + _termReason = ONE_UNRECOVERABLE_ERROR; + _fatalErrorMessage = "controller.db is present in our home path! run migrate-sqlite to migrate to new controller.d format."; + return _termReason; + } + + _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str(),(FILE *)0); _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER -- cgit v1.2.3 From fd460d93c471d640cea2a8bbd7eb30b81b948254 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 19 Jan 2017 10:53:44 -0800 Subject: docs --- controller/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/README.md b/controller/README.md index edbdf804..093300a6 100644 --- a/controller/README.md +++ b/controller/README.md @@ -7,7 +7,7 @@ As of ZeroTier One version 1.2.0 this code is included in normal builds for desk Controller data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, rsync'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, and if they are edited directly take care not to save corrupt JSON since that can also lead to data loss when the controller is restarted. Going through the API is strongly preferred to directly modifying these files. -### Upgrading from Older Versions +### Upgrading from Older (1.1.14 or earlier) Versions Older versions of this code used a SQLite database instead of in-filesystem JSON. A migration utility called `migrate-sqlite` is included here and *must* be used to migrate this data to the new format. If the controller is started with an old `controller.db` in its working directory it will terminate after printing an error to *stderr*. This is done to prevent "surprises" for those running DIY controllers using the old code. -- cgit v1.2.3 From 31db768e4d4c2815d2be0493b2c76ea5f5edbffa Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sat, 4 Feb 2017 00:23:31 -0800 Subject: A bit of code cleanup. --- controller/EmbeddedNetworkController.cpp | 2 +- doc/MANUAL.md | 2 - node/Address.hpp | 77 +++++++++----------------------- node/Identity.cpp | 2 +- node/NetworkConfig.cpp | 2 +- node/Switch.cpp | 3 +- service/OneService.cpp | 2 +- 7 files changed, 25 insertions(+), 65 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3c13e7ce..e7bcdd07 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1690,7 +1690,7 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid } if (OSUtils::jsonBool(member["activeBridge"],false)) { - nmi.activeBridges.insert(OSUtils::jsonString(member["id"],"0000000000")); + nmi.activeBridges.insert(Address(Utils::hexStrToU64(OSUtils::jsonString(member["id"],"0000000000").c_str()))); } if (member.count("ipAssignments")) { diff --git a/doc/MANUAL.md b/doc/MANUAL.md index abaeef26..8af8fc2d 100644 --- a/doc/MANUAL.md +++ b/doc/MANUAL.md @@ -109,8 +109,6 @@ In the simplest case using only global roots, an initial connection setup betwee 3. R also sends a message called *RENDEZVOUS* to A containing hints about how it might reach B, and to B informing it how it might reach A. We call this "transport triggered link provisioning." 4. A and B get *RENDEZVOUS* and attempt to send test messages to each other, possibly accomplishing [hole punching](https://en.wikipedia.org/wiki/UDP_hole_punching) of any NATs or stateful firewalls that happen to be in the way. If this works a direct link is established and packets no longer need to take the scenic route. -If R is a federated root the same process occurs, but possibly with additional steps. - VL1 provides instant always-on virtual L1 connectivity between all devices in the world. Indirect paths are automatically and transparently upgraded to direct paths whenever possible, and if a direct path is lost ZeroTier falls back to indirect communication and the process begins again. If a direct path can never be established, indirect communication can continue forever with direct connection attempts also continuing indefinitely on a periodic basis. The protocol also contains other facilities for direct connectivity establishment such as LAN peer discovery, port prediction to traverse IPv4 symmetric NATs, and explicit endpoint advertisement that can be coupled with port mapping using uPnP or NAT-PMP if available. VL1 is persistent and determined when it comes to finding the most efficient way to move data. diff --git a/node/Address.hpp b/node/Address.hpp index 9bf5605a..4a5883b0 100644 --- a/node/Address.hpp +++ b/node/Address.hpp @@ -38,57 +38,26 @@ namespace ZeroTier { class Address { public: - Address() - throw() : - _a(0) - { - } - - Address(const Address &a) - throw() : - _a(a._a) - { - } - - Address(uint64_t a) - throw() : - _a(a & 0xffffffffffULL) - { - } - - Address(const char *s) - throw() - { - unsigned char foo[ZT_ADDRESS_LENGTH]; - setTo(foo,Utils::unhex(s,foo,ZT_ADDRESS_LENGTH)); - } - - Address(const std::string &s) - throw() - { - unsigned char foo[ZT_ADDRESS_LENGTH]; - setTo(foo,Utils::unhex(s.c_str(),foo,ZT_ADDRESS_LENGTH)); - } + Address() : _a(0) {} + Address(const Address &a) : _a(a._a) {} + Address(uint64_t a) : _a(a & 0xffffffffffULL) {} /** * @param bits Raw address -- 5 bytes, big-endian byte order * @param len Length of array */ Address(const void *bits,unsigned int len) - throw() { setTo(bits,len); } inline Address &operator=(const Address &a) - throw() { _a = a._a; return *this; } inline Address &operator=(const uint64_t a) - throw() { _a = (a & 0xffffffffffULL); return *this; @@ -99,7 +68,6 @@ public: * @param len Length of array */ inline void setTo(const void *bits,unsigned int len) - throw() { if (len < ZT_ADDRESS_LENGTH) { _a = 0; @@ -119,7 +87,6 @@ public: * @param len Length of array */ inline void copyTo(void *bits,unsigned int len) const - throw() { if (len < ZT_ADDRESS_LENGTH) return; @@ -138,7 +105,6 @@ public: */ template inline void appendTo(Buffer &b) const - throw(std::out_of_range) { unsigned char *p = (unsigned char *)b.appendField(ZT_ADDRESS_LENGTH); *(p++) = (unsigned char)((_a >> 32) & 0xff); @@ -152,7 +118,6 @@ public: * @return Integer containing address (0 to 2^40) */ inline uint64_t toInt() const - throw() { return _a; } @@ -161,7 +126,6 @@ public: * @return Hash code for use with Hashtable */ inline unsigned long hashCode() const - throw() { return (unsigned long)_a; } @@ -188,12 +152,12 @@ public: /** * @return True if this address is not zero */ - inline operator bool() const throw() { return (_a != 0); } + inline operator bool() const { return (_a != 0); } /** * Set to null/zero */ - inline void zero() throw() { _a = 0; } + inline void zero() { _a = 0; } /** * Check if this address is reserved @@ -205,7 +169,6 @@ public: * @return True if address is reserved and may not be used */ inline bool isReserved() const - throw() { return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX)); } @@ -214,21 +177,21 @@ public: * @param i Value from 0 to 4 (inclusive) * @return Byte at said position (address interpreted in big-endian order) */ - inline unsigned char operator[](unsigned int i) const throw() { return (unsigned char)((_a >> (32 - (i * 8))) & 0xff); } - - inline bool operator==(const uint64_t &a) const throw() { return (_a == (a & 0xffffffffffULL)); } - inline bool operator!=(const uint64_t &a) const throw() { return (_a != (a & 0xffffffffffULL)); } - inline bool operator>(const uint64_t &a) const throw() { return (_a > (a & 0xffffffffffULL)); } - inline bool operator<(const uint64_t &a) const throw() { return (_a < (a & 0xffffffffffULL)); } - inline bool operator>=(const uint64_t &a) const throw() { return (_a >= (a & 0xffffffffffULL)); } - inline bool operator<=(const uint64_t &a) const throw() { return (_a <= (a & 0xffffffffffULL)); } - - inline bool operator==(const Address &a) const throw() { return (_a == a._a); } - inline bool operator!=(const Address &a) const throw() { return (_a != a._a); } - inline bool operator>(const Address &a) const throw() { return (_a > a._a); } - inline bool operator<(const Address &a) const throw() { return (_a < a._a); } - inline bool operator>=(const Address &a) const throw() { return (_a >= a._a); } - inline bool operator<=(const Address &a) const throw() { return (_a <= a._a); } + inline unsigned char operator[](unsigned int i) const { return (unsigned char)((_a >> (32 - (i * 8))) & 0xff); } + + inline bool operator==(const uint64_t &a) const { return (_a == (a & 0xffffffffffULL)); } + inline bool operator!=(const uint64_t &a) const { return (_a != (a & 0xffffffffffULL)); } + inline bool operator>(const uint64_t &a) const { return (_a > (a & 0xffffffffffULL)); } + inline bool operator<(const uint64_t &a) const { return (_a < (a & 0xffffffffffULL)); } + inline bool operator>=(const uint64_t &a) const { return (_a >= (a & 0xffffffffffULL)); } + inline bool operator<=(const uint64_t &a) const { return (_a <= (a & 0xffffffffffULL)); } + + inline bool operator==(const Address &a) const { return (_a == a._a); } + inline bool operator!=(const Address &a) const { return (_a != a._a); } + inline bool operator>(const Address &a) const { return (_a > a._a); } + inline bool operator<(const Address &a) const { return (_a < a._a); } + inline bool operator>=(const Address &a) const { return (_a >= a._a); } + inline bool operator<=(const Address &a) const { return (_a <= a._a); } private: uint64_t _a; diff --git a/node/Identity.cpp b/node/Identity.cpp index c47805d9..05b70873 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -160,7 +160,7 @@ bool Identity::fromString(const char *str) for(char *f=Utils::stok(tmp,":",&saveptr);(f);f=Utils::stok((char *)0,":",&saveptr)) { switch(fno++) { case 0: - _address = Address(f); + _address = Address(Utils::hexStrToU64(f)); if (_address.isReserved()) return false; break; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 6acc48ea..2f356b15 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -258,7 +258,7 @@ bool NetworkConfig::fromDictionary(const Dictionary 0) { char *saveptr = (char *)0; for(char *f=Utils::stok(tmp2,",",&saveptr);(f);f=Utils::stok((char *)0,",",&saveptr)) { - this->addSpecialist(Address(f),ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + this->addSpecialist(Address(Utils::hexStrToU64(f)),ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); } } #else diff --git a/node/Switch.cpp b/node/Switch.cpp index 9fcd379f..a769faea 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -224,7 +224,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from #endif SharedPtr relayTo = RR->topology->getPeer(destination); - if ((relayTo)&&((relayTo->sendDirect(packet.data(),packet.size(),now,false)))) { + if ((relayTo)&&(relayTo->sendDirect(packet.data(),packet.size(),now,false))) { if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) { // don't send RENDEZVOUS for cluster frontplane relays const InetAddress *hintToSource = (InetAddress *)0; const InetAddress *hintToDest = (InetAddress *)0; @@ -245,7 +245,6 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } if ((hintToSource)&&(hintToDest)) { - TRACE(">> RENDEZVOUS: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str()); unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for obscure NAT-t reasons const unsigned int completed = alt + 2; while (alt != completed) { diff --git a/service/OneService.cpp b/service/OneService.cpp index 49c5f4a0..9a1503e5 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -926,7 +926,7 @@ public: for(json::iterator v(virt.begin());v!=virt.end();++v) { const std::string nstr = v.key(); if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) { - const Address ztaddr(nstr.c_str()); + const Address ztaddr(Utils::hexStrToU64(nstr.c_str())); if (ztaddr) { const uint64_t ztaddr2 = ztaddr.toInt(); std::vector &v4h = _v4Hints[ztaddr2]; -- cgit v1.2.3 From ac3e883c05e97ff7ffea48e18e2d578f729df91c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 14:07:30 -0800 Subject: One more place to add "break". --- controller/EmbeddedNetworkController.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index e7bcdd07..90018f0d 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -93,8 +93,8 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["address"] = Address(rule.v.fwd.address).toString(); r["flags"] = (unsigned int)rule.v.fwd.flags; break; - case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: - r["type"] = "ACTION_DEBUG_LOG"; + case ZT_NETWORK_RULE_ACTION_BREAK: + r["type"] = "ACTION_BREAK"; break; default: break; @@ -266,8 +266,8 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.fwd.address = Utils::hexStrToU64(OSUtils::jsonString(r["address"],"0").c_str()) & 0xffffffffffULL; rule.v.fwd.flags = (uint32_t)(OSUtils::jsonInt(r["flags"],0ULL) & 0xffffffffULL); return true; - } else if (t == "ACTION_DEBUG_LOG") { - rule.t |= ZT_NETWORK_RULE_ACTION_DEBUG_LOG; + } else if (t == "ACTION_BREAK") { + rule.t |= ZT_NETWORK_RULE_ACTION_BREAK; return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; -- cgit v1.2.3 From 672f17c6e9ae981a70c51fdd62882e4847c5b6ba Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 7 Feb 2017 09:33:39 -0800 Subject: Add a mask and value range to the IP tos rule field. This allows TOS to be matched more usefully. This will break anyone using tos in the beta, but nobody seems to be and its pre-release so now is the time. --- controller/EmbeddedNetworkController.cpp | 8 ++++++-- include/ZeroTierOne.h | 5 ++++- node/Capability.hpp | 10 +++++++--- node/Network.cpp | 8 +++++--- 4 files changed, 22 insertions(+), 9 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 90018f0d..e798a80c 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -150,7 +150,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_IP_TOS: r["type"] = "MATCH_IP_TOS"; - r["ipTos"] = (unsigned int)rule.v.ipTos; + r["mask"] = (unsigned int)rule.v.ipTos.mask; + r["start"] = (unsigned int)rule.v.ipTos.value[0]; + r["end"] = (unsigned int)rule.v.ipTos.value[1]; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: r["type"] = "MATCH_IP_PROTOCOL"; @@ -329,7 +331,9 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_IP_TOS") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; - rule.v.ipTos = (uint8_t)(OSUtils::jsonInt(r["ipTos"],0ULL) & 0xffULL); + rule.v.ipTos.mask = (uint8_t)(OSUtils::jsonInt(r["mask"],0ULL) & 0xffULL); + rule.v.ipTos.value[0] = (uint8_t)(OSUtils::jsonInt(r["start"],0ULL) & 0xffULL); + rule.v.ipTos.value[1] = (uint8_t)(OSUtils::jsonInt(r["end"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_IP_PROTOCOL") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 860343ba..583f9b6a 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -705,7 +705,10 @@ typedef struct /** * IP type of service a.k.a. DSCP field */ - uint8_t ipTos; + struct { + uint8_t mask; + uint8_t value[2]; + } ipTos; /** * Ethernet packet size in host byte order (start-end, inclusive) diff --git a/node/Capability.hpp b/node/Capability.hpp index ddbfd9ee..08714038 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -216,8 +216,10 @@ public: b.append((uint8_t)rules[i].v.ipv6.mask); break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - b.append((uint8_t)1); - b.append((uint8_t)rules[i].v.ipTos); + b.append((uint8_t)3); + b.append((uint8_t)rules[i].v.ipTos.mask); + b.append((uint8_t)rules[i].v.ipTos.value[0]); + b.append((uint8_t)rules[i].v.ipTos.value[1]); break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: b.append((uint8_t)1); @@ -308,7 +310,9 @@ public: rules[ruleCount].v.ipv6.mask = (uint8_t)b[p + 16]; break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - rules[ruleCount].v.ipTos = (uint8_t)b[p]; + rules[ruleCount].v.ipTos.mask = (uint8_t)b[p]; + rules[ruleCount].v.ipTos.value[0] = (uint8_t)b[p+1]; + rules[ruleCount].v.ipTos.value[1] = (uint8_t)b[p+2]; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: rules[ruleCount].v.ipProtocol = (uint8_t)b[p]; diff --git a/node/Network.cpp b/node/Network.cpp index 5961b087..7412e3e7 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -368,11 +368,13 @@ static _doZtFilterResult _doZtFilter( break; case ZT_NETWORK_RULE_MATCH_IP_TOS: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + //thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + const uint8_t tosMasked = frameData[1] & rules[rn].v.ipTos.mask; + thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((frameData[1] & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { - const uint8_t trafficClass = ((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f); - thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((trafficClass & 0xfc) >> 2)); + const uint8_t tosMasked = (((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f)) & rules[rn].v.ipTos.mask; + thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((trafficClass & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; -- cgit v1.2.3 From 32f5a0ab1856485fb021fbf98920e8a9d63f0e23 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 21 Feb 2017 13:27:20 -0800 Subject: Add default tag values and default set capabilities for new members. --- controller/EmbeddedNetworkController.cpp | 121 +++++++++++++++++++++++-------- controller/EmbeddedNetworkController.hpp | 1 + rule-compiler/rule-compiler.js | 67 +++++++++++++---- 3 files changed, 144 insertions(+), 45 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index e798a80c..615cbb23 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -969,6 +969,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json ncap = json::object(); const uint64_t capId = OSUtils::jsonInt(cap["id"],0ULL); ncap["id"] = capId; + ncap["default"] = OSUtils::jsonBool(cap["default"],false); json &rules = cap["rules"]; json nrules = json::array(); @@ -994,6 +995,31 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["capabilities"] = ncapsa; } } + + if (b.count("tags")) { + json &tags = b["tags"]; + if (tags.is_array()) { + std::map< uint64_t,json > ntags; + for(unsigned long i=0;i::iterator t(ntags.begin());t!=ntags.end();++t) + ntagsa.push_back(t->second); + network["tags"] = ntagsa; + } + } + } catch ( ... ) { responseBody = "{ \"message\": \"exception occurred while parsing body variables\" }"; responseContentType = "application/json"; @@ -1207,6 +1233,8 @@ void EmbeddedNetworkController::_request( return; } + const bool newMember = (member.size() == 0); + json origMember(member); // for detecting modification later _initMember(member); @@ -1392,6 +1420,7 @@ void EmbeddedNetworkController::_request( json &routes = network["routes"]; json &rules = network["rules"]; json &capabilities = network["capabilities"]; + json &tags = network["tags"]; json &memberCapabilities = member["capabilities"]; json &memberTags = member["tags"]; @@ -1411,53 +1440,83 @@ void EmbeddedNetworkController::_request( } } - if ((memberCapabilities.is_array())&&(memberCapabilities.size() > 0)&&(capabilities.is_array())) { - std::map< uint64_t,json * > capsById; + std::map< uint64_t,json * > capsById; + if (!memberCapabilities.is_array()) + memberCapabilities = json::array(); + if (capabilities.is_array()) { for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { - ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; - unsigned int caprc = 0; - json &caprj = (*cap)["rules"]; - if ((caprj.is_array())&&(caprj.size() > 0)) { - for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) + if (cap.is_object()) { + const uint64_t id = OSUtils::jsonInt(cap["id"],0ULL) & 0xffffffffULL; + capsById[id] = ∩ + if ((newMember)&&(OSUtils::jsonBool(cap["default"],false))) { + bool have = false; + for(unsigned long i=0;i= ZT_MAX_NETWORK_CAPABILITIES) - break; } } } + for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { + ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; + unsigned int caprc = 0; + json &caprj = (*cap)["rules"]; + if ((caprj.is_array())&&(caprj.size() > 0)) { + for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) + break; + if (_parseRule(caprj[j],capr[caprc])) + ++caprc; + } + } + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); + if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) + ++nc.capabilityCount; + if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + break; + } + } + std::map< uint32_t,uint32_t > memberTagsById; if (memberTags.is_array()) { - std::map< uint32_t,uint32_t > tagsById; for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { - if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) - break; - nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(_signingId)) - ++nc.tagCount; + } + if (tags.is_array()) { // check network tags array for defaults that are not present in member tags + for(unsigned long i=0;i::const_iterator t(memberTagsById.begin());t!=memberTagsById.end();++t) { + if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + break; + nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); + if (nc.tags[nc.tagCount].sign(_signingId)) + ++nc.tagCount; + } } if (routes.is_array()) { diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index bd3a6666..3e39eaf5 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -165,6 +165,7 @@ private: if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); + if (!network.count("tags")) network["tags"] = nlohmann::json::array(); if (!network.count("routes")) network["routes"] = nlohmann::json::array(); if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); if (!network.count("rules")) { diff --git a/rule-compiler/rule-compiler.js b/rule-compiler/rule-compiler.js index 76358606..5e8b56c8 100644 --- a/rule-compiler/rule-compiler.js +++ b/rule-compiler/rule-compiler.js @@ -19,7 +19,7 @@ const CHARACTERISTIC_BITS = { 'tcp_rs0': 11 }; -// Shorthand names for ethernet types +// Shorthand names for common ethernet types const ETHERTYPES = { 'ipv4': 0x0800, 'arp': 0x0806, @@ -31,9 +31,11 @@ const ETHERTYPES = { 'ipx_b': 0x8138 }; -// Shorthand names for IP protocols +// Shorthand names for common IP protocols const IP_PROTOCOLS = { 'icmp': 0x01, + 'icmp4': 0x01, + 'icmpv4': 0x01, 'igmp': 0x02, 'ipip': 0x04, 'tcp': 0x06, @@ -68,6 +70,7 @@ const RESERVED_WORDS = { 'macro': true, 'tag': true, 'cap': true, + 'default': true, 'drop': true, 'accept': true, @@ -176,21 +179,22 @@ const MATCH_ARG_COUNTS = { 'teq': 2 }; +// Regex of all alphanumeric characters in Unicode const INTL_ALPHANUM_REGEX = new RegExp('[0-9A-Za-z\xAA\xB5\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0\u08A2-\u08AC\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097F\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09F0\u09F1\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C3D\u0C58\u0C59\u0C60\u0C61\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0\u0CE1\u0CF1\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D60\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F4\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191C\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19C1-\u19C7\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183\u2184\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005\u3006\u3031-\u3035\u303B\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FCC\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A\uA62B\uA640-\uA66E\uA67F-\uA697\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA78E\uA790-\uA793\uA7A0-\uA7AA\uA7F8-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA80-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uABC0-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]'); + +// Checks whether something is a valid capability, tag, or macro name function _isValidName(n) { - if ((typeof n !== 'string')||(n.length === 0)) - return false; - if ("0123456789".indexOf(n.charAt(0)) >= 0) - return false; + if ((typeof n !== 'string')||(n.length === 0)) return false; + if ("0123456789".indexOf(n.charAt(0)) >= 0) return false; for(let i=0;i= parsed.length) || (!Array.isArray(parsed[i + 1])) || (parsed[i + 1].length < 1) || (!Array.isArray(parsed[i + 1][0])) ) return [ parsed[i][1],parsed[i][2],'Macro definition is missing name.' ]; let macro = parsed[++i]; @@ -711,6 +717,8 @@ function compile(src,rules,caps,tags) rules: macro.slice(1) }; } else if (keyword === 'tag') { + // Define tags + if ( ((i + 1) >= parsed.length) || (!Array.isArray(parsed[i + 1])) || (parsed[i + 1].length < 1) || (!Array.isArray(parsed[i + 1][0])) ) return [ parsed[i][1],parsed[i][2],'Tag definition is missing name.' ]; let tag = parsed[++i]; @@ -727,6 +735,7 @@ function compile(src,rules,caps,tags) let flags = {}; let enums = {}; let id = -1; + let dfl = null; for(let k=1;k 0xffffffff)) return [ tag[k][1],tag[k][2],'Invalid or out of range tag ID.' ]; + } else if (tkeyword === 'default') { + if (id >= 0) + return [ tag[k][1],tag[k][2],'Duplicate tag id definition.' ]; + if ((k + 1) >= tag.length) + return [ tag[k][1],tag[k][2],'Missing value for default.' ]; + dfl = tag[++k][0]||null; } else if (tkeyword === 'flag') { if ((k + 2) >= tag.length) return [ tag[k][1],tag[k][2],'Missing tag flag name or bit index.' ]; @@ -780,12 +795,32 @@ function compile(src,rules,caps,tags) if (id < 0) return [ tag[0][1],tag[0][2],'Tag definition is missing a numeric ID.' ]; + if (dfl) { + let dfl2 = enums[dfl]; + if (typeof dfl2 === 'number') { + dfl = dfl2; + } else { + dfl2 = flags[dfl]; + if (typeof dfl2 === 'number') { + dfl = dfl2; + } else { + dfl = _parseNum(dfl)||0; + if (dfl < 0) + dfl = 0; + else dfl &= 0xffffffff; + } + } + } + tags[tagName] = { - id: id, - enums: enums, - flags: flags + 'id': id, + 'default': dfl, + 'enums': enums, + 'flags': flags }; } else if (keyword === 'cap') { + // Define capabilities + if ( ((i + 1) >= parsed.length) || (!Array.isArray(parsed[i + 1])) || (parsed[i + 1].length < 1) || (!Array.isArray(parsed[i + 1][0])) ) return [ parsed[i][1],parsed[i][2],'Capability definition is missing name.' ]; let cap = parsed[++i]; @@ -801,8 +836,9 @@ function compile(src,rules,caps,tags) let capRules = []; let id = -1; + let dfl = false; for(let k=1;k= 0) return [ cap[k][1],cap[k][2],'Duplicate id directive in capability definition.' ]; if ((k + 1) >= cap.length) @@ -814,6 +850,8 @@ function compile(src,rules,caps,tags) if (caps[cn].id === id) return [ cap[k - 1][1],cap[k - 1][2],'Duplicate capability ID.' ]; } + } else if (cap[k][0].toLowerCase() === 'default') { + dfl = true; } else { capRules.push(cap[k]); } @@ -822,8 +860,9 @@ function compile(src,rules,caps,tags) return [ cap[0][1],cap[0][2],'Capability definition is missing a numeric ID.' ]; caps[capName] = { - id: id, - rules: capRules + 'id': id, + 'default': dfl, + 'rules': capRules }; } else { baseRuleTree.push(parsed[i]); -- cgit v1.2.3 From 54fa73844c0037138c7d5bad8ad27d1c441ca12c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 21 Feb 2017 13:48:29 -0800 Subject: Fix crash. --- controller/EmbeddedNetworkController.cpp | 35 +++++++++++++++++--------------- 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 615cbb23..ca548fd4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1465,24 +1465,27 @@ void EmbeddedNetworkController::_request( } for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { - ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; - unsigned int caprc = 0; - json &caprj = (*cap)["rules"]; - if ((caprj.is_array())&&(caprj.size() > 0)) { - for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) - break; - if (_parseRule(caprj[j],capr[caprc])) - ++caprc; + std::map< uint64_t,json * >::const_iterator ctmp = capsById.find(capId); + if (ctmp != capsById.end()) { + json *cap = ctmp->second; + if ((cap)&&(cap->is_object())&&(cap->size() > 0)) { + ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; + unsigned int caprc = 0; + json &caprj = (*cap)["rules"]; + if ((caprj.is_array())&&(caprj.size() > 0)) { + for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) + break; + if (_parseRule(caprj[j],capr[caprc])) + ++caprc; + } } + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); + if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) + ++nc.capabilityCount; + if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + break; } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) - ++nc.capabilityCount; - if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) - break; } } -- cgit v1.2.3 From 10185e92faa77a4b032a27a7c01b4186727b91b9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 23 Feb 2017 11:47:36 -0800 Subject: Certificate of ownership -- used to secure against IP address spoofing, especially for IPv4 and regular IPv6. --- controller/EmbeddedNetworkController.cpp | 9 ++ include/ZeroTierOne.h | 5 + node/Capability.hpp | 1 - node/CertificateOfOwnership.cpp | 46 ++++++ node/CertificateOfOwnership.hpp | 251 +++++++++++++++++++++++++++++++ node/IncomingPacket.cpp | 19 +++ node/Membership.cpp | 164 +++++++++++++++----- node/Membership.hpp | 81 +++++----- node/Network.hpp | 11 ++ node/NetworkConfig.cpp | 25 ++- node/NetworkConfig.hpp | 21 ++- node/Packet.hpp | 2 + node/Revocation.hpp | 5 +- node/Tag.hpp | 3 +- objects.mk | 1 + 15 files changed, 550 insertions(+), 94 deletions(-) create mode 100644 node/CertificateOfOwnership.cpp create mode 100644 node/CertificateOfOwnership.hpp (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index ca548fd4..78a9b7c7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1706,6 +1706,15 @@ void EmbeddedNetworkController::_request( } } + // Issue a certificate of ownership for all static IPs + if (nc.staticIpCount) { + nc.certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); + for(unsigned int i=0;i(p); p += 8; _ts = b.template at(p); p += 8; _id = b.template at(p); p += 4; diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp new file mode 100644 index 00000000..8305c489 --- /dev/null +++ b/node/CertificateOfOwnership.cpp @@ -0,0 +1,46 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "CertificateOfOwnership.hpp" +#include "RuntimeEnvironment.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Switch.hpp" +#include "Network.hpp" + +namespace ZeroTier { + +int CertificateOfOwnership::verify(const RuntimeEnvironment *RR) const +{ + if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) + return -1; + const Identity id(RR->topology->getIdentity(_signedBy)); + if (!id) { + RR->sw->requestWhois(_signedBy); + return 1; + } + try { + Buffer<(sizeof(CertificateOfOwnership) + 64)> tmp; + this->serialize(tmp,true); + return (id.verify(tmp.data(),tmp.size(),_signature) ? 0 : -1); + } catch ( ... ) { + return -1; + } +} + +} // namespace ZeroTier diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp new file mode 100644 index 00000000..69b26aec --- /dev/null +++ b/node/CertificateOfOwnership.hpp @@ -0,0 +1,251 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_CERTIFICATEOFOWNERSHIP_HPP +#define ZT_CERTIFICATEOFOWNERSHIP_HPP + +#include +#include +#include +#include + +#include "Constants.hpp" +#include "C25519.hpp" +#include "Address.hpp" +#include "Identity.hpp" +#include "Buffer.hpp" +#include "InetAddress.hpp" +#include "MAC.hpp" + +// Max things per CertificateOfOwnership +#define ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS 16 + +// Maximum size of a thing's value field in bytes +#define ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE 16 + +namespace ZeroTier { + +class RuntimeEnvironment; + +/** + * Certificate indicating ownership of a network identifier + */ +class CertificateOfOwnership +{ +public: + enum Thing + { + THING_NULL = 0, + THING_MAC_ADDRESS = 1, + THING_IPV4_ADDRESS = 2, + THING_IPV6_ADDRESS = 3 + }; + + CertificateOfOwnership() : + _networkId(0), + _ts(0), + _id(0), + _thingCount(0) + { + } + + CertificateOfOwnership(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id) : + _networkId(nwid), + _ts(ts), + _flags(0), + _id(id), + _thingCount(0), + _issuedTo(issuedTo) + { + } + + inline uint64_t networkId() const { return _networkId; } + inline uint64_t timestamp() const { return _ts; } + inline uint32_t id() const { return _id; } + inline unsigned int thingCount() const { return (unsigned int)_thingCount; } + + inline Thing thingType(const unsigned int i) const { return (Thing)_thingTypes[i]; } + inline const uint8_t *thingValue(const unsigned int i) const { return _thingValues[i]; } + + inline const Address &issuedTo() const { return _issuedTo; } + + inline bool owns(const Thing &t,const void *v,unsigned int l) + { + for(unsigned int i=0,j=_thingCount;i(v)[k] != _thingValues[i][k]) + break; + ++k; + } + if (k == l) + return true; + } + } + return false; + } + + inline bool owns(const InetAddress &ip) + { + if (ip.ss_family == AF_INET) + return this->owns(THING_IPV4_ADDRESS,&(reinterpret_cast(&ip)->sin_addr.s_addr),4); + if (ip.ss_family == AF_INET6) + return this->owns(THING_IPV6_ADDRESS,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + return false; + } + + inline bool owns(const MAC &mac) + { + uint8_t tmp[6]; + mac.copyTo(tmp,6); + return this->owns(THING_MAC_ADDRESS,tmp,6); + } + + inline void addThing(const InetAddress &ip) + { + if (_thingCount >= ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS) return; + if (ip.ss_family == AF_INET) { + _thingTypes[_thingCount] = THING_IPV4_ADDRESS; + memcpy(_thingValues[_thingCount],&(reinterpret_cast(&ip)->sin_addr.s_addr),4); + ++_thingCount; + } else if (ip.ss_family == AF_INET6) { + _thingTypes[_thingCount] = THING_IPV6_ADDRESS; + memcpy(_thingValues[_thingCount],reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + ++_thingCount; + } + } + + inline void addThing(const MAC &mac) + { + if (_thingCount >= ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS) return; + _thingTypes[_thingCount] = THING_MAC_ADDRESS; + mac.copyTo(_thingValues[_thingCount],6); + ++_thingCount; + } + + /** + * @param signer Signing identity, must have private key + * @return True if signature was successful + */ + inline bool sign(const Identity &signer) + { + if (signer.hasPrivate()) { + Buffer tmp; + _signedBy = signer.address(); + this->serialize(tmp,true); + _signature = signer.sign(tmp.data(),tmp.size()); + return true; + } + return false; + } + + /** + * @param RR Runtime environment to allow identity lookup for signedBy + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature + */ + int verify(const RuntimeEnvironment *RR) const; + + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append(_networkId); + b.append(_ts); + b.append(_flags); + b.append(_id); + b.append((uint16_t)_thingCount); + for(unsigned int i=0,j=_thingCount;i + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + unsigned int p = startAt; + + memset(this,0,sizeof(CertificateOfOwnership)); + + _networkId = b.template at(p); p += 8; + _ts = b.template at(p); p += 8; + _flags = b.template at(p); p += 8; + _id = b.template at(p); p += 4; + _thingCount = b.template at(p); p += 2; + for(unsigned int i=0,j=_thingCount;i(p) != ZT_C25519_SIGNATURE_LEN) + throw std::runtime_error("invalid signature length"); + p += 2; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + } else { + p += 2 + b.template at(p); + } + + p += 2 + b.template at(p); + if (p > b.size()) + throw std::runtime_error("extended field overflow"); + + return (p - startAt); + } + + // Provides natural sort order by ID + inline bool operator<(const CertificateOfOwnership &coo) const { return (_id < coo._id); } + + inline bool operator==(const CertificateOfOwnership &coo) const { return (memcmp(this,&coo,sizeof(CertificateOfOwnership)) == 0); } + inline bool operator!=(const CertificateOfOwnership &coo) const { return (memcmp(this,&coo,sizeof(CertificateOfOwnership)) != 0); } + +private: + uint64_t _networkId; + uint64_t _ts; + uint64_t _flags; + uint32_t _id; + uint16_t _thingCount; + uint8_t _thingTypes[ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS]; + uint8_t _thingValues[ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS][ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE]; + Address _issuedTo; + Address _signedBy; + C25519::Signature _signature; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b077f7e2..b5b2bcb3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -832,6 +832,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S Capability cap; Tag tag; Revocation revocation; + CertificateOfOwnership coo; bool trustEstablished = false; unsigned int p = ZT_PACKET_IDX_PAYLOAD; @@ -909,6 +910,24 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } } + + const unsigned int numCoos = at(p); p += 2; + for(unsigned int i=0;i network(RR->node->network(coo.networkId())); + if (network) { + switch(network->addCredential(coo)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; + } + } + } } peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); diff --git a/node/Membership.cpp b/node/Membership.cpp index 8c6dab64..1eacb93d 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -37,6 +37,7 @@ Membership::Membership() : { for(unsigned int i=0;i= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCoos[c].lastPushed = now; + _localCoos[c].id = nconf.certificatesOfOwnership[c].id(); + sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]); + } + } + unsigned int tagPtr = 0; - while ((tagPtr < sendTagCount)||(sendCom)||(sendCap)) { + unsigned int cooPtr = 0; + while ((tagPtr < sendTagCount)||(cooPtr < sendCooCount)||(sendCom)||(sendCap)) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (sendCom) { @@ -82,7 +94,7 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now const unsigned int tagCountAt = outp.size(); outp.addSize(2); unsigned int thisPacketTagCount = 0; - while ((tagPtr < sendTagCount)&&((outp.size() + sizeof(Tag) + 32) < ZT_PROTO_MAX_PACKET_LENGTH)) { + while ((tagPtr < sendTagCount)&&((outp.size() + sizeof(Tag) + 16) < ZT_PROTO_MAX_PACKET_LENGTH)) { sendTags[tagPtr++]->serialize(outp); ++thisPacketTagCount; } @@ -91,6 +103,15 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now // No revocations, these propagate differently outp.append((uint16_t)0); + const unsigned int cooCountAt = outp.size(); + outp.addSize(2); + unsigned int thisPacketCooCount = 0; + while ((cooPtr < sendCooCount)&&((outp.size() + sizeof(CertificateOfOwnership) + 16) < ZT_PROTO_MAX_PACKET_LENGTH)) { + sendCoos[cooPtr++]->serialize(outp); + ++thisPacketCooCount; + } + outp.setAt(cooCountAt,(uint16_t)thisPacketCooCount); + outp.compress(); RR->sw->send(outp,true); } @@ -98,14 +119,14 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now const Capability *Membership::getCapability(const NetworkConfig &nconf,const uint32_t id) const { - const _RemoteCapability *const *c = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)id,_RemoteCredentialSorter<_RemoteCapability>()); - return ( ((c != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*c)->id == (uint64_t)id)) ? ((((*c)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*c)->cap,**c))) ? &((*c)->cap) : (const Capability *)0) : (const Capability *)0); + const _RemoteCredential *const *c = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)id,_RemoteCredentialComp()); + return ( ((c != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*c)->id == (uint64_t)id)) ? ((((*c)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*c)->credential,**c))) ? &((*c)->credential) : (const Capability *)0) : (const Capability *)0); } const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) const { - const _RemoteTag *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialSorter<_RemoteTag>()); - return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*t)->tag,**t))) ? &((*t)->tag) : (const Tag *)0) : (const Tag *)0); + const _RemoteCredential *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialComp()); + return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*t)->credential,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com) @@ -141,14 +162,14 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag) { - _RemoteTag *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialSorter<_RemoteTag>()); - _RemoteTag *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteTag *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteCredential *)0; if (have) { - if ( (!_isCredentialTimestampValid(nconf,tag,*have)) || (have->tag.timestamp() > tag.timestamp()) ) { + if ( (!_isCredentialTimestampValid(nconf,tag,*have)) || (have->credential.timestamp() > tag.timestamp()) ) { TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); return ADD_REJECTED; } - if (have->tag == tag) { + if (have->credential == tag) { TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); return ADD_ACCEPTED_REDUNDANT; } @@ -162,7 +183,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); if (!have) have = _newTag(tag.id()); have->lastReceived = RR->node->now(); - have->tag = tag; + have->credential = tag; return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -171,14 +192,14 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap) { - _RemoteCapability *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialSorter<_RemoteCapability>()); - _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCapability *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCredential *)0; if (have) { - if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->cap.timestamp() > cap.timestamp()) ) { + if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->credential.timestamp() > cap.timestamp()) ) { TRACE("addCredential(Capability) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; } - if (have->cap == cap) { + if (have->credential == cap) { TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_ACCEPTED_REDUNDANT; } @@ -192,7 +213,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); if (!have) have = _newCapability(cap.id()); have->lastReceived = RR->node->now(); - have->cap = cap; + have->credential = cap; return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -209,13 +230,15 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme switch(rev.type()) { default: //case Revocation::CREDENTIAL_TYPE_ALL: - return ( (_revokeCom(rev)||_revokeCap(rev,now)||_revokeTag(rev,now)) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT ); + return ( (_revokeCom(rev)||_revokeCap(rev,now)||_revokeTag(rev,now)||_revokeCoo(rev,now)) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT ); case Revocation::CREDENTIAL_TYPE_COM: return (_revokeCom(rev) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); case Revocation::CREDENTIAL_TYPE_CAPABILITY: return (_revokeCap(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); case Revocation::CREDENTIAL_TYPE_TAG: return (_revokeTag(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); + case Revocation::CREDENTIAL_TYPE_COO: + return (_revokeCoo(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); } } case 1: @@ -223,9 +246,40 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::_RemoteTag *Membership::_newTag(const uint64_t id) + +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo) +{ + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; + if (have) { + if ( (!_isCredentialTimestampValid(nconf,coo,*have)) || (have->credential.timestamp() > coo.timestamp()) ) { + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); + return ADD_REJECTED; + } + if (have->credential == coo) { + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } + } + + switch(coo.verify(RR)) { + default: + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); + return ADD_REJECTED; + case 0: + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); + if (!have) have = _newCoo(coo.id()); + have->lastReceived = RR->node->now(); + have->credential = coo; + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; + } +} + +Membership::_RemoteCredential *Membership::_newTag(const uint64_t id) { - _RemoteTag *t = NULL; + _RemoteCredential *t = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -236,21 +290,21 @@ Membership::_RemoteTag *Membership::_newTag(const uint64_t id) minlr = _remoteTags[i]->lastReceived; } } - - if (t) { - t->id = id; - t->lastReceived = 0; - t->revocationThreshold = 0; - t->tag = Tag(); - } - - std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); + + if (t) { + t->id = id; + t->lastReceived = 0; + t->revocationThreshold = 0; + t->credential = Tag(); + } + + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS])); return t; } -Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) +Membership::_RemoteCredential *Membership::_newCapability(const uint64_t id) { - _RemoteCapability *c = NULL; + _RemoteCredential *c = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -266,10 +320,35 @@ Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) c->id = id; c->lastReceived = 0; c->revocationThreshold = 0; - c->cap = Capability(); + c->credential = Capability(); + } + + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES])); + return c; +} + +Membership::_RemoteCredential *Membership::_newCoo(const uint64_t id) +{ + _RemoteCredential *c = NULL; + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { + c = _remoteCoos[i]; + break; + } else if (_remoteCoos[i]->lastReceived <= minlr) { + c = _remoteCoos[i]; + minlr = _remoteCoos[i]->lastReceived; + } + } + + if (c) { + c->id = id; + c->lastReceived = 0; + c->revocationThreshold = 0; + c->credential = CertificateOfOwnership(); } - std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); + std::sort(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP])); return c; } @@ -284,8 +363,8 @@ bool Membership::_revokeCom(const Revocation &rev) bool Membership::_revokeCap(const Revocation &rev,const uint64_t now) { - _RemoteCapability *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)rev.credentialId(),_RemoteCredentialSorter<_RemoteCapability>()); - _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCapability *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; if (!have) have = _newCapability(rev.credentialId()); if (rev.threshold() > have->revocationThreshold) { have->lastReceived = now; @@ -297,8 +376,8 @@ bool Membership::_revokeCap(const Revocation &rev,const uint64_t now) bool Membership::_revokeTag(const Revocation &rev,const uint64_t now) { - _RemoteTag *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)rev.credentialId(),_RemoteCredentialSorter<_RemoteTag>()); - _RemoteTag *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteTag *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; if (!have) have = _newTag(rev.credentialId()); if (rev.threshold() > have->revocationThreshold) { have->lastReceived = now; @@ -308,4 +387,17 @@ bool Membership::_revokeTag(const Revocation &rev,const uint64_t now) return false; } +bool Membership::_revokeCoo(const Revocation &rev,const uint64_t now) +{ + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; + if (!have) have = _newCoo(rev.credentialId()); + if (rev.threshold() > have->revocationThreshold) { + have->lastReceived = now; + have->revocationThreshold = rev.threshold(); + return true; + } + return false; +} + } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 9814dce8..4e9d7769 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -39,49 +39,30 @@ class Network; /** * A container for certificates of membership and other network credentials * - * This is kind of analogous to a join table between Peer and Network. It is - * held by the Network object for each participating Peer. + * This is essentially a relational join between Peer and Network. * * This class is not thread safe. It must be locked externally. */ class Membership { private: - // Tags and related state - struct _RemoteTag - { - _RemoteTag() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} - // Tag ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) - uint64_t id; - // Last time we received THEIR tag (with this ID) - uint64_t lastReceived; - // Revocation blacklist threshold or 0 if none - uint64_t revocationThreshold; - // THEIR tag - Tag tag; - }; - - // Credentials and related state - struct _RemoteCapability + template + struct _RemoteCredential { - _RemoteCapability() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} - // Capability ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) + _RemoteCredential() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} uint64_t id; - // Last time we received THEIR capability (with this ID) - uint64_t lastReceived; - // Revocation blacklist threshold or 0 if none - uint64_t revocationThreshold; - // THEIR capability - Capability cap; + uint64_t lastReceived; // last time we got this credential + uint64_t revocationThreshold; // credentials before this time are invalid + T credential; + inline bool operator<(const _RemoteCredential &c) const { return (id < c.id); } }; - // Comparison operator for remote credential entries template - struct _RemoteCredentialSorter + struct _RemoteCredentialComp { - inline bool operator()(const T *a,const T *b) const { return (a->id < b->id); } - inline bool operator()(const uint64_t a,const T *b) const { return (a < b->id); } - inline bool operator()(const T *a,const uint64_t b) const { return (a->id < b); } + inline bool operator()(const _RemoteCredential *a,const _RemoteCredential *b) const { return (a->id < b->id); } + inline bool operator()(const uint64_t a,const _RemoteCredential *b) const { return (a < b->id); } + inline bool operator()(const _RemoteCredential *a,const uint64_t b) const { return (a->id < b); } inline bool operator()(const uint64_t a,const uint64_t b) const { return (a < b); } }; @@ -89,8 +70,8 @@ private: struct _LocalCredentialPushState { _LocalCredentialPushState() : lastPushed(0),id(0) {} - uint64_t lastPushed; - uint32_t id; + uint64_t lastPushed; // last time we sent our own copy of this credential + uint64_t id; }; public: @@ -117,7 +98,7 @@ public: { for(;;) { if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Capability *tmp = &((*_i)->cap); + const Capability *tmp = &((*_i)->credential); if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { ++_i; return tmp; @@ -131,7 +112,7 @@ public: private: const Membership *_m; const NetworkConfig *_c; - const _RemoteCapability *const *_i; + const _RemoteCredential *const *_i; }; friend class CapabilityIterator; @@ -150,7 +131,7 @@ public: { for(;;) { if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Tag *tmp = &((*_i)->tag); + const Tag *tmp = &((*_i)->credential); if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { ++_i; return tmp; @@ -164,7 +145,7 @@ public: private: const Membership *_m; const NetworkConfig *_c; - const _RemoteTag *const *_i; + const _RemoteCredential *const *_i; }; friend class TagIterator; @@ -249,12 +230,19 @@ public: */ AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev); + /** + * Validate and add a credential if signature is okay and it's otherwise good + */ + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo); + private: - _RemoteTag *_newTag(const uint64_t id); - _RemoteCapability *_newCapability(const uint64_t id); + _RemoteCredential *_newTag(const uint64_t id); + _RemoteCredential *_newCapability(const uint64_t id); + _RemoteCredential *_newCoo(const uint64_t id); bool _revokeCom(const Revocation &rev); bool _revokeCap(const Revocation &rev,const uint64_t now); bool _revokeTag(const Revocation &rev,const uint64_t now); + bool _revokeCoo(const Revocation &rev,const uint64_t now); template inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred,const CS &state) const @@ -275,17 +263,20 @@ private: // Remote member's latest network COM CertificateOfMembership _com; - // Sorted (in ascending order of ID) arrays of pointers to remote tags and capabilities - _RemoteTag *_remoteTags[ZT_MAX_NETWORK_TAGS]; - _RemoteCapability *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; + // Sorted (in ascending order of ID) arrays of pointers to remote credentials + _RemoteCredential *_remoteTags[ZT_MAX_NETWORK_TAGS]; + _RemoteCredential *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; + _RemoteCredential *_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; - // This is the RAM allocated for remote tags and capabilities from which the sorted arrays are populated - _RemoteTag _tagMem[ZT_MAX_NETWORK_TAGS]; - _RemoteCapability _capMem[ZT_MAX_NETWORK_CAPABILITIES]; + // This is the RAM allocated for remote credential cache objects + _RemoteCredential _tagMem[ZT_MAX_NETWORK_TAGS]; + _RemoteCredential _capMem[ZT_MAX_NETWORK_CAPABILITIES]; + _RemoteCredential _cooMem[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; // Local credential push state tracking _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; _LocalCredentialPushState _localCaps[ZT_MAX_NETWORK_CAPABILITIES]; + _LocalCredentialPushState _localCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; }; } // namespace ZeroTier diff --git a/node/Network.hpp b/node/Network.hpp index 85ee6e9a..56c7fc60 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -301,6 +301,17 @@ public: */ Membership::AddCredentialResult addCredential(const Address &sentFrom,const Revocation &rev); + /** + * Validate a credential and learn it if it passes certificate and other checks + */ + inline Membership::AddCredentialResult addCredential(const CertificateOfOwnership &coo) + { + if (coo.networkId() != _id) + return Membership::ADD_REJECTED; + Mutex::Lock _l(_lock); + return _membership(coo.issuedTo()).addCredential(RR,_config,coo); + } + /** * Force push credentials (COM, etc.) to a peer now * diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 2f356b15..fe7393e8 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -21,7 +21,6 @@ #include #include "NetworkConfig.hpp" -#include "Utils.hpp" namespace ZeroTier { @@ -137,6 +136,13 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TAGS,*tmp)) return false; } + tmp->clear(); + for(unsigned int i=0;icertificateOfOwnershipCount;++i) + this->certificatesOfOwnership[i].serialize(*tmp); + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP,*tmp)) return false; + } + tmp->clear(); for(unsigned int i=0;ispecialistCount;++i) tmp->append((uint64_t)this->specialists[i]); @@ -297,10 +303,23 @@ bool NetworkConfig::fromDictionary(const Dictionarytags[0]),&(this->tags[this->tagCount])); } + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP,*tmp)) { + unsigned int p = 0; + while (p < tmp->size()) { + if (certificateOfOwnershipCount < ZT_MAX_CERTIFICATES_OF_OWNERSHIP) + p += certificatesOfOwnership[certificateOfOwnershipCount++].deserialize(*tmp,p); + else { + CertificateOfOwnership foo; + p += foo.deserialize(*tmp,p); + } + } + } + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_SPECIALISTS,*tmp)) { unsigned int p = 0; - while (((p + 8) <= tmp->size())&&(specialistCount < ZT_MAX_NETWORK_SPECIALISTS)) { - this->specialists[this->specialistCount++] = tmp->at(p); + while ((p + 8) <= tmp->size()) { + if (specialistCount < ZT_MAX_NETWORK_SPECIALISTS) + this->specialists[this->specialistCount++] = tmp->at(p); p += 8; } } diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 39087395..85c24090 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -35,10 +35,12 @@ #include "MulticastGroup.hpp" #include "Address.hpp" #include "CertificateOfMembership.hpp" +#include "CertificateOfOwnership.hpp" #include "Capability.hpp" #include "Tag.hpp" #include "Dictionary.hpp" #include "Identity.hpp" +#include "Utils.hpp" /** * Default maximum time delta for COMs, tags, and capabilities @@ -99,7 +101,7 @@ namespace ZeroTier { // Dictionary capacity needed for max size network config -#define ZT_NETWORKCONFIG_DICT_CAPACITY (4096 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS)) +#define ZT_NETWORKCONFIG_DICT_CAPACITY (1024 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP)) // Dictionary capacity needed for max size network meta-data #define ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY 1024 @@ -173,6 +175,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_CAPABILITIES "CAP" // tags (binary blobs) #define ZT_NETWORKCONFIG_DICT_KEY_TAGS "TAG" +// tags (binary blobs) +#define ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP "COO" // curve25519 signature #define ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE "C25519" @@ -473,11 +477,6 @@ public: */ unsigned int staticIpCount; - /** - * Number of pinned devices (devices with physical address hints) - */ - unsigned int pinnedCount; - /** * Number of rule table entries */ @@ -493,6 +492,11 @@ public: */ unsigned int tagCount; + /** + * Number of certificates of ownership + */ + unsigned int certificateOfOwnershipCount; + /** * Specialist devices * @@ -526,6 +530,11 @@ public: */ Tag tags[ZT_MAX_NETWORK_TAGS]; + /** + * Certificates of ownership for this network member + */ + CertificateOfOwnership certificatesOfOwnership[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + /** * Network type (currently just public or private) */ diff --git a/node/Packet.hpp b/node/Packet.hpp index b736b84a..6482356a 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -730,6 +730,8 @@ public: * <[...] one or more serialized Tags> * <[2] 16-bit number of revocations> * <[...] one or more serialized Revocations> + * <[2] 16-bit number of certificates of ownership> + * <[...] one or more serialized CertificateOfOwnership> * * This can be sent by anyone at any time to push network credentials. * These will of course only be accepted if they are properly signed. diff --git a/node/Revocation.hpp b/node/Revocation.hpp index bc290e75..3903f440 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -50,9 +50,10 @@ public: enum CredentialType { CREDENTIAL_TYPE_ALL = 0, - CREDENTIAL_TYPE_COM = 1, + CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership CREDENTIAL_TYPE_CAPABILITY = 2, - CREDENTIAL_TYPE_TAG = 3 + CREDENTIAL_TYPE_TAG = 3, + CREDENTIAL_TYPE_COO = 4 // CertificateOfOwnership }; Revocation() diff --git a/node/Tag.hpp b/node/Tag.hpp index 65348200..146e8da9 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -139,7 +139,8 @@ public: { unsigned int p = startAt; - // These are the same between Tag and Capability + memset(this,0,sizeof(Tag)); + _networkId = b.template at(p); p += 8; _ts = b.template at(p); p += 8; _id = b.template at(p); p += 4; diff --git a/objects.mk b/objects.mk index 31498b72..427024eb 100644 --- a/objects.mk +++ b/objects.mk @@ -4,6 +4,7 @@ OBJS=\ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ + node/CertificateOfOwnership.o \ node/Cluster.o \ node/Identity.o \ node/IncomingPacket.o \ -- cgit v1.2.3 From 2b10a982e9c9aad4a6ed9b10f5d7bda93a55ffcf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 28 Feb 2017 09:22:10 -0800 Subject: Match on tag sender equals or tag recipient equals. --- controller/EmbeddedNetworkController.cpp | 36 ++++++++++++++++++++---------- include/ZeroTierOne.h | 2 ++ node/Capability.hpp | 8 ++++--- node/Network.cpp | 38 +++++++++++++++++++++++++++++++- rule-compiler/package.json | 2 +- rule-compiler/rule-compiler.js | 14 +++++++++--- 6 files changed, 80 insertions(+), 20 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 78a9b7c7..0e57fc14 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -218,6 +218,16 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; break; + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + r["type"] = "MATCH_TAG_SENDER"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: + r["type"] = "MATCH_TAG_RECEIVER"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; default: break; } @@ -245,6 +255,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) if (OSUtils::jsonBool(r["or"],false)) rule.t |= 0x40; + bool tag = false; if (t == "ACTION_DROP") { rule.t |= ZT_NETWORK_RULE_ACTION_DROP; return true; @@ -388,26 +399,27 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_TAGS_DIFFERENCE") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_EQUAL") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_EQUAL; + tag = true; + } else if (t == "MATCH_TAG_SENDER") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAG_SENDER; + tag = true; + } else if (t == "MATCH_TAG_RECEIVER") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAG_RECEIVER; + tag = true; + } + if (tag) { rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); return true; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 8a13c117..2c141f47 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -604,6 +604,8 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 46, ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 47, ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 48, + ZT_NETWORK_RULE_MATCH_TAG_SENDER = 49, + ZT_NETWORK_RULE_MATCH_TAG_RECEIVER = 50, /** * Maximum ID allowed for a MATCH entry in the rules table diff --git a/node/Capability.hpp b/node/Capability.hpp index d884625e..1ad6ea42 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -167,9 +167,6 @@ public: // rules to be ignored but still parsed. b.append((uint8_t)rules[i].t); switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x3f)) { - //case ZT_NETWORK_RULE_ACTION_DROP: - //case ZT_NETWORK_RULE_ACTION_ACCEPT: - //case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: default: b.append((uint8_t)0); break; @@ -258,6 +255,9 @@ public: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: b.append((uint8_t)8); b.append((uint32_t)rules[i].v.tag.id); b.append((uint32_t)rules[i].v.tag.value); @@ -345,6 +345,8 @@ public: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: rules[ruleCount].v.tag.id = b.template at(p); rules[ruleCount].v.tag.value = b.template at(p + 4); break; diff --git a/node/Network.cpp b/node/Network.cpp index aad6e716..dc976f03 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -515,7 +515,6 @@ static _doZtFilterResult _doZtFilter( src.set((const void *)(frameData + 12),4,0); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { // IPv6 NDP requires special handling, since the src and dest IPs in the packet are empty or link-local. - unsigned int pos = 0,proto = 0; if ( (frameLen >= (40 + 8 + 16)) && (frameData[6] == 0x3a) && ((frameData[40] == 0x87)||(frameData[40] == 0x88)) ) { if (frameData[40] == 0x87) { // Neighbor solicitations contain no reliable source address, so we implement a small @@ -609,6 +608,11 @@ static _doZtFilterResult _doZtFilter( thisRuleMatches = 0; FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { + // Outbound side is not strict since if we have to match both tags and + // we are sending a first packet to a recipient, we probably do not know + // about their tags yet. They will filter on inbound and we will filter + // once we get their tag. If we are a tee/redirect target we are also + // not strict since we likely do not have these tags. thisRuleMatches = 1; FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side and TEE/REDIRECT targets are not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } @@ -618,6 +622,38 @@ static _doZtFilterResult _doZtFilter( FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } break; + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: { + if (superAccept) { + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c we are a TEE/REDIRECT target -> 1",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } else if ( ((rt == ZT_NETWORK_RULE_MATCH_TAG_SENDER)&&(inbound)) || ((rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER)&&(!inbound)) ) { + const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0); + if (remoteTag) { + thisRuleMatches = (uint8_t)(remoteTag->value() == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + } else { + if (rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER) { + // If we are checking the receiver and this is an outbound packet, we + // can't be strict since we may not yet know the receiver's tag. + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c (inbound) remote tag %u not found -> 1 (outbound receiver match is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c (inbound) remote tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } + } + } else { // sender and outbound or receiver and inbound + const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); + if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { + thisRuleMatches = (uint8_t)(localTag->value() == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } + } + } break; // The result of an unsupported MATCH is configurable at the network // level via a flag. diff --git a/rule-compiler/package.json b/rule-compiler/package.json index f0e747a2..451295a4 100644 --- a/rule-compiler/package.json +++ b/rule-compiler/package.json @@ -1,6 +1,6 @@ { "name": "zerotier-rule-compiler", - "version": "1.1.17-2", + "version": "1.1.17-3", "description": "ZeroTier Rule Script Compiler", "main": "cli.js", "scripts": { diff --git a/rule-compiler/rule-compiler.js b/rule-compiler/rule-compiler.js index 7445d39f..feb30f90 100644 --- a/rule-compiler/rule-compiler.js +++ b/rule-compiler/rule-compiler.js @@ -105,6 +105,8 @@ const RESERVED_WORDS = { 'txor': true, 'tdiff': true, 'teq': true, + 'tseq': true, + 'treq': true, 'type': true, 'enum': true, @@ -152,7 +154,9 @@ const KEYWORD_TO_API_MAP = { 'tor': 'MATCH_TAGS_BITWISE_OR', 'txor': 'MATCH_TAGS_BITWISE_XOR', 'tdiff': 'MATCH_TAGS_DIFFERENCE', - 'teq': 'MATCH_TAGS_EQUAL' + 'teq': 'MATCH_TAGS_EQUAL', + 'tseq': 'MATCH_TAG_SENDER', + 'treq': 'MATCH_TAG_RECEIVER' }; // Number of args for each match @@ -179,7 +183,9 @@ const MATCH_ARG_COUNTS = { 'tor': 2, 'txor': 2, 'tdiff': 2, - 'teq': 2 + 'teq': 2, + 'tseq': 2, + 'treq': 2 }; // Regex of all alphanumeric characters in Unicode @@ -477,7 +483,9 @@ function _renderMatches(mtree,rules,macros,caps,tags,params) case 'tor': case 'txor': case 'tdiff': - case 'teq': { + case 'teq': + case 'tseq': + case 'treq': { let tag = tags[args[0][0]]; let tagId = -1; let tagValue = -1; -- cgit v1.2.3 From a577b8d3816069a448a946302048a377b55cd74a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Mar 2017 16:33:34 -0800 Subject: Update how controller handles circuit tests -- save results to filesystem. --- controller/EmbeddedNetworkController.cpp | 137 +++++++++++++++---------------- controller/EmbeddedNetworkController.hpp | 15 ++-- controller/JSONDB.cpp | 25 ++++-- controller/JSONDB.hpp | 6 +- controller/README.md | 11 +-- node/Peer.cpp | 6 +- service/OneService.cpp | 2 +- 7 files changed, 100 insertions(+), 102 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0e57fc14..7915765b 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -428,9 +428,9 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return false; } -EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath,FILE *feed) : +EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _threadsStarted(false), - _db(dbPath,feed), + _db(dbPath), _node(node) { OSUtils::mkdir(dbPath); @@ -546,21 +546,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( return 200; } - } else if ((path[2] == "test")&&(path.size() >= 4)) { - - Mutex::Lock _l(_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); - if ((cte != _circuitTests.end())&&(cte->second.test)) { - - responseBody = "["; - responseBody.append(cte->second.jsonResults); - responseBody.push_back(']'); - responseContentType = "application/json"; - - return 200; - - } // else 404 - } // else 404 } else { @@ -755,9 +740,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( return 200; } else if ((path.size() == 3)&&(path[2] == "test")) { - Mutex::Lock _l(_circuitTests_m); + Mutex::Lock _l(_tests_m); - ZT_CircuitTest *test = (ZT_CircuitTest *)malloc(sizeof(ZT_CircuitTest)); + _tests.push_back(ZT_CircuitTest()); + ZT_CircuitTest *const test = &(_tests.back()); memset(test,0,sizeof(ZT_CircuitTest)); Utils::getSecureRandom(&(test->testId),sizeof(test->testId)); @@ -781,7 +767,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( test->reportAtEveryHop = (OSUtils::jsonBool(b["reportAtEveryHop"],true) ? 1 : 0); if (!test->hopCount) { - ::free((void *)test); + _tests.pop_back(); responseBody = "{ \"message\": \"a test must contain at least one hop\" }"; responseContentType = "application/json"; return 400; @@ -789,18 +775,18 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( test->timestamp = OSUtils::now(); - _CircuitTestEntry &te = _circuitTests[test->testId]; - te.test = test; - te.jsonResults = ""; - - if (_node) + if (_node) { _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); - else return 500; + } else { + _tests.pop_back(); + return 500; + } char json[1024]; Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); responseBody = json; responseContentType = "application/json"; + return 200; } // else 404 @@ -1137,62 +1123,67 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::threadMain() throw() { + uint64_t lastCircuitTestCheck = 0; for(;;) { - _RQEntry *const qe = _queue.get(); + _RQEntry *const qe = _queue.get(); // waits on next request if (!qe) break; // enqueue a NULL to terminate threads try { _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); } catch ( ... ) {} delete qe; + + uint64_t now = OSUtils::now(); + if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + lastCircuitTestCheck = now; + Mutex::Lock _l(_tests_m); + for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { + if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + _node->circuitTestEnd(&(*i)); + _tests.erase(i++); + } else ++i; + } + } } } void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) { - char tmp[65535]; + char tmp[1024],id[128]; EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); - if (!test) - return; - if (!report) - return; - - Mutex::Lock _l(self->_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); - - if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? - self->_node->circuitTestEnd(test); - ::free((void *)test); - return; - } + if ((!test)||(!report)||(!test->credentialNetworkId)) return; // sanity check + const uint64_t now = OSUtils::now(); + Utils::snprintf(id,sizeof(id),"network/%.16llx/test/%.16llx-%.16llx-%.10llx-%.10llx",test->credentialNetworkId,test->testId,now,report->upstream,report->current); Utils::snprintf(tmp,sizeof(tmp), - "%s{\n" - "\t\"timestamp\": %llu," ZT_EOL_S - "\t\"testId\": \"%.16llx\"," ZT_EOL_S - "\t\"upstream\": \"%.10llx\"," ZT_EOL_S - "\t\"current\": \"%.10llx\"," ZT_EOL_S - "\t\"receivedTimestamp\": %llu," ZT_EOL_S - "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S - "\t\"flags\": %llu," ZT_EOL_S - "\t\"sourcePacketHopCount\": %u," ZT_EOL_S - "\t\"errorCode\": %u," ZT_EOL_S - "\t\"vendor\": %d," ZT_EOL_S - "\t\"protocolVersion\": %u," ZT_EOL_S - "\t\"majorVersion\": %u," ZT_EOL_S - "\t\"minorVersion\": %u," ZT_EOL_S - "\t\"revision\": %u," ZT_EOL_S - "\t\"platform\": %d," ZT_EOL_S - "\t\"architecture\": %d," ZT_EOL_S - "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S - "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S - "}", - ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), - (unsigned long long)report->timestamp, + "{\"id\": \"%s\"," + "\"timestamp\": %llu," + "\"networkId\": \"%.16llx\"," + "\"testId\": \"%.16llx\"," + "\"upstream\": \"%.10llx\"," + "\"current\": \"%.10llx\"," + "\"receivedTimestamp\": %llu," + "\"sourcePacketId\": \"%.16llx\"," + "\"flags\": %llu," + "\"sourcePacketHopCount\": %u," + "\"errorCode\": %u," + "\"vendor\": %d," + "\"protocolVersion\": %u," + "\"majorVersion\": %u," + "\"minorVersion\": %u," + "\"revision\": %u," + "\"platform\": %d," + "\"architecture\": %d," + "\"receivedOnLocalAddress\": \"%s\"," + "\"receivedFromRemoteAddress\": \"%s\"," + "\"receivedFromLinkQuality\": %f}", + id + 30, // last bit only, not leading path + (unsigned long long)test->timestamp, + (unsigned long long)test->credentialNetworkId, (unsigned long long)test->testId, (unsigned long long)report->upstream, (unsigned long long)report->current, - (unsigned long long)OSUtils::now(), + (unsigned long long)now, (unsigned long long)report->sourcePacketId, (unsigned long long)report->flags, report->sourcePacketHopCount, @@ -1205,9 +1196,11 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes (int)report->platform, (int)report->architecture, reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), - reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); + reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str(), + ((double)report->receivedFromLinkQuality / (double)ZT_PATH_LINK_QUALITY_MAX)); - cte->second.jsonResults.append(tmp); + Mutex::Lock _l(self->_db_m); + self->_db.writeRaw(id,std::string(tmp)); } void EmbeddedNetworkController::_request( @@ -1354,12 +1347,12 @@ void EmbeddedNetworkController::_request( if (requestPacketId) { // only log if this is a request, not for generated pushes json rlEntry = json::object(); rlEntry["ts"] = now; - rlEntry["authorized"] = (authorizedBy) ? true : false; - rlEntry["authorizedBy"] = (authorizedBy) ? authorizedBy : ""; - rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); + rlEntry["auth"] = (authorizedBy) ? true : false; + rlEntry["authBy"] = (authorizedBy) ? authorizedBy : ""; + rlEntry["vMajor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + rlEntry["vMinor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + rlEntry["vRev"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + rlEntry["vProto"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); if (fromAddr) rlEntry["fromAddr"] = fromAddr.toString(); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 3e39eaf5..ab7cdd53 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -46,6 +46,9 @@ // Number of background threads to start -- not actually started until needed #define ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT 2 +// TTL for circuit tests +#define ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION 120000 + namespace ZeroTier { class Node; @@ -56,9 +59,8 @@ public: /** * @param node Parent node * @param dbPath Path to store data - * @param feed FILE to send feed of all data and changes to (zero-delimited JSON objects) or NULL for none */ - EmbeddedNetworkController(Node *node,const char *dbPath,FILE *feed); + EmbeddedNetworkController(Node *node,const char *dbPath); virtual ~EmbeddedNetworkController(); virtual void init(const Identity &signingId,Sender *sender); @@ -199,13 +201,8 @@ private: NetworkController::Sender *_sender; Identity _signingId; - struct _CircuitTestEntry - { - ZT_CircuitTest *test; - std::string jsonResults; - }; - std::map< uint64_t,_CircuitTestEntry > _circuitTests; - Mutex _circuitTests_m; + std::list< ZT_CircuitTest > _tests; + Mutex _tests_m; std::map< std::pair,uint64_t > _lastRequestTime; Mutex _lastRequestTime_m; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 044f791c..1277aabb 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -22,6 +22,22 @@ namespace ZeroTier { static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); +bool JSONDB::writeRaw(const std::string &n,const std::string &obj) +{ + if (!_isValidObjectName(n)) + return false; + + const std::string path(_genPath(n,true)); + if (!path.length()) + return false; + + const std::string buf(obj); + if (!OSUtils::writeFile(path.c_str(),buf)) + return false; + + return true; +} + bool JSONDB::put(const std::string &n,const nlohmann::json &obj) { if (!_isValidObjectName(n)) @@ -35,9 +51,6 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) if (!OSUtils::writeFile(path.c_str(),buf)) return false; - if (_feed) - fwrite(buf.c_str(),buf.length()+1,1,_feed); - _E &e = _db[n]; e.obj = obj; e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); @@ -72,9 +85,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe e->second.obj = OSUtils::jsonParse(buf); e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP e->second.lastCheck = now; - - if (_feed) - fwrite(buf.c_str(),buf.length()+1,1,_feed); // it changed, so send to feed (also sends all objects on startup, which we want for Central) } catch ( ... ) {} // parse errors result in "holding pattern" behavior } } @@ -99,9 +109,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe e2.lastModifiedOnDisk = lm; e2.lastCheck = now; - if (_feed) - fwrite(buf.c_str(),buf.length()+1,1,_feed); - return e2.obj; } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 40113655..5b7c5e50 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -42,8 +42,7 @@ namespace ZeroTier { class JSONDB { public: - JSONDB(const std::string &basePath,FILE *feed) : - _feed(feed), + JSONDB(const std::string &basePath) : _basePath(basePath) { _reload(_basePath); @@ -55,6 +54,8 @@ public: _reload(_basePath); } + bool writeRaw(const std::string &n,const std::string &obj); + bool put(const std::string &n,const nlohmann::json &obj); inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } @@ -108,7 +109,6 @@ private: inline bool operator!=(const _E &e) const { return (obj != e.obj); } }; - FILE *_feed; std::string _basePath; std::map _db; }; diff --git a/controller/README.md b/controller/README.md index 093300a6..db8d0153 100644 --- a/controller/README.md +++ b/controller/README.md @@ -237,11 +237,12 @@ Note that managed IP assignments are only used if they fall within a managed rou | Field | Type | Description | | --------------------- | ------------- | ------------------------------------------------- | | ts | integer | Time of request, ms since epoch | -| authorized | boolean | Was member authorized? | -| clientMajorVersion | integer | Client major version or -1 if unknown | -| clientMinorVersion | integer | Client minor version or -1 if unknown | -| clientRevision | integer | Client revision or -1 if unknown | -| clientProtocolVersion | integer | ZeroTier protocol version reported by client | +| auth | boolean | Was member authorized? | +| authBy | string | How was member authorized? | +| vMajor | integer | Client major version or -1 if unknown | +| vMinor | integer | Client minor version or -1 if unknown | +| vRev | integer | Client revision or -1 if unknown | +| vProto | integer | ZeroTier protocol version reported by client | | fromAddr | string | Physical address if known | The controller can only know a member's `fromAddr` if it's able to establish a direct path to it. Members behind very restrictive firewalls may not have this information since the controller will be receiving the member's requests by way of a relay. ZeroTier does not back-trace IP paths as packets are relayed since this would add a lot of protocol overhead. diff --git a/node/Peer.cpp b/node/Peer.cpp index 1dde8b65..fa3ce6c8 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -141,10 +141,10 @@ void Peer::received( path->trustedPacketReceived(now); } - if (hops == 0) { - if (_vProto >= 9) - path->updateLinkQuality((unsigned int)(packetId & 7)); + if (_vProto >= 9) + path->updateLinkQuality((unsigned int)(packetId & 7)); + if (hops == 0) { bool pathIsConfirmed = false; { Mutex::Lock _l(_paths_m); diff --git a/service/OneService.cpp b/service/OneService.cpp index 81950e26..8d8856a2 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -638,7 +638,7 @@ public: return _termReason; } - _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str(),(FILE *)0); + _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str()); _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER -- cgit v1.2.3 From a109d341ef72149dad5eae0b616a1f47d24487f9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 2 Mar 2017 14:35:38 -0800 Subject: Send timestamp with new circuit test response. --- controller/EmbeddedNetworkController.cpp | 6 +++--- controller/EmbeddedNetworkController.hpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7915765b..b7360220 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -782,8 +782,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( return 500; } - char json[1024]; - Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); + char json[512]; + Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\",\"timestamp\":%llu}",test->testId,test->timestamp); responseBody = json; responseContentType = "application/json"; @@ -1808,7 +1808,7 @@ void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,con bool online; { Mutex::Lock _l(_lastRequestTime_m); - std::map,uint64_t>::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); + std::map< std::pair,uint64_t >::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); } if (online) { diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index ab7cdd53..7163c004 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -204,7 +204,7 @@ private: std::list< ZT_CircuitTest > _tests; Mutex _tests_m; - std::map< std::pair,uint64_t > _lastRequestTime; + std::map< std::pair,uint64_t > _lastRequestTime; // last request time by Mutex _lastRequestTime_m; }; -- cgit v1.2.3 From 66dfc33de91577012bb0e9ec22d2ef6bf18805ef Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Mar 2017 11:23:46 -0800 Subject: Fix circuit test post in controller. --- controller/EmbeddedNetworkController.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b7360220..78fa79f2 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -758,9 +758,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( std::string s = hops2[j]; test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; } + ++test->hopCount; } else if (hops2.is_string()) { std::string s = hops2; test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; + ++test->hopCount; } } } -- cgit v1.2.3 From 5e6a4e5f5e0022dccbc2f6cf8a8b38c038720866 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Mar 2017 15:12:28 -0800 Subject: Send revocations automatically on deauth for instant kill, also fix some issues with the RP. --- controller/EmbeddedNetworkController.cpp | 16 ++++++++++++++-- node/Membership.hpp | 2 +- node/Network.cpp | 2 +- node/NetworkController.hpp | 11 ++++++++++- node/Node.cpp | 18 ++++++++++++++++++ node/Node.hpp | 1 + node/Packet.hpp | 3 +-- node/Revocation.hpp | 2 +- 8 files changed, 47 insertions(+), 8 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 78fa79f2..2f6142a9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -661,6 +661,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( ah["ct"] = json(); ah["c"] = json(); member["authHistory"].push_back(ah); + + // Member is being de-authorized, so spray Revocation objects to all online members + if (!newAuth) { + Revocation rev(_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); + rev.sign(_signingId); + Mutex::Lock _l(_lastRequestTime_m); + for(std::map< std::pair,uint64_t >::iterator i(_lastRequestTime.begin());i!=_lastRequestTime.end();++i) { + if ((now - i->second) < ZT_NETWORK_AUTOCONF_DELAY) + _node->ncSendRevocation(Address(i->first.first),rev); + } + } } } @@ -1037,8 +1048,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Mutex::Lock _l(_db_m); _db.put("network",nwids,network); } - std::string pfx("network/"); pfx.append(nwids); pfx.append("/member/"); - _db.filter(pfx,120000,[this,&now,&nwid](const std::string &n,const json &obj) { + + // Send an update to all members of the network + _db.filter((std::string("network/") + nwids + "/member/"),120000,[this,&now,&nwid](const std::string &n,const json &obj) { _pushMemberUpdate(now,nwid,obj); return true; // do not delete }); diff --git a/node/Membership.hpp b/node/Membership.hpp index a7794328..97510b57 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -191,7 +191,7 @@ public: { if (nconf.isPublic()) return true; - if ((_comRevocationThreshold)&&(_com.timestamp().first <= _comRevocationThreshold)) + if (_com.timestamp().first <= _comRevocationThreshold) return false; return nconf.com.agreesWith(_com); } diff --git a/node/Network.cpp b/node/Network.cpp index 9223987c..dd812cab 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1422,8 +1422,8 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c outp.append((uint16_t)0); // no capabilities outp.append((uint16_t)0); // no tags outp.append((uint16_t)1); // one revocation! - outp.append((uint16_t)0); // no certificates of ownership rev.serialize(outp); + outp.append((uint16_t)0); // no certificates of ownership RR->sw->send(outp,true); } } diff --git a/node/NetworkController.hpp b/node/NetworkController.hpp index fc5db4af..0634f435 100644 --- a/node/NetworkController.hpp +++ b/node/NetworkController.hpp @@ -24,11 +24,12 @@ #include "Constants.hpp" #include "Dictionary.hpp" #include "NetworkConfig.hpp" +#include "Revocation.hpp" +#include "Address.hpp" namespace ZeroTier { class Identity; -class Address; struct InetAddress; /** @@ -62,6 +63,14 @@ public: */ virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) = 0; + /** + * Send revocation to a node + * + * @param destination Destination node address + * @param rev Revocation to send + */ + virtual void ncSendRevocation(const Address &destination,const Revocation &rev) = 0; + /** * Send a network configuration request error * diff --git a/node/Node.cpp b/node/Node.cpp index a75a56b4..1125ca7a 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -774,6 +774,24 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de } } +void Node::ncSendRevocation(const Address &destination,const Revocation &rev) +{ + if (destination == RR->identity.address()) { + SharedPtr n(network(rev.networkId())); + if (!n) return; + n->addCredential(RR->identity.address(),rev); + } else { + Packet outp(destination,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + outp.append((uint8_t)0x00); + outp.append((uint16_t)0); + outp.append((uint16_t)0); + outp.append((uint16_t)1); + rev.serialize(outp); + outp.append((uint16_t)0); + RR->sw->send(outp,true); + } +} + void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) { if (destination == RR->identity.address()) { diff --git a/node/Node.hpp b/node/Node.hpp index ab201f06..21eac617 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -271,6 +271,7 @@ public: } virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig); + virtual void ncSendRevocation(const Address &destination,const Revocation &rev); virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode); private: diff --git a/node/Packet.hpp b/node/Packet.hpp index 87863b19..fb332b7d 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -731,8 +731,7 @@ public: /** * Network credentials push: - * <[...] serialized certificate of membership> - * [<[...] additional certificates of membership>] + * [<[...] one or more certificates of membership>] * <[1] 0x00, null byte marking end of COM array> * <[2] 16-bit number of capabilities> * <[...] one or more serialized Capability> diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 3903f440..1697b52f 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -89,8 +89,8 @@ public: { if (signer.hasPrivate()) { Buffer tmp; - this->serialize(tmp,true); _signedBy = signer.address(); + this->serialize(tmp,true); _signature = signer.sign(tmp.data(),tmp.size()); return true; } -- cgit v1.2.3 From 7ea7e1898a88ee60a346582fd6185e4f727759bf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Mar 2017 16:21:49 -0800 Subject: Fix a bug that could cause duplicate IP address assignment on networks if many new members join the controller at once. --- controller/EmbeddedNetworkController.cpp | 3 +++ controller/EmbeddedNetworkController.hpp | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 2f6142a9..b731db83 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -664,6 +664,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( // Member is being de-authorized, so spray Revocation objects to all online members if (!newAuth) { + _clearNetworkMemberInfoCache(nwid); Revocation rev(_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); Mutex::Lock _l(_lastRequestTime_m); @@ -1662,6 +1663,7 @@ void EmbeddedNetworkController::_request( if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) nc.staticIps[nc.staticIpCount++] = ip6; haveManagedIpv6AutoAssignment = true; + _clearNetworkMemberInfoCache(nwid); // clear cache to prevent IP assignment duplication on many rapid assigns break; } } @@ -1717,6 +1719,7 @@ void EmbeddedNetworkController::_request( v4ip->sin_addr.s_addr = Utils::hton(ip); } haveManagedIpv4AutoAssignment = true; + _clearNetworkMemberInfoCache(nwid); // clear cache to prevent IP assignment duplication on many rapid assigns break; } } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 7163c004..bca0956e 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -121,7 +121,6 @@ private: Mutex _threads_m; // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places - // This does lock _networkMemberCache_m struct _NetworkMemberInfo { _NetworkMemberInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} @@ -136,6 +135,11 @@ private: std::map _nmiCache; Mutex _nmiCache_m; void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); + inline void _clearNetworkMemberInfoCache(const uint64_t nwid) + { + Mutex::Lock _l(_nmiCache_m); + _nmiCache.erase(nwid); + } void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); -- cgit v1.2.3 From 37629aaf87c300ee5f9a56ecbbcdc1c4bb7db97f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Mar 2017 12:22:06 -0700 Subject: Use cache on requests to decrease DB load. --- controller/EmbeddedNetworkController.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b731db83..6fc5ba20 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -61,6 +61,9 @@ using json = nlohmann::json; // Nodes are considered active if they've queried in less than this long #define ZT_NETCONF_NODE_ACTIVE_THRESHOLD (ZT_NETWORK_AUTOCONF_DELAY * 2) +// Timeout for disk read cache (ms) +#define ZT_NETCONF_DB_CACHE_TTL 5000 + namespace ZeroTier { static json _renderRule(ZT_VirtualNetworkRule &rule) @@ -1244,8 +1247,8 @@ void EmbeddedNetworkController::_request( json member; { Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,0); - member = _db.get("network",nwids,"member",identity.address().toString(),0); + network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); + member = _db.get("network",nwids,"member",identity.address().toString(),ZT_NETCONF_DB_CACHE_TTL); } if (!network.size()) { -- cgit v1.2.3 From 8f592ff6e8d806537fe94bc2609eee483ee8e62d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Mar 2017 13:58:29 -0700 Subject: Controller performance tweaks. --- controller/EmbeddedNetworkController.cpp | 20 ++++++++++---------- controller/EmbeddedNetworkController.hpp | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 6fc5ba20..d2f40b1c 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -53,7 +53,7 @@ using json = nlohmann::json; #define ZT_NETCONF_CONTROLLER_API_VERSION 3 // Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 24 +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 2 // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 @@ -62,7 +62,7 @@ using json = nlohmann::json; #define ZT_NETCONF_NODE_ACTIVE_THRESHOLD (ZT_NETWORK_AUTOCONF_DELAY * 2) // Timeout for disk read cache (ms) -#define ZT_NETCONF_DB_CACHE_TTL 5000 +#define ZT_NETCONF_DB_CACHE_TTL 60000 namespace ZeroTier { @@ -503,7 +503,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( json network; { Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,0); + network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); } if (!network.size()) return 404; @@ -518,7 +518,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( json member; { Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString(),0); + member = _db.get("network",nwids,"member",Address(address).toString(),ZT_NETCONF_DB_CACHE_TTL); } if (!member.size()) return 404; @@ -534,7 +534,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( responseBody = "{"; std::string pfx(std::string("network/") + nwids + "member/"); - _db.filter(pfx,120000,[&responseBody](const std::string &n,const json &member) { + _db.filter(pfx,ZT_NETCONF_DB_CACHE_TTL,[&responseBody](const std::string &n,const json &member) { if (member.size() > 0) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(OSUtils::jsonString(member["id"],"")); @@ -642,7 +642,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json member; { Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString(),0); + member = _db.get("network",nwids,"member",Address(address).toString(),ZT_NETCONF_DB_CACHE_TTL); } json origMember(member); // for detecting changes _initMember(member); @@ -825,7 +825,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (_db.get("network",nwids,120000).size() <= 0) { + if (_db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL).size() <= 0) { nwid = tryNwid; break; } @@ -834,7 +834,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( return 503; } - network = _db.get("network",nwids,0); + network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); } json origNetwork(network); // for detecting changes _initNetwork(network); @@ -1096,7 +1096,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( json network; { Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,0); + network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); } if (!network.size()) return 404; @@ -1107,7 +1107,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( Mutex::Lock _l(_db_m); - json member = _db.get("network",nwids,"member",Address(address).toString(),0); + json member = _db.get("network",nwids,"member",Address(address).toString(),ZT_NETCONF_DB_CACHE_TTL); _db.erase("network",nwids,"member",Address(address).toString()); if (!member.size()) diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index bca0956e..a7277ace 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -44,7 +44,7 @@ #include "JSONDB.hpp" // Number of background threads to start -- not actually started until needed -#define ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT 2 +#define ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT 4 // TTL for circuit tests #define ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION 120000 -- cgit v1.2.3 From 4f3f471b4c7726b9ac66304788613c3c12e09d12 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 18:19:51 -0700 Subject: GitHub issue #460 --- controller/EmbeddedNetworkController.cpp | 7 +++--- controller/JSONDB.cpp | 22 ++++++------------- controller/JSONDB.hpp | 6 +++--- osdep/OSUtils.cpp | 37 -------------------------------- osdep/OSUtils.hpp | 8 ------- 5 files changed, 13 insertions(+), 67 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d2f40b1c..8fbc1cf8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -533,11 +533,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( Mutex::Lock _l(_db_m); responseBody = "{"; - std::string pfx(std::string("network/") + nwids + "member/"); - _db.filter(pfx,ZT_NETCONF_DB_CACHE_TTL,[&responseBody](const std::string &n,const json &member) { - if (member.size() > 0) { + _db.filter((std::string("network/") + nwids + "/member/"),ZT_NETCONF_DB_CACHE_TTL,[&responseBody](const std::string &n,const json &member) { + if ((member.is_object())&&(member.size() > 0)) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(OSUtils::jsonString(member["id"],"")); + responseBody.append(OSUtils::jsonString(member["id"],"0")); responseBody.append("\":"); responseBody.append(OSUtils::jsonString(member["revision"],"0")); } diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 1277aabb..298ac721 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -126,22 +126,14 @@ void JSONDB::erase(const std::string &n) _db.erase(n); } -void JSONDB::_reload(const std::string &p) +void JSONDB::_reload(const std::string &p,const std::string &b) { - std::map l(OSUtils::listDirectoryFull(p.c_str())); - for(std::map::iterator li(l.begin());li!=l.end();++li) { - if (li->second == 'f') { - // assume p starts with _basePath, which it always does -- will throw otherwise - std::string n(p.substr(_basePath.length())); - while ((n.length() > 0)&&(n[0] == ZT_PATH_SEPARATOR)) n = n.substr(1); - if (ZT_PATH_SEPARATOR != '/') std::replace(n.begin(),n.end(),ZT_PATH_SEPARATOR,'/'); - if ((n.length() > 0)&&(n[n.length() - 1] != '/')) n.push_back('/'); - n.append(li->first); - if ((n.length() > 5)&&(n.substr(n.length() - 5) == ".json")) { - this->get(n.substr(0,n.length() - 5),0); // causes load and cache or update - } - } else if (li->second == 'd') { - this->_reload(p + ZT_PATH_SEPARATOR + li->first); + std::vector dl(OSUtils::listDirectory(p.c_str())); + for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { + if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { + this->get(b + di->substr(0,di->length() - 5),0); + } else { + this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); } } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 5b7c5e50..d2549173 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -45,13 +45,13 @@ public: JSONDB(const std::string &basePath) : _basePath(basePath) { - _reload(_basePath); + _reload(_basePath,std::string()); } inline void reload() { _db.clear(); - _reload(_basePath); + _reload(_basePath,std::string()); } bool writeRaw(const std::string &n,const std::string &obj); @@ -95,7 +95,7 @@ public: inline bool operator!=(const JSONDB &db) const { return (!(*this == db)); } private: - void _reload(const std::string &p); + void _reload(const std::string &p,const std::string &b); bool _isValidObjectName(const std::string &n); std::string _genPath(const std::string &n,bool create); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 6b95258c..aac6bdd8 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -108,43 +108,6 @@ std::vector OSUtils::listDirectory(const char *path) return r; } -std::map OSUtils::listDirectoryFull(const char *path) -{ - std::map r; - -#ifdef __WINDOWS__ - HANDLE hFind; - WIN32_FIND_DATAA ffd; - if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { - do { - if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))) { - r[ffd.cFileName] = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) ? 'd' : 'f'; - } - } while (FindNextFileA(hFind,&ffd)); - FindClose(hFind); - } -#else - struct dirent de; - struct dirent *dptr; - DIR *d = opendir(path); - if (!d) - return r; - dptr = (struct dirent *)0; - for(;;) { - if (readdir_r(d,&de,&dptr)) - break; - if (dptr) { - if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))) { - r[dptr->d_name] = (dptr->d_type == DT_DIR) ? 'd' : 'f'; - } - } else break; - } - closedir(d); -#endif - - return r; -} - long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) { long cleaned = 0; diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index adf1488e..2e007ef2 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -111,14 +111,6 @@ public: */ static std::vector listDirectory(const char *path); - /** - * List all contents in a directory - * - * @param path Path to list - * @return Names of things and types, currently just 'f' and 'd' - */ - static std::map listDirectoryFull(const char *path); - /** * Clean a directory of files whose last modified time is older than this * -- cgit v1.2.3 From c62141fd9870eabf6f987da8e2a82ff5a999ddcf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 21 Mar 2017 06:15:49 -0700 Subject: Make controller do a simple write-through cache without revalidating. Means you must restart if files change on disk, but will decrease I/O considerably. --- controller/EmbeddedNetworkController.cpp | 31 +++++++--------- controller/EmbeddedNetworkController.hpp | 14 ++----- controller/JSONDB.cpp | 63 +++++++++----------------------- controller/JSONDB.hpp | 17 ++++----- 4 files changed, 41 insertions(+), 84 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8fbc1cf8..0e704288 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -61,9 +61,6 @@ using json = nlohmann::json; // Nodes are considered active if they've queried in less than this long #define ZT_NETCONF_NODE_ACTIVE_THRESHOLD (ZT_NETWORK_AUTOCONF_DELAY * 2) -// Timeout for disk read cache (ms) -#define ZT_NETCONF_DB_CACHE_TTL 60000 - namespace ZeroTier { static json _renderRule(ZT_VirtualNetworkRule &rule) @@ -503,7 +500,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( json network; { Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); + network = _db.get("network",nwids); } if (!network.size()) return 404; @@ -518,7 +515,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( json member; { Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString(),ZT_NETCONF_DB_CACHE_TTL); + member = _db.get("network",nwids,"member",Address(address).toString()); } if (!member.size()) return 404; @@ -533,7 +530,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( Mutex::Lock _l(_db_m); responseBody = "{"; - _db.filter((std::string("network/") + nwids + "/member/"),ZT_NETCONF_DB_CACHE_TTL,[&responseBody](const std::string &n,const json &member) { + _db.filter((std::string("network/") + nwids + "/member/"),[&responseBody](const std::string &n,const json &member) { if ((member.is_object())&&(member.size() > 0)) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(OSUtils::jsonString(member["id"],"0")); @@ -566,7 +563,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( std::set networkIds; { Mutex::Lock _l(_db_m); - _db.filter("network/",120000,[&networkIds](const std::string &n,const json &obj) { + _db.filter("network/",[&networkIds](const std::string &n,const json &obj) { if (n.length() == (16 + 8)) networkIds.insert(n.substr(8)); return true; // do not delete @@ -641,7 +638,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json member; { Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString(),ZT_NETCONF_DB_CACHE_TTL); + member = _db.get("network",nwids,"member",Address(address).toString()); } json origMember(member); // for detecting changes _initMember(member); @@ -824,7 +821,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (_db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL).size() <= 0) { + if (_db.get("network",nwids).size() <= 0) { nwid = tryNwid; break; } @@ -833,7 +830,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( return 503; } - network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); + network = _db.get("network",nwids); } json origNetwork(network); // for detecting changes _initNetwork(network); @@ -1053,7 +1050,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } // Send an update to all members of the network - _db.filter((std::string("network/") + nwids + "/member/"),120000,[this,&now,&nwid](const std::string &n,const json &obj) { + _db.filter((std::string("network/") + nwids + "/member/"),[this,&now,&nwid](const std::string &n,const json &obj) { _pushMemberUpdate(now,nwid,obj); return true; // do not delete }); @@ -1095,7 +1092,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( json network; { Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); + network = _db.get("network",nwids); } if (!network.size()) return 404; @@ -1106,7 +1103,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( Mutex::Lock _l(_db_m); - json member = _db.get("network",nwids,"member",Address(address).toString(),ZT_NETCONF_DB_CACHE_TTL); + json member = _db.get("network",nwids,"member",Address(address).toString()); _db.erase("network",nwids,"member",Address(address).toString()); if (!member.size()) @@ -1119,7 +1116,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( Mutex::Lock _l(_db_m); std::string pfx("network/"); pfx.append(nwids); - _db.filter(pfx,120000,[](const std::string &n,const json &obj) { + _db.filter(pfx,[](const std::string &n,const json &obj) { return false; // delete }); @@ -1246,8 +1243,8 @@ void EmbeddedNetworkController::_request( json member; { Mutex::Lock _l(_db_m); - network = _db.get("network",nwids,ZT_NETCONF_DB_CACHE_TTL); - member = _db.get("network",nwids,"member",identity.address().toString(),ZT_NETCONF_DB_CACHE_TTL); + network = _db.get("network",nwids); + member = _db.get("network",nwids,"member",identity.address().toString()); } if (!network.size()) { @@ -1772,7 +1769,7 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid { Mutex::Lock _l(_db_m); - _db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) { + _db.filter(pfx,[&nmi,&now](const std::string &n,const json &member) { try { if (OSUtils::jsonBool(member["authorized"],false)) { ++nmi.authorizedMemberCount; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index a7277ace..5d2454ed 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -99,12 +99,7 @@ public: private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); - void _request( - uint64_t nwid, - const InetAddress &fromAddr, - uint64_t requestPacketId, - const Identity &identity, - const Dictionary &metaData); + void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); struct _RQEntry { @@ -134,12 +129,9 @@ private: }; std::map _nmiCache; Mutex _nmiCache_m; + void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); - inline void _clearNetworkMemberInfoCache(const uint64_t nwid) - { - Mutex::Lock _l(_nmiCache_m); - _nmiCache.erase(nwid); - } + inline void _clearNetworkMemberInfoCache(const uint64_t nwid) { Mutex::Lock _l(_nmiCache_m); _nmiCache.erase(nwid); } void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 298ac721..9c28ddf1 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -53,8 +53,6 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) _E &e = _db[n]; e.obj = obj; - e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); - e.lastCheck = OSUtils::now(); return true; } @@ -64,53 +62,26 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe if (!_isValidObjectName(n)) return _EMPTY_JSON; - const uint64_t now = OSUtils::now(); - std::string buf; std::map::iterator e(_db.find(n)); - - if (e != _db.end()) { - if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck) - return e->second.obj; - - const std::string path(_genPath(n,false)); - if (!path.length()) // sanity check - return _EMPTY_JSON; - - // We are somewhat tolerant to momentary disk failures here. This may - // occur over e.g. EC2's elastic filesystem (NFS). - const uint64_t lm = OSUtils::getLastModified(path.c_str()); - if (e->second.lastModifiedOnDisk != lm) { - if (OSUtils::readFile(path.c_str(),buf)) { - try { - e->second.obj = OSUtils::jsonParse(buf); - e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP - e->second.lastCheck = now; - } catch ( ... ) {} // parse errors result in "holding pattern" behavior - } - } - + if (e != _db.end()) return e->second.obj; - } else { - const std::string path(_genPath(n,false)); - if (!path.length()) - return _EMPTY_JSON; - - if (!OSUtils::readFile(path.c_str(),buf)) - return _EMPTY_JSON; - - const uint64_t lm = OSUtils::getLastModified(path.c_str()); - _E &e2 = _db[n]; - try { - e2.obj = OSUtils::jsonParse(buf); - } catch ( ... ) { - e2.obj = _EMPTY_JSON; - buf = "{}"; - } - e2.lastModifiedOnDisk = lm; - e2.lastCheck = now; - return e2.obj; + const std::string path(_genPath(n,false)); + if (!path.length()) + return _EMPTY_JSON; + std::string buf; + if (!OSUtils::readFile(path.c_str(),buf)) + return _EMPTY_JSON; + + _E &e2 = _db[n]; + try { + e2.obj = OSUtils::jsonParse(buf); + } catch ( ... ) { + e2.obj = _EMPTY_JSON; + buf = "{}"; } + + return e2.obj; } void JSONDB::erase(const std::string &n) @@ -131,7 +102,7 @@ void JSONDB::_reload(const std::string &p,const std::string &b) std::vector dl(OSUtils::listDirectory(p.c_str())); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { - this->get(b + di->substr(0,di->length() - 5),0); + this->get(b + di->substr(0,di->length() - 5)); } else { this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index d2549173..00eeb8d8 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -63,12 +63,12 @@ public: inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); } inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); } - const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0); + const nlohmann::json &get(const std::string &n); - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2),maxSinceCheck); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3),maxSinceCheck); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); } + inline const nlohmann::json &get(const std::string &n1,const std::string &n2) { return this->get((n1 + "/" + n2)); } + inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3) { return this->get((n1 + "/" + n2 + "/" + n3)); } + inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4)); } + inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5)); } void erase(const std::string &n); @@ -78,11 +78,11 @@ public: inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5); } template - inline void filter(const std::string &prefix,unsigned long maxSinceCheck,F func) + inline void filter(const std::string &prefix,F func) { for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { - if (!func(i->first,get(i->first,maxSinceCheck))) { + if (!func(i->first,get(i->first))) { std::map::iterator i2(i); ++i2; this->erase(i->first); i = i2; @@ -102,9 +102,6 @@ private: struct _E { nlohmann::json obj; - uint64_t lastModifiedOnDisk; - uint64_t lastCheck; - inline bool operator==(const _E &e) const { return (obj == e.obj); } inline bool operator!=(const _E &e) const { return (obj != e.obj); } }; -- cgit v1.2.3 From 52689090755c8535b4764d8991e9d9d5b3264c5e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 21 Mar 2017 06:31:15 -0700 Subject: Add a facility for full flow-through uptime test of controller by Central. --- controller/EmbeddedNetworkController.cpp | 11 ++++++++++- controller/EmbeddedNetworkController.hpp | 23 ++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0e704288..51500ed7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -429,6 +429,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : + _startTime(OSUtils::now()), _threadsStarted(false), _db(dbPath), _node(node) @@ -1067,7 +1068,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } // else 404 - } // else 404 + } else if (path[0] == "dbtest") { + + json testRec; + const uint64_t now = OSUtils::now(); + testRec["clock"] = now; + testRec["uptime"] = (now - _startTime); + _db.put("dbtest",testRec); + + } return 404; } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 5d2454ed..0ae2f3b5 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -98,9 +98,6 @@ public: throw(); private: - static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); - void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); - struct _RQEntry { uint64_t nwid; @@ -109,11 +106,6 @@ private: Identity identity; Dictionary metaData; }; - BlockingQueue<_RQEntry *> _queue; - - Thread _threads[ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT]; - bool _threadsStarted; - Mutex _threads_m; // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places struct _NetworkMemberInfo @@ -127,12 +119,11 @@ private: uint64_t mostRecentDeauthTime; uint64_t nmiTimestamp; // time this NMI structure was computed }; - std::map _nmiCache; - Mutex _nmiCache_m; + static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); + void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); inline void _clearNetworkMemberInfoCache(const uint64_t nwid) { Mutex::Lock _l(_nmiCache_m); _nmiCache.erase(nwid); } - void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); // These init objects with default and static/informational fields @@ -188,6 +179,16 @@ private: member["clock"] = now; } + const uint64_t _startTime; + + BlockingQueue<_RQEntry *> _queue; + Thread _threads[ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT]; + bool _threadsStarted; + Mutex _threads_m; + + std::map _nmiCache; + Mutex _nmiCache_m; + JSONDB _db; Mutex _db_m; -- cgit v1.2.3 From 6bb19e7947bfa955afb288126158fe0155ebd7f4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 21 Mar 2017 09:08:32 -0700 Subject: build fix --- controller/JSONDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 9c28ddf1..8b6de9b4 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -57,7 +57,7 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) return true; } -const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceCheck) +const nlohmann::json &JSONDB::get(const std::string &n) { if (!_isValidObjectName(n)) return _EMPTY_JSON; -- cgit v1.2.3 From e4896b257fde05a216500804d9bcef3b84b0980e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 27 Mar 2017 17:03:17 -0700 Subject: Add thread PTR that gets passed through the entire ZT core call stack and then passed to handler functions resulting from a call. --- controller/EmbeddedNetworkController.cpp | 2 +- include/ZeroTierOne.h | 51 ++++-- node/Capability.cpp | 6 +- node/Capability.hpp | 2 +- node/CertificateOfMembership.cpp | 6 +- node/CertificateOfMembership.hpp | 3 +- node/CertificateOfOwnership.cpp | 6 +- node/CertificateOfOwnership.hpp | 3 +- node/IncomingPacket.cpp | 294 +++++++++++++++---------------- node/IncomingPacket.hpp | 43 ++--- node/Membership.cpp | 24 +-- node/Membership.hpp | 13 +- node/Multicaster.cpp | 23 +-- node/Multicaster.hpp | 13 +- node/Network.cpp | 112 ++++++------ node/Network.hpp | 59 ++++--- node/Node.cpp | 136 +++++++------- node/Node.hpp | 42 +++-- node/OutboundMulticast.cpp | 6 +- node/OutboundMulticast.hpp | 13 +- node/Path.cpp | 4 +- node/Path.hpp | 3 +- node/Peer.cpp | 37 ++-- node/Peer.hpp | 22 ++- node/Revocation.cpp | 6 +- node/Revocation.hpp | 3 +- node/SelfAwareness.cpp | 10 +- node/SelfAwareness.hpp | 2 +- node/Switch.cpp | 97 +++++----- node/Switch.hpp | 22 ++- node/Tag.cpp | 6 +- node/Tag.hpp | 3 +- node/Topology.cpp | 54 +++--- node/Topology.hpp | 26 +-- osdep/BSDEthernetTap.cpp | 5 +- osdep/BSDEthernetTap.hpp | 4 +- osdep/LinuxEthernetTap.cpp | 4 +- osdep/LinuxEthernetTap.hpp | 4 +- osdep/OSXEthernetTap.cpp | 4 +- osdep/OSXEthernetTap.hpp | 4 +- osdep/WindowsEthernetTap.cpp | 5 +- osdep/WindowsEthernetTap.hpp | 4 +- service/OneService.cpp | 58 +++--- service/SoftwareUpdater.cpp | 12 +- 44 files changed, 673 insertions(+), 583 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 51500ed7..ce56e906 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -790,7 +790,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( test->timestamp = OSUtils::now(); if (_node) { - _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); + _node->circuitTestBegin((void *)0,test,&(EmbeddedNetworkController::_circuitTestCallback)); } else { _tests.pop_back(); return 500; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 98413a21..747e1855 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1408,6 +1408,7 @@ typedef void ZT_Node; typedef int (*ZT_VirtualNetworkConfigFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* Network ID */ void **, /* Modifiable network user PTR */ enum ZT_VirtualNetworkConfigOperation, /* Config operation */ @@ -1423,6 +1424,7 @@ typedef int (*ZT_VirtualNetworkConfigFunction)( typedef void (*ZT_VirtualNetworkFrameFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* Network ID */ void **, /* Modifiable network user PTR */ uint64_t, /* Source MAC */ @@ -1442,10 +1444,11 @@ typedef void (*ZT_VirtualNetworkFrameFunction)( * in the definition of ZT_Event. */ typedef void (*ZT_EventCallback)( - ZT_Node *, - void *, - enum ZT_Event, - const void *); + ZT_Node *, /* Node */ + void *, /* User ptr */ + void *, /* Thread ptr */ + enum ZT_Event, /* Event type */ + const void *); /* Event payload (if applicable) */ /** * Function to get an object from the data store @@ -1468,8 +1471,9 @@ typedef void (*ZT_EventCallback)( * object. */ typedef long (*ZT_DataStoreGetFunction)( - ZT_Node *, - void *, + ZT_Node *, /* Node */ + void *, /* User ptr */ + void *, /* Thread ptr */ const char *, void *, unsigned long, @@ -1495,6 +1499,7 @@ typedef long (*ZT_DataStoreGetFunction)( typedef int (*ZT_DataStorePutFunction)( ZT_Node *, void *, + void *, /* Thread ptr */ const char *, const void *, unsigned long, @@ -1529,6 +1534,7 @@ typedef int (*ZT_DataStorePutFunction)( typedef int (*ZT_WirePacketSendFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ const struct sockaddr_storage *, /* Local address */ const struct sockaddr_storage *, /* Remote address */ const void *, /* Packet data */ @@ -1562,6 +1568,7 @@ typedef int (*ZT_WirePacketSendFunction)( typedef int (*ZT_PathCheckFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* ZeroTier address */ const struct sockaddr_storage *, /* Local address */ const struct sockaddr_storage *); /* Remote address */ @@ -1584,6 +1591,7 @@ typedef int (*ZT_PathCheckFunction)( typedef int (*ZT_PathLookupFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* ZeroTier address (40 bits) */ int, /* Desired ss_family or -1 for any */ struct sockaddr_storage *); /* Result buffer */ @@ -1654,11 +1662,12 @@ struct ZT_Node_Callbacks * * @param node Result: pointer is set to new node instance on success * @param uptr User pointer to pass to functions/callbacks + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param callbacks Callback function configuration * @param now Current clock in milliseconds * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); /** * Delete a node and free all resources it consumes @@ -1674,6 +1683,7 @@ void ZT_Node_delete(ZT_Node *node); * Process a packet received from the physical wire * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param now Current clock in milliseconds * @param localAddress Local address, or point to ZT_SOCKADDR_NULL if unspecified * @param remoteAddress Origin of packet @@ -1684,6 +1694,7 @@ void ZT_Node_delete(ZT_Node *node); */ enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -1695,6 +1706,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( * Process a frame from a virtual network port (tap) * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param now Current clock in milliseconds * @param nwid ZeroTier 64-bit virtual network ID * @param sourceMac Source MAC address (least significant 48 bits) @@ -1708,6 +1720,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( */ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -1722,11 +1735,12 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( * Perform periodic background operations * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param now Current clock in milliseconds * @param nextBackgroundTaskDeadline Value/result: set to deadline for next call to processBackgroundTasks() * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); +enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); /** * Join a network @@ -1742,7 +1756,7 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,vol * @param uptr An arbitrary pointer to associate with this network (default: NULL) * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr); +enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr,void *tptr); /** * Leave a network @@ -1759,7 +1773,7 @@ enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr); * @param uptr Target pointer is set to uptr (if not NULL) * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr); +enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr,void *tptr); /** * Subscribe to an Ethernet multicast group @@ -1781,12 +1795,13 @@ enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr); * This does not generate an update call to networkConfigCallback(). * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param nwid 64-bit network ID * @param multicastGroup Ethernet multicast or broadcast MAC (least significant 48 bits) * @param multicastAdi Multicast ADI (least significant 32 bits only, use 0 if not needed) * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); +enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); /** * Unsubscribe from an Ethernet multicast group (or all groups) @@ -1811,21 +1826,24 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint * across invocations if the contents of moon.d are scanned and orbit is * called for each on startup. * + * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param moonWorldId Moon's world ID * @param moonSeed If non-zero, the ZeroTier address of any member of the moon to query for moon definition * @param len Length of moonWorld in bytes * @return Error if moon was invalid or failed to be added */ -enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed); +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,uint64_t moonSeed); /** * Remove a moon (does nothing if not present) * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param moonWorldId World ID of moon to remove * @return Error if anything bad happened */ -enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId); +enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId); /** * Get this node's 40-bit ZeroTier address @@ -1919,13 +1937,15 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node); * There is no delivery guarantee here. Failure can occur if the message is * too large or if dest is not a valid ZeroTier address. * + * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param dest Destination ZeroTier address * @param typeId VERB_USER_MESSAGE type ID * @param data Payload data to attach to user message * @param len Length of data in bytes * @return Boolean: non-zero on success, zero on failure */ -int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); +int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); /** * Set a network configuration master instance for this node @@ -1957,11 +1977,12 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance); * for results forever. * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param test Test configuration * @param reportCallback Function to call each time a report is received * @return OK or error if, for example, test is too big for a packet or support isn't compiled in */ -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *)); +enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *)); /** * Stop listening for results to a given circuit test diff --git a/node/Capability.cpp b/node/Capability.cpp index 0a736ca8..c178e566 100644 --- a/node/Capability.cpp +++ b/node/Capability.cpp @@ -25,7 +25,7 @@ namespace ZeroTier { -int Capability::verify(const RuntimeEnvironment *RR) const +int Capability::verify(const RuntimeEnvironment *RR,void *tPtr) const { try { // There must be at least one entry, and sanity check for bad chain max length @@ -46,12 +46,12 @@ int Capability::verify(const RuntimeEnvironment *RR) const return -1; // otherwise if we have another entry it must be from the previous holder in the chain } - const Identity id(RR->topology->getIdentity(_custody[c].from)); + const Identity id(RR->topology->getIdentity(tPtr,_custody[c].from)); if (id) { if (!id.verify(tmp.data(),tmp.size(),_custody[c].signature)) return -1; } else { - RR->sw->requestWhois(_custody[c].from); + RR->sw->requestWhois(tPtr,_custody[c].from); return 1; } } diff --git a/node/Capability.hpp b/node/Capability.hpp index d070f2ad..5ef6c994 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -161,7 +161,7 @@ public: * @param RR Runtime environment to provide for peer lookup, etc. * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template static inline void serializeRules(Buffer &b,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) diff --git a/node/CertificateOfMembership.cpp b/node/CertificateOfMembership.cpp index 43efcd20..9bf70216 100644 --- a/node/CertificateOfMembership.cpp +++ b/node/CertificateOfMembership.cpp @@ -207,14 +207,14 @@ bool CertificateOfMembership::sign(const Identity &with) } } -int CertificateOfMembership::verify(const RuntimeEnvironment *RR) const +int CertificateOfMembership::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(networkId()))||(_qualifierCount > ZT_NETWORK_COM_MAX_QUALIFIERS)) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 2d7c2cb3..ae976b50 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -250,9 +250,10 @@ public: * Verify this COM and its signature * * @param RR Runtime environment for looking up peers + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; /** * @return True if signed diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp index 6fc59ad1..2bd181e0 100644 --- a/node/CertificateOfOwnership.cpp +++ b/node/CertificateOfOwnership.cpp @@ -25,13 +25,13 @@ namespace ZeroTier { -int CertificateOfOwnership::verify(const RuntimeEnvironment *RR) const +int CertificateOfOwnership::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } try { diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 57fd8259..8c47582d 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -137,9 +137,10 @@ public: /** * @param RR Runtime environment to allow identity lookup for signedBy + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template inline void serialize(Buffer &b,const bool forSign = false) const diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e2275a04..52794fd7 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -44,7 +44,7 @@ namespace ZeroTier { -bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) +bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) { const Address sourceAddress(source()); @@ -65,10 +65,10 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { // Only HELLO is allowed in the clear, but will still have a MAC - return _doHELLO(RR,false); + return _doHELLO(RR,tPtr,false); } - const SharedPtr peer(RR->topology->getPeer(sourceAddress)); + const SharedPtr peer(RR->topology->getPeer(tPtr,sourceAddress)); if (peer) { if (!trusted) { if (!dearmor(peer->key())) { @@ -89,30 +89,30 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); return true; - case Packet::VERB_HELLO: return _doHELLO(RR,true); - case Packet::VERB_ERROR: return _doERROR(RR,peer); - case Packet::VERB_OK: return _doOK(RR,peer); - case Packet::VERB_WHOIS: return _doWHOIS(RR,peer); - case Packet::VERB_RENDEZVOUS: return _doRENDEZVOUS(RR,peer); - case Packet::VERB_FRAME: return _doFRAME(RR,peer); - case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,peer); - case Packet::VERB_ECHO: return _doECHO(RR,peer); - case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); - case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,peer); - case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); - case Packet::VERB_NETWORK_CONFIG: return _doNETWORK_CONFIG(RR,peer); - case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); - case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); - case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); - case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,peer); - case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,peer); - case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,peer); + case Packet::VERB_HELLO: return _doHELLO(RR,tPtr,true); + case Packet::VERB_ERROR: return _doERROR(RR,tPtr,peer); + case Packet::VERB_OK: return _doOK(RR,tPtr,peer); + case Packet::VERB_WHOIS: return _doWHOIS(RR,tPtr,peer); + case Packet::VERB_RENDEZVOUS: return _doRENDEZVOUS(RR,tPtr,peer); + case Packet::VERB_FRAME: return _doFRAME(RR,tPtr,peer); + case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,tPtr,peer); + case Packet::VERB_ECHO: return _doECHO(RR,tPtr,peer); + case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,tPtr,peer); + case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,tPtr,peer); + case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,tPtr,peer); + case Packet::VERB_NETWORK_CONFIG: return _doNETWORK_CONFIG(RR,tPtr,peer); + case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,tPtr,peer); + case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,tPtr,peer); + case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,tPtr,peer); + case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,tPtr,peer); + case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,tPtr,peer); + case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,tPtr,peer); } } else { - RR->sw->requestWhois(sourceAddress); + RR->sw->requestWhois(tPtr,sourceAddress); return false; } } catch ( ... ) { @@ -123,7 +123,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } } -bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB]; @@ -163,7 +163,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_IDENTITY_COLLISION: // FIXME: for federation this will need a payload with a signature or something. if (RR->topology->isUpstream(peer->identity())) - RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); + RR->node->postEvent(tPtr,ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { @@ -171,7 +171,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); const uint64_t now = RR->node->now(); if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) - network->pushCredentialsNow(peer->address(),now); + network->pushCredentialsNow(tPtr,peer->address(),now); } break; case Packet::ERROR_NETWORK_ACCESS_DENIED_: { @@ -185,7 +185,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr // Members of networks can use this error to indicate that they no longer // want to receive multicasts on a given channel. const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); - if ((network)&&(network->gate(peer))) { + if ((network)&&(network->gate(tPtr,peer))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str()); RR->mc->remove(network->id(),mg,peer->address()); @@ -195,14 +195,14 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated) +bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated) { try { const uint64_t now = RR->node->now(); @@ -226,7 +226,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut return true; } - SharedPtr peer(RR->topology->getPeer(id.address())); + SharedPtr peer(RR->topology->getPeer(tPtr,id.address())); if (peer) { // We already have an identity with this address -- check for collisions if (!alreadyAuthenticated) { @@ -246,7 +246,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut outp.append((uint64_t)pid); outp.append((uint8_t)Packet::ERROR_IDENTITY_COLLISION); outp.armor(key,true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } else { TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); } @@ -292,7 +292,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut return true; } - peer = RR->topology->addPeer(newPeer); + peer = RR->topology->addPeer(tPtr,newPeer); // Continue at // VALID } @@ -304,7 +304,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut if (ptr < size()) { ptr += externalSurfaceAddress.deserialize(*this,ptr); if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); + RR->sa->iam(tPtr,id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); } // Get primary planet world ID and world timestamp if present @@ -408,17 +408,17 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),now); + _path->send(RR,tPtr,outp.data(),outp.size(),now); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; @@ -463,7 +463,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p while (ptr < endOfWorlds) { World w; ptr += w.deserialize(*this,ptr); - RR->topology->addWorld(w,false); + RR->topology->addWorld(tPtr,w,false); } } else { ptr += worldsLen; @@ -490,20 +490,20 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); + RR->sa->iam(tPtr,peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); } break; case Packet::VERB_WHOIS: if (RR->topology->isUpstream(peer->identity())) { const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY); - RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr(new Peer(RR,RR->identity,id)))); + RR->sw->doAnythingWaitingForPeer(tPtr,RR->topology->addPeer(tPtr,SharedPtr(new Peer(RR,RR->identity,id)))); } break; case Packet::VERB_NETWORK_CONFIG_REQUEST: { const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_OK_IDX_PAYLOAD))); if (network) - network->handleConfigChunk(packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); + network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); } break; case Packet::VERB_MULTICAST_GATHER: { @@ -513,7 +513,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + RR->mc->addMultiple(tPtr,RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); } } break; @@ -532,7 +532,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); if (com) - network->addCredential(com); + network->addCredential(tPtr,com); } if ((flags & 0x02) != 0) { @@ -540,7 +540,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; unsigned int totalKnown = at(offset); offset += 4; unsigned int count = at(offset); offset += 2; - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); + RR->mc->addMultiple(tPtr,RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); } } } break; @@ -548,14 +548,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if ((!RR->topology->amRoot())&&(!peer->rateGateInboundWhoisRequest(RR->node->now()))) { @@ -573,13 +573,13 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr const Address addr(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const Identity id(RR->topology->getIdentity(addr)); + const Identity id(RR->topology->getIdentity(tPtr,addr)); if (id) { id.serialize(outp,false); ++count; } else { // Request unknown WHOIS from upstream from us (if we have one) - RR->sw->requestWhois(addr); + RR->sw->requestWhois(tPtr,addr); #ifdef ZT_ENABLE_CLUSTER // Distribute WHOIS queries across a cluster if we do not know the ID. // This may result in duplicate OKs to the querying peer, which is fine. @@ -591,32 +591,32 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr if (count > 0) { outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (!RR->topology->isUpstream(peer->identity())) { TRACE("RENDEZVOUS from %s ignored since source is not upstream",peer->address().toString().c_str()); } else { const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - const SharedPtr rendezvousWith(RR->topology->getPeer(with)); + const SharedPtr rendezvousWith(RR->topology->getPeer(tPtr,with)); if (rendezvousWith) { const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - if (RR->node->shouldUsePathForZeroTierTraffic(with,_path->localAddress(),atAddr)) { - RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now(),false,0); + if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localAddress(),atAddr)) { + RR->node->putPacket(tPtr,_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + rendezvousWith->attemptToContactAt(tPtr,_path->localAddress(),atAddr,RR->node->now(),false,0); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -628,46 +628,46 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); bool trustEstablished = false; if (network) { - if (network->gate(peer)) { + if (network->gate(tPtr,peer)) { trustEstablished = true; if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); const MAC sourceMac(peer->address(),nwid); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) - RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); + if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) + RR->node->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); } } else { TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } } else { TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } - peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID); @@ -680,13 +680,13 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

addCredential(com); + network->addCredential(tPtr,com); } - if (!network->gate(peer)) { + if (!network->gate(tPtr,peer)) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -699,36 +699,36 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC %s",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),from.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } - switch (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { + switch (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { case 1: if (from != MAC(peer->address(),nwid)) { if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (to.isMulticast()) { if (network->config().multicastLimit == 0) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: network %.16llx does not allow multicast",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (!network->config().permitsBridging(RR->identity.address())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } // fall through -- 2 means accept regardless of bridging checks or other restrictions case 2: - RR->node->putFrame(nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); + RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); break; } } @@ -739,14 +739,14 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -754,7 +754,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

&peer) +bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (!peer->rateGateEchoRequest(RR->node->now())) { @@ -769,16 +769,16 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr if (size() > ZT_PACKET_IDX_PAYLOAD) outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t now = RR->node->now(); @@ -802,9 +802,9 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared if (!auth) { if ((!network)||(network->id() != nwid)) network = RR->node->network(nwid); - const bool authOnNet = ((network)&&(network->gate(peer))); + const bool authOnNet = ((network)&&(network->gate(tPtr,peer))); if (!authOnNet) - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); trustEstablished |= authOnNet; if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { auth = true; @@ -815,18 +815,18 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared if (auth) { const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); - RR->mc->add(now,nwid,group,peer->address()); + RR->mc->add(tPtr,now,nwid,group,peer->address()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (!peer->rateGateCredentialsReceived(RR->node->now())) { @@ -847,7 +847,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S if (com) { const SharedPtr network(RR->node->network(com.networkId())); if (network) { - switch (network->addCredential(com)) { + switch (network->addCredential(tPtr,com)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -857,7 +857,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S case Membership::ADD_DEFERRED_FOR_WHOIS: return false; } - } else RR->mc->addCredential(com,false); + } else RR->mc->addCredential(tPtr,com,false); } } ++p; // skip trailing 0 after COMs if present @@ -868,7 +868,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += cap.deserialize(*this,p); const SharedPtr network(RR->node->network(cap.networkId())); if (network) { - switch (network->addCredential(cap)) { + switch (network->addCredential(tPtr,cap)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -888,7 +888,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += tag.deserialize(*this,p); const SharedPtr network(RR->node->network(tag.networkId())); if (network) { - switch (network->addCredential(tag)) { + switch (network->addCredential(tPtr,tag)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -908,7 +908,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += revocation.deserialize(*this,p); const SharedPtr network(RR->node->network(revocation.networkId())); if (network) { - switch(network->addCredential(peer->address(),revocation)) { + switch(network->addCredential(tPtr,peer->address(),revocation)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -928,7 +928,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += coo.deserialize(*this,p); const SharedPtr network(RR->node->network(coo.networkId())); if (network) { - switch(network->addCredential(coo)) { + switch(network->addCredential(tPtr,coo)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -942,7 +942,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); } catch (std::exception &exc) { //fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -953,7 +953,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S return true; } -bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); @@ -972,10 +972,10 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION); outp.append(nwid); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); } catch (std::exception &exc) { //fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -986,12 +986,12 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons return true; } -bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const SharedPtr network(RR->node->network(at(ZT_PACKET_IDX_PAYLOAD))); if (network) { - const uint64_t configUpdateId = network->handleConfigChunk(packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD); + const uint64_t configUpdateId = network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD); if (configUpdateId) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((uint8_t)Packet::VERB_ECHO); @@ -999,17 +999,17 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared outp.append((uint64_t)network->id()); outp.append((uint64_t)configUpdateId); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); @@ -1027,17 +1027,17 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM); if (com) { if (network) - network->addCredential(com); - else RR->mc->addCredential(com,false); + network->addCredential(tPtr,com); + else RR->mc->addCredential(tPtr,com,false); } } catch ( ... ) { TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str()); } } - const bool trustEstablished = ((network)&&(network->gate(peer))); + const bool trustEstablished = ((network)&&(network->gate(tPtr,peer))); if (!trustEstablished) - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); @@ -1048,7 +1048,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit); if (gatheredLocally > 0) { outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } // If we are a member of a cluster, distribute this GATHER across it @@ -1058,14 +1058,14 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar #endif } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID); @@ -1081,19 +1081,19 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); if (com) - network->addCredential(com); + network->addCredential(tPtr,com); } - if (!network->gate(peer)) { + if (!network->gate(tPtr,peer)) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } if (network->config().multicastLimit == 0) { TRACE("dropped MULTICAST_FRAME from %s(%s): network %.16llx does not allow multicast",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -1120,12 +1120,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { if (!to.mac().isMulticast()) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -1134,14 +1134,14 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); - if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { - RR->node->putFrame(nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); + if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { + RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); } } @@ -1155,14 +1155,14 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share outp.append((unsigned char)0x02); // flag 0x02 = contains gather results if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) { outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); } else { - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -1170,7 +1170,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share return true; } -bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t now = RR->node->now(); @@ -1178,7 +1178,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; } @@ -1209,10 +1209,10 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now,false,0); + peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1228,10 +1228,10 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now,false,0); + peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1241,20 +1241,20 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha ptr += addrLen; } - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const Address originatorAddress(field(ZT_PACKET_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - SharedPtr originator(RR->topology->getPeer(originatorAddress)); + SharedPtr originator(RR->topology->getPeer(tPtr,originatorAddress)); if (!originator) { - RR->sw->requestWhois(originatorAddress); + RR->sw->requestWhois(tPtr,originatorAddress); return false; } @@ -1285,7 +1285,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } vlf += signatureLength; @@ -1304,14 +1304,14 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt SharedPtr network(RR->node->network(originatorCredentialNetworkId)); if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } - if (network->gate(peer)) + if (network->gate(tPtr,peer)) reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH; } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } @@ -1327,7 +1327,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt for(unsigned int h=0;h nhp(RR->topology->getPeer(nextHop[h])); + SharedPtr nhp(RR->topology->getPeer(tPtr,nextHop[h])); if (nhp) { SharedPtr nhbp(nhp->getBestPath(now,false)); if ((nhbp)&&(nhbp->alive(now))) @@ -1362,7 +1362,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt nextHop[h].appendTo(outp); nextHopBestPathAddress[h].serialize(outp); // appends 0 if null InetAddress } - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } // If there are next hops, forward the test along through the graph @@ -1377,19 +1377,19 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt if (RR->identity.address() != nextHop[h]) { // next hops that loop back to the current hop are not valid outp.newInitializationVector(); outp.setDestination(nextHop[h]); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } } - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { ZT_CircuitTestReport report; @@ -1431,14 +1431,14 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S RR->node->postCircuitTestReport(&report); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (size() >= (ZT_PACKET_IDX_PAYLOAD + 8)) { @@ -1447,16 +1447,16 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPt um.typeId = at(ZT_PACKET_IDX_PAYLOAD); um.data = reinterpret_cast(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD + 8); um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); - RR->node->postEvent(ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); + RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); } - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid) +void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid) { const uint64_t now = RR->node->now(); if (peer->rateGateOutgoingComRequest(now)) { @@ -1466,7 +1466,7 @@ void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,cons outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); outp.append(nwid); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),now); + _path->send(RR,tPtr,outp.data(),outp.size(),now); } } diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index febff28a..3d4a2e05 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -102,9 +102,10 @@ public: * may no longer be valid. * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return True if decoding and processing is complete, false if caller should try again */ - bool tryDecode(const RuntimeEnvironment *RR); + bool tryDecode(const RuntimeEnvironment *RR,void *tPtr); /** * @return Time of packet receipt / start of decode @@ -114,26 +115,26 @@ public: private: // These are called internally to handle packet contents once it has // been authenticated, decrypted, decompressed, and classified. - bool _doERROR(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated); - bool _doOK(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doFRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doECHO(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CONFIG(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPtr &peer); - - void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid); + bool _doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated); + bool _doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doFRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + + void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid); uint64_t _receiveTime; SharedPtr _path; diff --git a/node/Membership.cpp b/node/Membership.cpp index 3b2e3b1c..22c13c88 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -40,7 +40,7 @@ Membership::Membership() : for(unsigned int i=0;i= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); @@ -113,7 +113,7 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now outp.setAt(cooCountAt,(uint16_t)thisPacketCooCount); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } @@ -123,7 +123,7 @@ const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) cons return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com) { const uint64_t newts = com.timestamp().first; if (newts <= _comRevocationThreshold) { @@ -141,7 +141,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme return ADD_ACCEPTED_REDUNDANT; } - switch(com.verify(RR)) { + switch(com.verify(RR,tPtr)) { default: TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (invalid signature or object)",com.issuedTo().toString().c_str(),com.networkId()); return ADD_REJECTED; @@ -154,7 +154,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) { _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteCredential *)0; @@ -169,7 +169,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } - switch(tag.verify(RR)) { + switch(tag.verify(RR,tPtr)) { default: TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); return ADD_REJECTED; @@ -184,7 +184,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) { _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCredential *)0; @@ -199,7 +199,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } - switch(cap.verify(RR)) { + switch(cap.verify(RR,tPtr)) { default: TRACE("addCredential(Capability) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; @@ -214,9 +214,9 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev) { - switch(rev.verify(RR)) { + switch(rev.verify(RR,tPtr)) { default: return ADD_REJECTED; case 0: { @@ -239,7 +239,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) { _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; @@ -254,7 +254,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } - switch(coo.verify(RR)) { + switch(coo.verify(RR,tPtr)) { default: TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",coo.issuedTo().toString().c_str(),coo.networkId()); return ADD_REJECTED; diff --git a/node/Membership.hpp b/node/Membership.hpp index 97510b57..c28d598c 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -158,13 +158,14 @@ public: * sends VERB_NETWORK_CREDENTIALS if the recipient might need them. * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @param peerAddress Address of member peer (the one that this Membership describes) * @param nconf My network config * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none * @param force If true, send objects regardless of last push time */ - void pushCredentials(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); + void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** * Check whether we should push MULTICAST_LIKEs to this peer @@ -226,27 +227,27 @@ public: /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo); private: _RemoteCredential *_newTag(const uint64_t id); diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index f8d58501..8e534b5e 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -43,14 +43,14 @@ Multicaster::~Multicaster() { } -void Multicaster::addMultiple(uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) +void Multicaster::addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) { const unsigned char *p = (const unsigned char *)addresses; const unsigned char *e = p + (5 * count); Mutex::Lock _l(_groups_m); MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)]; while (p != e) { - _add(now,nwid,mg,gs,Address(p,5)); + _add(tPtr,now,nwid,mg,gs,Address(p,5)); p += 5; } } @@ -152,6 +152,7 @@ std::vector

Multicaster::getMembers(uint64_t nwid,const MulticastGroup } void Multicaster::send( + void *tPtr, unsigned int limit, uint64_t now, uint64_t nwid, @@ -207,7 +208,7 @@ void Multicaster::send( for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { if (*ast != RR->identity.address()) { - out.sendOnly(RR,*ast); // optimization: don't use dedup log if it's a one-pass send + out.sendOnly(RR,tPtr,*ast); // optimization: don't use dedup log if it's a one-pass send if (++count >= limit) break; } @@ -217,7 +218,7 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { Address ma(gs.members[indexes[idx++]].address); if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { - out.sendOnly(RR,ma); // optimization: don't use dedup log if it's a one-pass send + out.sendOnly(RR,tPtr,ma); // optimization: don't use dedup log if it's a one-pass send ++count; } } @@ -256,7 +257,7 @@ void Multicaster::send( if (com) com->serialize(outp); RR->node->expectReplyTo(outp.packetId()); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } @@ -280,7 +281,7 @@ void Multicaster::send( for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { if (*ast != RR->identity.address()) { - out.sendAndLog(RR,*ast); + out.sendAndLog(RR,tPtr,*ast); if (++count >= limit) break; } @@ -290,7 +291,7 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { Address ma(gs.members[indexes[idx++]].address); if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { - out.sendAndLog(RR,ma); + out.sendAndLog(RR,tPtr,ma); ++count; } } @@ -352,15 +353,15 @@ void Multicaster::clean(uint64_t now) } } -void Multicaster::addCredential(const CertificateOfMembership &com,bool alreadyValidated) +void Multicaster::addCredential(void *tPtr,const CertificateOfMembership &com,bool alreadyValidated) { - if ((alreadyValidated)||(com.verify(RR) == 0)) { + if ((alreadyValidated)||(com.verify(RR,tPtr) == 0)) { Mutex::Lock _l(_gatherAuth_m); _gatherAuth[_GatherAuthKey(com.networkId(),com.issuedTo())] = RR->node->now(); } } -void Multicaster::_add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) +void Multicaster::_add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) { // assumes _groups_m is locked @@ -383,7 +384,7 @@ void Multicaster::_add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,Multi if (tx->atLimit()) gs.txQueue.erase(tx++); else { - tx->sendIfNew(RR,member); + tx->sendIfNew(RR,tPtr,member); if (tx->atLimit()) gs.txQueue.erase(tx++); else ++tx; diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 32dec9cf..f646a5be 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -90,10 +90,10 @@ public: * @param mg Multicast group * @param member New member address */ - inline void add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) + inline void add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) { Mutex::Lock _l(_groups_m); - _add(now,nwid,mg,_groups[Multicaster::Key(nwid,mg)],member); + _add(tPtr,now,nwid,mg,_groups[Multicaster::Key(nwid,mg)],member); } /** @@ -101,6 +101,7 @@ public: * * It's up to the caller to check bounds on the array before calling this. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @param nwid Network ID * @param mg Multicast group @@ -108,7 +109,7 @@ public: * @param count Number of addresses * @param totalKnown Total number of known addresses as reported by peer */ - void addMultiple(uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); + void addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); /** * Remove a multicast group member (if present) @@ -150,6 +151,7 @@ public: /** * Send a multicast * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param limit Multicast limit * @param now Current time * @param nwid Network ID @@ -162,6 +164,7 @@ public: * @param len Length of packet data */ void send( + void *tPtr, unsigned int limit, uint64_t now, uint64_t nwid, @@ -191,7 +194,7 @@ public: * @param com Certificate of membership * @param alreadyValidated If true, COM has already been checked and found to be valid and signed */ - void addCredential(const CertificateOfMembership &com,bool alreadyValidated); + void addCredential(void *tPtr,const CertificateOfMembership &com,bool alreadyValidated); /** * Check authorization for GATHER and LIKE for non-network-members @@ -209,7 +212,7 @@ public: } private: - void _add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); + void _add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); const RuntimeEnvironment *RR; diff --git a/node/Network.cpp b/node/Network.cpp index 38c1b0d9..0abfdf86 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -674,7 +674,7 @@ static _doZtFilterResult _doZtFilter( const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); -Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : +Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr) : RR(renv), _uPtr(uptr), _id(nwid), @@ -696,11 +696,11 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : Dictionary *dconf = new Dictionary(); NetworkConfig *nconf = new NetworkConfig(); try { - std::string conf(RR->node->dataStoreGet(confn)); + std::string conf(RR->node->dataStoreGet(tPtr,confn)); if (conf.length()) { dconf->load(conf.c_str()); if (nconf->fromDictionary(*dconf)) { - this->setConfiguration(*nconf,false); + this->setConfiguration(tPtr,*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; } @@ -711,13 +711,13 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : if (!gotConf) { // Save a one-byte CR to persist membership while we request a real netconf - RR->node->dataStorePut(confn,"\n",1,false); + RR->node->dataStorePut(tPtr,confn,"\n",1,false); } if (!_portInitialized) { ZT_VirtualNetworkConfig ctmp; _externalConfig(&ctmp); - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); _portInitialized = true; } } @@ -729,15 +729,16 @@ Network::~Network() char n[128]; if (_destroyed) { - RR->node->configureVirtualNetworkPort(_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); + RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - RR->node->dataStoreDelete(n); + RR->node->dataStoreDelete((void *)0,n); } else { - RR->node->configureVirtualNetworkPort(_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp); + RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp); } } bool Network::filterOutgoingPacket( + void *tPtr, const bool noTee, const Address &ztSource, const Address &ztDest, @@ -781,7 +782,7 @@ bool Network::filterOutgoingPacket( if ((!noTee)&&(cc2)) { Membership &m2 = _membership(cc2); - m2.pushCredentials(RR,now,cc2,_config,localCapabilityIndex,false); + m2.pushCredentials(RR,tPtr,now,cc2,_config,localCapabilityIndex,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -791,7 +792,7 @@ bool Network::filterOutgoingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength2); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } break; @@ -813,11 +814,11 @@ bool Network::filterOutgoingPacket( if (accept) { if (membership) - membership->pushCredentials(RR,now,ztDest,_config,localCapabilityIndex,false); + membership->pushCredentials(RR,tPtr,now,ztDest,_config,localCapabilityIndex,false); if ((!noTee)&&(cc)) { Membership &m2 = _membership(cc); - m2.pushCredentials(RR,now,cc,_config,localCapabilityIndex,false); + m2.pushCredentials(RR,tPtr,now,cc,_config,localCapabilityIndex,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -827,12 +828,12 @@ bool Network::filterOutgoingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { Membership &m2 = _membership(ztFinalDest); - m2.pushCredentials(RR,now,ztFinalDest,_config,localCapabilityIndex,false); + m2.pushCredentials(RR,tPtr,now,ztFinalDest,_config,localCapabilityIndex,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -842,7 +843,7 @@ bool Network::filterOutgoingPacket( outp.append((uint16_t)etherType); outp.append(frameData,frameLen); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); return false; // DROP locally, since we redirected } else { @@ -854,6 +855,7 @@ bool Network::filterOutgoingPacket( } int Network::filterIncomingPacket( + void *tPtr, const SharedPtr &sourcePeer, const Address &ztDest, const MAC &macSource, @@ -898,7 +900,7 @@ int Network::filterIncomingPacket( if (accept) { if (cc2) { - _membership(cc2).pushCredentials(RR,RR->node->now(),cc2,_config,-1,false); + _membership(cc2).pushCredentials(RR,tPtr,RR->node->now(),cc2,_config,-1,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -908,7 +910,7 @@ int Network::filterIncomingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength2); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } break; } @@ -929,7 +931,7 @@ int Network::filterIncomingPacket( if (accept) { if (cc) { - _membership(cc).pushCredentials(RR,RR->node->now(),cc,_config,-1,false); + _membership(cc).pushCredentials(RR,tPtr,RR->node->now(),cc,_config,-1,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -939,11 +941,11 @@ int Network::filterIncomingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { - _membership(ztFinalDest).pushCredentials(RR,RR->node->now(),ztFinalDest,_config,-1,false); + _membership(ztFinalDest).pushCredentials(RR,tPtr,RR->node->now(),ztFinalDest,_config,-1,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -953,7 +955,7 @@ int Network::filterIncomingPacket( outp.append((uint16_t)etherType); outp.append(frameData,frameLen); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); return 0; // DROP locally, since we redirected } @@ -972,12 +974,12 @@ bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBr return false; } -void Network::multicastSubscribe(const MulticastGroup &mg) +void Network::multicastSubscribe(void *tPtr,const MulticastGroup &mg) { Mutex::Lock _l(_lock); if (!std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) { _myMulticastGroups.insert(std::upper_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg),mg); - _sendUpdatesToMembers(&mg); + _sendUpdatesToMembers(tPtr,&mg); } } @@ -989,7 +991,7 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) _myMulticastGroups.erase(i); } -uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr) +uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr) { const unsigned int start = ptr; @@ -1043,7 +1045,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc } // If it's not a duplicate, check chunk signature - const Identity controllerId(RR->topology->getIdentity(controller())); + const Identity controllerId(RR->topology->getIdentity(tPtr,controller())); if (!controllerId) { // we should always have the controller identity by now, otherwise how would we have queried it the first time? TRACE("unable to verify chunk from %s: don't have controller identity",source.toString().c_str()); return 0; @@ -1067,7 +1069,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc if ((*a != source)&&(*a != controller())) { Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CONFIG); outp.append(reinterpret_cast(chunk.data()) + start,chunk.size() - start); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } } @@ -1126,7 +1128,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc } if (nc) { - this->setConfiguration(*nc,true); + this->setConfiguration(tPtr,*nc,true); delete nc; return configUpdateId; } else { @@ -1136,7 +1138,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc return 0; } -int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) +int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToDisk) { // _lock is NOT locked when this is called try { @@ -1156,7 +1158,7 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) _portInitialized = true; _externalConfig(&ctmp); } - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); if (saveToDisk) { Dictionary *d = new Dictionary(); @@ -1164,7 +1166,7 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) char n[64]; Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); if (nconf.toDictionary(*d,false)) - RR->node->dataStorePut(n,(const void *)d->data(),d->sizeBytes(),true); + RR->node->dataStorePut(tPtr,n,(const void *)d->data(),d->sizeBytes(),true); } catch ( ... ) {} delete d; } @@ -1176,7 +1178,7 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) return 0; } -void Network::requestConfiguration() +void Network::requestConfiguration(void *tPtr) { /* ZeroTier addresses can't begin with 0xff, so this is used to mark controllerless * network IDs. Controllerless network IDs only support unicast IPv6 using the 6plane @@ -1236,7 +1238,7 @@ void Network::requestConfiguration() nconf->type = ZT_NETWORK_TYPE_PUBLIC; Utils::snprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); - this->setConfiguration(*nconf,false); + this->setConfiguration(tPtr,*nconf,false); delete nconf; } else { this->setNotFound(); @@ -1284,10 +1286,10 @@ void Network::requestConfiguration() } outp.compress(); RR->node->expectReplyTo(outp.packetId()); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } -bool Network::gate(const SharedPtr &peer) +bool Network::gate(void *tPtr,const SharedPtr &peer) { const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); @@ -1298,8 +1300,8 @@ bool Network::gate(const SharedPtr &peer) if (!m) m = &(_membership(peer->address())); if (m->shouldLikeMulticasts(now)) { - m->pushCredentials(RR,now,peer->address(),_config,-1,false); - _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); + m->pushCredentials(RR,tPtr,now,peer->address(),_config,-1,false); + _announceMulticastGroupsTo(tPtr,peer->address(),_allMulticastGroups()); m->likingMulticasts(now); } return true; @@ -1377,31 +1379,31 @@ void Network::learnBridgeRoute(const MAC &mac,const Address &addr) } } -void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) +void Network::learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now) { Mutex::Lock _l(_lock); const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); _multicastGroupsBehindMe.set(mg,now); if (tmp != _multicastGroupsBehindMe.size()) - _sendUpdatesToMembers(&mg); + _sendUpdatesToMembers(tPtr,&mg); } -Membership::AddCredentialResult Network::addCredential(const CertificateOfMembership &com) +Membership::AddCredentialResult Network::addCredential(void *tPtr,const CertificateOfMembership &com) { if (com.networkId() != _id) return Membership::ADD_REJECTED; const Address a(com.issuedTo()); Mutex::Lock _l(_lock); Membership &m = _membership(a); - const Membership::AddCredentialResult result = m.addCredential(RR,_config,com); + const Membership::AddCredentialResult result = m.addCredential(RR,tPtr,_config,com); if ((result == Membership::ADD_ACCEPTED_NEW)||(result == Membership::ADD_ACCEPTED_REDUNDANT)) { - m.pushCredentials(RR,RR->node->now(),a,_config,-1,false); - RR->mc->addCredential(com,true); + m.pushCredentials(RR,tPtr,RR->node->now(),a,_config,-1,false); + RR->mc->addCredential(tPtr,com,true); } return result; } -Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,const Revocation &rev) +Membership::AddCredentialResult Network::addCredential(void *tPtr,const Address &sentFrom,const Revocation &rev) { if (rev.networkId() != _id) return Membership::ADD_REJECTED; @@ -1409,7 +1411,7 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c Mutex::Lock _l(_lock); Membership &m = _membership(rev.target()); - const Membership::AddCredentialResult result = m.addCredential(RR,_config,rev); + const Membership::AddCredentialResult result = m.addCredential(RR,tPtr,_config,rev); if ((result == Membership::ADD_ACCEPTED_NEW)&&(rev.fastPropagate())) { Address *a = (Address *)0; @@ -1424,7 +1426,7 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c outp.append((uint16_t)1); // one revocation! rev.serialize(outp); outp.append((uint16_t)0); // no certificates of ownership - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } } @@ -1495,7 +1497,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const } } -void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup) +void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup) { // Assumes _lock is locked const uint64_t now = RR->node->now(); @@ -1521,9 +1523,9 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou outp.append((uint16_t)0); // no tags outp.append((uint16_t)0); // no revocations outp.append((uint16_t)0); // no certificates of ownership - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } - _announceMulticastGroupsTo(*a,groups); + _announceMulticastGroupsTo(tPtr,*a,groups); } // Also announce to controller, and send COM to simplify and generalize behavior even though in theory it does not need it @@ -1537,9 +1539,9 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou outp.append((uint16_t)0); // no tags outp.append((uint16_t)0); // no revocations outp.append((uint16_t)0); // no certificates of ownership - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } - _announceMulticastGroupsTo(c,groups); + _announceMulticastGroupsTo(tPtr,c,groups); } } @@ -1556,17 +1558,17 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - m->pushCredentials(RR,now,*a,_config,-1,false); + m->pushCredentials(RR,tPtr,now,*a,_config,-1,false); if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { if (!newMulticastGroup) m->likingMulticasts(now); - _announceMulticastGroupsTo(*a,groups); + _announceMulticastGroupsTo(tPtr,*a,groups); } } } } -void Network::_announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups) +void Network::_announceMulticastGroupsTo(void *tPtr,const Address &peer,const std::vector &allMulticastGroups) { // Assumes _lock is locked Packet outp(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); @@ -1574,7 +1576,7 @@ void Network::_announceMulticastGroupsTo(const Address &peer,const std::vector::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) { if ((outp.size() + 24) >= ZT_PROTO_MAX_PACKET_LENGTH) { outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); outp.reset(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); } @@ -1586,7 +1588,7 @@ void Network::_announceMulticastGroupsTo(const Address &peer,const std::vector ZT_PROTO_MIN_PACKET_LENGTH) { outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } diff --git a/node/Network.hpp b/node/Network.hpp index 6cf6d974..fccc267a 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -77,10 +77,11 @@ public: * constructed to actually configure the port. * * @param renv Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param nwid Network ID * @param uptr Arbitrary pointer used by externally-facing API (for user use) */ - Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr); + Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr); ~Network(); @@ -101,6 +102,7 @@ public: * such as TEE may be taken, and credentials may be pushed, so this is not * side-effect-free. It's basically step one in sending something over VL2. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param noTee If true, do not TEE anything anywhere (for two-pass filtering as done with multicast and bridging) * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address @@ -113,6 +115,7 @@ public: * @return True if packet should be sent, false if dropped or redirected */ bool filterOutgoingPacket( + void *tPtr, const bool noTee, const Address &ztSource, const Address &ztDest, @@ -131,6 +134,7 @@ public: * a match certain actions may be taken such as sending a copy of the packet * to a TEE or REDIRECT target. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param sourcePeer Source Peer * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -142,6 +146,7 @@ public: * @return 0 == drop, 1 == accept, 2 == accept even if bridged */ int filterIncomingPacket( + void *tPtr, const SharedPtr &sourcePeer, const Address &ztDest, const MAC &macSource, @@ -163,9 +168,10 @@ public: /** * Subscribe to a multicast group * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param mg New multicast group */ - void multicastSubscribe(const MulticastGroup &mg); + void multicastSubscribe(void *tPtr,const MulticastGroup &mg); /** * Unsubscribe from a multicast group @@ -181,22 +187,24 @@ public: * chunks via OK(NETWORK_CONFIG_REQUEST) or NETWORK_CONFIG. It verifies * each chunk and once assembled applies the configuration. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param packetId Packet ID or 0 if none (e.g. via cluster path) * @param source Address of sender of chunk or NULL if none (e.g. via cluster path) * @param chunk Buffer containing chunk * @param ptr Index of chunk and related fields in packet * @return Update ID if update was fully assembled and accepted or 0 otherwise */ - uint64_t handleConfigChunk(const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr); + uint64_t handleConfigChunk(void *tPtr,const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr); /** * Set network configuration * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param nconf Network configuration * @param saveToDisk Save to disk? Used during loading, should usually be true otherwise. * @return 0 == bad, 1 == accepted but duplicate/unchanged, 2 == accepted and new */ - int setConfiguration(const NetworkConfig &nconf,bool saveToDisk); + int setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToDisk); /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this @@ -218,13 +226,18 @@ public: /** * Causes this network to request an updated configuration from its master node now + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call */ - void requestConfiguration(); + void requestConfiguration(void *tPtr); /** * Determine whether this peer is permitted to communicate on this network + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param peer Peer to check */ - bool gate(const SharedPtr &peer); + bool gate(void *tPtr,const SharedPtr &peer); /** * Do periodic cleanup and housekeeping tasks @@ -233,11 +246,13 @@ public: /** * Push state to members such as multicast group memberships and latest COM (if needed) + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call */ - inline void sendUpdatesToMembers() + inline void sendUpdatesToMembers(void *tPtr) { Mutex::Lock _l(_lock); - _sendUpdatesToMembers((const MulticastGroup *)0); + _sendUpdatesToMembers(tPtr,(const MulticastGroup *)0); } /** @@ -264,64 +279,66 @@ public: /** * Learn a multicast group that is bridged to our tap device * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param mg Multicast group * @param now Current time */ - void learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now); + void learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now); /** * Validate a credential and learn it if it passes certificate and other checks */ - Membership::AddCredentialResult addCredential(const CertificateOfMembership &com); + Membership::AddCredentialResult addCredential(void *tPtr,const CertificateOfMembership &com); /** * Validate a credential and learn it if it passes certificate and other checks */ - inline Membership::AddCredentialResult addCredential(const Capability &cap) + inline Membership::AddCredentialResult addCredential(void *tPtr,const Capability &cap) { if (cap.networkId() != _id) return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(cap.issuedTo()).addCredential(RR,_config,cap); + return _membership(cap.issuedTo()).addCredential(RR,tPtr,_config,cap); } /** * Validate a credential and learn it if it passes certificate and other checks */ - inline Membership::AddCredentialResult addCredential(const Tag &tag) + inline Membership::AddCredentialResult addCredential(void *tPtr,const Tag &tag) { if (tag.networkId() != _id) return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(tag.issuedTo()).addCredential(RR,_config,tag); + return _membership(tag.issuedTo()).addCredential(RR,tPtr,_config,tag); } /** * Validate a credential and learn it if it passes certificate and other checks */ - Membership::AddCredentialResult addCredential(const Address &sentFrom,const Revocation &rev); + Membership::AddCredentialResult addCredential(void *tPtr,const Address &sentFrom,const Revocation &rev); /** * Validate a credential and learn it if it passes certificate and other checks */ - inline Membership::AddCredentialResult addCredential(const CertificateOfOwnership &coo) + inline Membership::AddCredentialResult addCredential(void *tPtr,const CertificateOfOwnership &coo) { if (coo.networkId() != _id) return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(coo.issuedTo()).addCredential(RR,_config,coo); + return _membership(coo.issuedTo()).addCredential(RR,tPtr,_config,coo); } /** * Force push credentials (COM, etc.) to a peer now * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param to Destination peer address * @param now Current time */ - inline void pushCredentialsNow(const Address &to,const uint64_t now) + inline void pushCredentialsNow(void *tPtr,const Address &to,const uint64_t now) { Mutex::Lock _l(_lock); - _membership(to).pushCredentials(RR,now,to,_config,-1,true); + _membership(to).pushCredentials(RR,tPtr,now,to,_config,-1,true); } /** @@ -353,8 +370,8 @@ private: ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); - void _sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup); - void _announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups); + void _sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup); + void _announceMulticastGroupsTo(void *tPtr,const Address &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; Membership &_membership(const Address &a); diff --git a/node/Node.cpp b/node/Node.cpp index 1125ca7a..4e8d6655 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -46,7 +46,7 @@ namespace ZeroTier { /* Public Node interface (C++, exposed via CAPI bindings) */ /****************************************************************************/ -Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : +Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : _RR(this), RR(&_RR), _uPtr(uptr), @@ -72,26 +72,26 @@ Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : memset(_prngStream,0,sizeof(_prngStream)); _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - std::string idtmp(dataStoreGet("identity.secret")); + std::string idtmp(dataStoreGet(tptr,"identity.secret")); if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { TRACE("identity.secret not found, generating..."); RR->identity.generate(); idtmp = RR->identity.toString(true); - if (!dataStorePut("identity.secret",idtmp,true)) + if (!dataStorePut(tptr,"identity.secret",idtmp,true)) throw std::runtime_error("unable to write identity.secret"); } RR->publicIdentityStr = RR->identity.toString(false); RR->secretIdentityStr = RR->identity.toString(true); - idtmp = dataStoreGet("identity.public"); + idtmp = dataStoreGet(tptr,"identity.public"); if (idtmp != RR->publicIdentityStr) { - if (!dataStorePut("identity.public",RR->publicIdentityStr,false)) + if (!dataStorePut(tptr,"identity.public",RR->publicIdentityStr,false)) throw std::runtime_error("unable to write identity.public"); } try { RR->sw = new Switch(RR); RR->mc = new Multicaster(RR); - RR->topology = new Topology(RR); + RR->topology = new Topology(RR,tptr); RR->sa = new SelfAwareness(RR); } catch ( ... ) { delete RR->sa; @@ -101,7 +101,7 @@ Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : throw; } - postEvent(ZT_EVENT_UP); + postEvent(tptr,ZT_EVENT_UP); } Node::~Node() @@ -121,6 +121,7 @@ Node::~Node() } ZT_ResultCode Node::processWirePacket( + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -129,11 +130,12 @@ ZT_ResultCode Node::processWirePacket( volatile uint64_t *nextBackgroundTaskDeadline) { _now = now; - RR->sw->onRemotePacket(*(reinterpret_cast(localAddress)),*(reinterpret_cast(remoteAddress)),packetData,packetLength); + RR->sw->onRemotePacket(tptr,*(reinterpret_cast(localAddress)),*(reinterpret_cast(remoteAddress)),packetData,packetLength); return ZT_RESULT_OK; } ZT_ResultCode Node::processVirtualNetworkFrame( + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -147,7 +149,7 @@ ZT_ResultCode Node::processVirtualNetworkFrame( _now = now; SharedPtr nw(this->network(nwid)); if (nw) { - RR->sw->onLocalEthernet(nw,MAC(sourceMac),MAC(destMac),etherType,vlanId,frameData,frameLength); + RR->sw->onLocalEthernet(tptr,nw,MAC(sourceMac),MAC(destMac),etherType,vlanId,frameData,frameLength); return ZT_RESULT_OK; } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } @@ -156,9 +158,10 @@ ZT_ResultCode Node::processVirtualNetworkFrame( class _PingPeersThatNeedPing { public: - _PingPeersThatNeedPing(const RuntimeEnvironment *renv,Hashtable< Address,std::vector > &upstreamsToContact,uint64_t now) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,void *tPtr,Hashtable< Address,std::vector > &upstreamsToContact,uint64_t now) : lastReceiveFromUpstream(0), RR(renv), + _tPtr(tPtr), _upstreamsToContact(upstreamsToContact), _now(now), _bestCurrentUpstream(RR->topology->getUpstreamPeer()) @@ -176,21 +179,21 @@ public: // Upstreams must be pinged constantly over both IPv4 and IPv6 to allow // them to perform three way handshake introductions for both stacks. - if (!p->doPingAndKeepalive(_now,AF_INET)) { + if (!p->doPingAndKeepalive(_tPtr,_now,AF_INET)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET) { - p->sendHELLO(InetAddress(),addr,_now,0); + p->sendHELLO(_tPtr,InetAddress(),addr,_now,0); contacted = true; break; } } } else contacted = true; - if (!p->doPingAndKeepalive(_now,AF_INET6)) { + if (!p->doPingAndKeepalive(_tPtr,_now,AF_INET6)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET6) { - p->sendHELLO(InetAddress(),addr,_now,0); + p->sendHELLO(_tPtr,InetAddress(),addr,_now,0); contacted = true; break; } @@ -200,24 +203,25 @@ public: if ((!contacted)&&(_bestCurrentUpstream)) { const SharedPtr up(_bestCurrentUpstream->getBestPath(_now,true)); if (up) - p->sendHELLO(up->localAddress(),up->address(),_now,up->nextOutgoingCounter()); + p->sendHELLO(_tPtr,up->localAddress(),up->address(),_now,up->nextOutgoingCounter()); } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); _upstreamsToContact.erase(p->address()); // erase from upstreams to contact so that we can WHOIS those that remain } else if (p->isActive(_now)) { - p->doPingAndKeepalive(_now,-1); + p->doPingAndKeepalive(_tPtr,_now,-1); } } private: const RuntimeEnvironment *RR; + void *_tPtr; Hashtable< Address,std::vector > &_upstreamsToContact; const uint64_t _now; const SharedPtr _bestCurrentUpstream; }; -ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) { _now = now; Mutex::Lock bl(_backgroundTasksLock); @@ -235,16 +239,16 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) needConfig.push_back(n->second); - n->second->sendUpdatesToMembers(); + n->second->sendUpdatesToMembers(tptr); } } for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) - (*n)->requestConfiguration(); + (*n)->requestConfiguration(tptr); // Do pings and keepalives Hashtable< Address,std::vector > upstreamsToContact; RR->topology->getUpstreamsToContact(upstreamsToContact); - _PingPeersThatNeedPing pfunc(RR,upstreamsToContact,now); + _PingPeersThatNeedPing pfunc(RR,tptr,upstreamsToContact,now); RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); // Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds) @@ -252,13 +256,13 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB Address *upstreamAddress = (Address *)0; std::vector *upstreamStableEndpoints = (std::vector *)0; while (i.next(upstreamAddress,upstreamStableEndpoints)) - RR->sw->requestWhois(*upstreamAddress); + RR->sw->requestWhois(tptr,*upstreamAddress); // Update online status, post status change as event const bool oldOnline = _online; _online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot())); if (oldOnline != _online) - postEvent(_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE); + postEvent(tptr,_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } @@ -286,7 +290,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB *nextBackgroundTaskDeadline = now + ZT_CLUSTER_PERIODIC_TASK_PERIOD; // this is really short so just tick at this rate } else { #endif - *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); + *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); #ifdef ZT_ENABLE_CLUSTER } #endif @@ -297,17 +301,17 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB return ZT_RESULT_OK; } -ZT_ResultCode Node::join(uint64_t nwid,void *uptr) +ZT_ResultCode Node::join(uint64_t nwid,void *uptr,void *tptr) { Mutex::Lock _l(_networks_m); SharedPtr nw = _network(nwid); if(!nw) - _networks.push_back(std::pair< uint64_t,SharedPtr >(nwid,SharedPtr(new Network(RR,nwid,uptr)))); + _networks.push_back(std::pair< uint64_t,SharedPtr >(nwid,SharedPtr(new Network(RR,tptr,nwid,uptr)))); std::sort(_networks.begin(),_networks.end()); // will sort by nwid since it's the first in a pair<> return ZT_RESULT_OK; } -ZT_ResultCode Node::leave(uint64_t nwid,void **uptr) +ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) { std::vector< std::pair< uint64_t,SharedPtr > > newn; Mutex::Lock _l(_networks_m); @@ -324,11 +328,11 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr) return ZT_RESULT_OK; } -ZT_ResultCode Node::multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) +ZT_ResultCode Node::multicastSubscribe(void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) { SharedPtr nw(this->network(nwid)); if (nw) { - nw->multicastSubscribe(MulticastGroup(MAC(multicastGroup),(uint32_t)(multicastAdi & 0xffffffff))); + nw->multicastSubscribe(tptr,MulticastGroup(MAC(multicastGroup),(uint32_t)(multicastAdi & 0xffffffff))); return ZT_RESULT_OK; } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } @@ -342,15 +346,15 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } -ZT_ResultCode Node::orbit(uint64_t moonWorldId,uint64_t moonSeed) +ZT_ResultCode Node::orbit(void *tptr,uint64_t moonWorldId,uint64_t moonSeed) { - RR->topology->addMoon(moonWorldId,Address(moonSeed)); + RR->topology->addMoon(tptr,moonWorldId,Address(moonSeed)); return ZT_RESULT_OK; } -ZT_ResultCode Node::deorbit(uint64_t moonWorldId) +ZT_ResultCode Node::deorbit(void *tptr,uint64_t moonWorldId) { - RR->topology->removeMoon(moonWorldId); + RR->topology->removeMoon(tptr,moonWorldId); return ZT_RESULT_OK; } @@ -465,7 +469,7 @@ void Node::clearLocalInterfaceAddresses() _directPaths.clear(); } -int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len) +int Node::sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len) { try { if (RR->identity.address().toInt() != dest) { @@ -473,7 +477,7 @@ int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigne outp.append(typeId); outp.append(data,len); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tptr,outp,true); return 1; } } catch ( ... ) {} @@ -486,7 +490,7 @@ void Node::setNetconfMaster(void *networkControllerInstance) RR->localNetworkController->init(RR->identity,this); } -ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) +ZT_ResultCode Node::circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) { if (test->hopCount > 0) { try { @@ -516,7 +520,7 @@ ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback) for(unsigned int a=0;ahops[0].breadth;++a) { outp.newInitializationVector(); outp.setDestination(Address(test->hops[0].addresses[a])); - RR->sw->send(outp,true); + RR->sw->send(tptr,outp,true); } } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet @@ -616,13 +620,13 @@ void Node::clusterStatus(ZT_ClusterStatus *cs) /* Node methods used only within node/ */ /****************************************************************************/ -std::string Node::dataStoreGet(const char *name) +std::string Node::dataStoreGet(void *tPtr,const char *name) { char buf[1024]; std::string r; unsigned long olen = 0; do { - long n = _cb.dataStoreGetFunction(reinterpret_cast(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen); + long n = _cb.dataStoreGetFunction(reinterpret_cast(this),_uPtr,tPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen); if (n <= 0) return std::string(); r.append(buf,n); @@ -630,7 +634,7 @@ std::string Node::dataStoreGet(const char *name) return r; } -bool Node::shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress) +bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress) { if (!Path::isAddressValidForPath(remoteAddress)) return false; @@ -650,7 +654,7 @@ bool Node::shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddre } } - return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),reinterpret_cast(&localAddress),reinterpret_cast(&remoteAddress)) != 0) : true); + return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),reinterpret_cast(&localAddress),reinterpret_cast(&remoteAddress)) != 0) : true); } #ifdef ZT_TRACE @@ -728,7 +732,7 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de if (destination == RR->identity.address()) { SharedPtr n(network(nwid)); if (!n) return; - n->setConfiguration(nc,true); + n->setConfiguration((void *)0,nc,true); } else { Dictionary *dconf = new Dictionary(); try { @@ -762,7 +766,7 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de outp.append(sig.data,ZT_C25519_SIGNATURE_LEN); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send((void *)0,outp,true); chunkIndex += chunkLen; } } @@ -779,7 +783,7 @@ void Node::ncSendRevocation(const Address &destination,const Revocation &rev) if (destination == RR->identity.address()) { SharedPtr n(network(rev.networkId())); if (!n) return; - n->addCredential(RR->identity.address(),rev); + n->addCredential((void *)0,RR->identity.address(),rev); } else { Packet outp(destination,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); outp.append((uint8_t)0x00); @@ -788,7 +792,7 @@ void Node::ncSendRevocation(const Address &destination,const Revocation &rev) outp.append((uint16_t)1); rev.serialize(outp); outp.append((uint16_t)0); - RR->sw->send(outp,true); + RR->sw->send((void *)0,outp,true); } } @@ -823,7 +827,7 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des break; } outp.append(nwid); - RR->sw->send(outp,true); + RR->sw->send((void *)0,outp,true); } // else we can't send an ERROR() in response to nothing, so discard } @@ -835,11 +839,11 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des extern "C" { -enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) { *node = (ZT_Node *)0; try { - *node = reinterpret_cast(new ZeroTier::Node(uptr,callbacks,now)); + *node = reinterpret_cast(new ZeroTier::Node(uptr,tptr,callbacks,now)); return ZT_RESULT_OK; } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; @@ -859,6 +863,7 @@ void ZT_Node_delete(ZT_Node *node) enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -867,7 +872,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( volatile uint64_t *nextBackgroundTaskDeadline) { try { - return reinterpret_cast(node)->processWirePacket(now,localAddress,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); + return reinterpret_cast(node)->processWirePacket(tptr,now,localAddress,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -877,6 +882,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -888,7 +894,7 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( volatile uint64_t *nextBackgroundTaskDeadline) { try { - return reinterpret_cast(node)->processVirtualNetworkFrame(now,nwid,sourceMac,destMac,etherType,vlanId,frameData,frameLength,nextBackgroundTaskDeadline); + return reinterpret_cast(node)->processVirtualNetworkFrame(tptr,now,nwid,sourceMac,destMac,etherType,vlanId,frameData,frameLength,nextBackgroundTaskDeadline); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -896,10 +902,10 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( } } -enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) { try { - return reinterpret_cast(node)->processBackgroundTasks(now,nextBackgroundTaskDeadline); + return reinterpret_cast(node)->processBackgroundTasks(tptr,now,nextBackgroundTaskDeadline); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -907,10 +913,10 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,vol } } -enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr) +enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr,void *tptr) { try { - return reinterpret_cast(node)->join(nwid,uptr); + return reinterpret_cast(node)->join(nwid,uptr,tptr); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -918,10 +924,10 @@ enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr) } } -enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr) +enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr,void *tptr) { try { - return reinterpret_cast(node)->leave(nwid,uptr); + return reinterpret_cast(node)->leave(nwid,uptr,tptr); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -929,10 +935,10 @@ enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr) } } -enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) +enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) { try { - return reinterpret_cast(node)->multicastSubscribe(nwid,multicastGroup,multicastAdi); + return reinterpret_cast(node)->multicastSubscribe(tptr,nwid,multicastGroup,multicastAdi); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -951,19 +957,19 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint } } -enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed) +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,uint64_t moonSeed) { try { - return reinterpret_cast(node)->orbit(moonWorldId,moonSeed); + return reinterpret_cast(node)->orbit(tptr,moonWorldId,moonSeed); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } } -ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId) +ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId) { try { - return reinterpret_cast(node)->deorbit(moonWorldId); + return reinterpret_cast(node)->deorbit(tptr,moonWorldId); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } @@ -1031,10 +1037,10 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node) } catch ( ... ) {} } -int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len) +int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len) { try { - return reinterpret_cast(node)->sendUserMessage(dest,typeId,data,len); + return reinterpret_cast(node)->sendUserMessage(tptr,dest,typeId,data,len); } catch ( ... ) { return 0; } @@ -1047,10 +1053,10 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) } catch ( ... ) {} } -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) +enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) { try { - return reinterpret_cast(node)->circuitTestBegin(test,reportCallback); + return reinterpret_cast(node)->circuitTestBegin(tptr,test,reportCallback); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } diff --git a/node/Node.hpp b/node/Node.hpp index 21eac617..03bd7a8c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -65,7 +65,7 @@ class World; class Node : public NetworkController::Sender { public: - Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); + Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); virtual ~Node(); // Get rid of alignment warnings on 32-bit Windows and possibly improve performance @@ -77,6 +77,7 @@ public: // Public API Functions ---------------------------------------------------- ZT_ResultCode processWirePacket( + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -84,6 +85,7 @@ public: unsigned int packetLength, volatile uint64_t *nextBackgroundTaskDeadline); ZT_ResultCode processVirtualNetworkFrame( + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -93,13 +95,13 @@ public: const void *frameData, unsigned int frameLength, volatile uint64_t *nextBackgroundTaskDeadline); - ZT_ResultCode processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); - ZT_ResultCode join(uint64_t nwid,void *uptr); - ZT_ResultCode leave(uint64_t nwid,void **uptr); - ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); + ZT_ResultCode processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); + ZT_ResultCode join(uint64_t nwid,void *uptr,void *tptr); + ZT_ResultCode leave(uint64_t nwid,void **uptr,void *tptr); + ZT_ResultCode multicastSubscribe(void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); - ZT_ResultCode orbit(uint64_t moonWorldId,uint64_t moonSeed); - ZT_ResultCode deorbit(uint64_t moonWorldId); + ZT_ResultCode orbit(void *tptr,uint64_t moonWorldId,uint64_t moonSeed); + ZT_ResultCode deorbit(void *tptr,uint64_t moonWorldId); uint64_t address() const; void status(ZT_NodeStatus *status) const; ZT_PeerList *peers() const; @@ -108,9 +110,9 @@ public: void freeQueryResult(void *qr); int addLocalInterfaceAddress(const struct sockaddr_storage *addr); void clearLocalInterfaceAddresses(); - int sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len); + int sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); void setNetconfMaster(void *networkControllerInstance); - ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); + ZT_ResultCode circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); ZT_ResultCode clusterInit( unsigned int myId, @@ -132,11 +134,12 @@ public: inline uint64_t now() const throw() { return _now; } - inline bool putPacket(const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) + inline bool putPacket(void *tPtr,const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) { return (_cb.wirePacketSendFunction( reinterpret_cast(this), _uPtr, + tPtr, reinterpret_cast(&localAddress), reinterpret_cast(&addr), data, @@ -144,11 +147,12 @@ public: ttl) == 0); } - inline void putFrame(uint64_t nwid,void **nuptr,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) + inline void putFrame(void *tPtr,uint64_t nwid,void **nuptr,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { _cb.virtualNetworkFrameFunction( reinterpret_cast(this), _uPtr, + tPtr, nwid, nuptr, source.toInt(), @@ -191,14 +195,14 @@ public: return _directPaths; } - inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,name,data,len,(int)secure) == 0); } - inline bool dataStorePut(const char *name,const std::string &data,bool secure) { return dataStorePut(name,(const void *)data.data(),(unsigned int)data.length(),secure); } - inline void dataStoreDelete(const char *name) { _cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,name,(const void *)0,0,0); } - std::string dataStoreGet(const char *name); + inline bool dataStorePut(void *tPtr,const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,tPtr,name,data,len,(int)secure) == 0); } + inline bool dataStorePut(void *tPtr,const char *name,const std::string &data,bool secure) { return dataStorePut(tPtr,name,(const void *)data.data(),(unsigned int)data.length(),secure); } + inline void dataStoreDelete(void *tPtr,const char *name) { _cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,tPtr,name,(const void *)0,0,0); } + std::string dataStoreGet(void *tPtr,const char *name); - inline void postEvent(ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast(this),_uPtr,ev,md); } + inline void postEvent(void *tPtr,ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast(this),_uPtr,tPtr,ev,md); } - inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,nwid,nuptr,op,nc); } + inline int configureVirtualNetworkPort(void *tPtr,uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,tPtr,nwid,nuptr,op,nc); } inline bool online() const throw() { return _online; } @@ -206,8 +210,8 @@ public: void postTrace(const char *module,unsigned int line,const char *fmt,...); #endif - bool shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress); - inline bool externalPathLookup(const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } + bool shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress); + inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } uint64_t prng(); void postCircuitTestReport(const ZT_CircuitTestReport *report); diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index d4cb87cb..285bfa5d 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -85,18 +85,18 @@ void OutboundMulticast::init( memcpy(_frameData,payload,_frameLen); } -void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) +void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr) { const SharedPtr nw(RR->node->network(_nwid)); const Address toAddr2(toAddr); - if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { + if ((nw)&&(nw->filterOutgoingPacket(tPtr,true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); _packet.setDestination(toAddr2); RR->node->expectReplyTo(_packet.packetId()); Packet tmp(_packet); // make a copy of packet so as not to garble the original -- GitHub issue #461 - RR->sw->send(tmp,true); + RR->sw->send(tPtr,tmp,true); } } diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 6370d0d7..0ecf113f 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -99,33 +99,36 @@ public: * Just send without checking log * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param toAddr Destination address */ - void sendOnly(const RuntimeEnvironment *RR,const Address &toAddr); + void sendOnly(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr); /** * Just send and log but do not check sent log * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param toAddr Destination address */ - inline void sendAndLog(const RuntimeEnvironment *RR,const Address &toAddr) + inline void sendAndLog(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr) { _alreadySentTo.push_back(toAddr); - sendOnly(RR,toAddr); + sendOnly(RR,tPtr,toAddr); } /** * Try to send this to a given peer if it hasn't been sent to them already * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param toAddr Destination address * @return True if address is new and packet was sent to switch, false if duplicate */ - inline bool sendIfNew(const RuntimeEnvironment *RR,const Address &toAddr) + inline bool sendIfNew(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr) { if (std::find(_alreadySentTo.begin(),_alreadySentTo.end(),toAddr) == _alreadySentTo.end()) { - sendAndLog(RR,toAddr); + sendAndLog(RR,tPtr,toAddr); return true; } else { return false; diff --git a/node/Path.cpp b/node/Path.cpp index 5592bacc..7366b56f 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -22,9 +22,9 @@ namespace ZeroTier { -bool Path::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now) +bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now) { - if (RR->node->putPacket(_localAddress,address(),data,len)) { + if (RR->node->putPacket(tPtr,_localAddress,address(),data,len)) { _lastOut = now; return true; } diff --git a/node/Path.hpp b/node/Path.hpp index 62f29c22..aef628d4 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -186,12 +186,13 @@ public: * Send a packet via this path (last out time is also updated) * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param data Packet data * @param len Packet length * @param now Current time * @return True if transport reported success */ - bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now); + bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now); /** * Manually update last sent time diff --git a/node/Peer.cpp b/node/Peer.cpp index fa3ce6c8..0cc23e33 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -68,6 +68,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident } void Peer::received( + void *tPtr, const SharedPtr &path, const unsigned int hops, const uint64_t packetId, @@ -161,7 +162,7 @@ void Peer::received( } } - if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(_id.address(),path->localAddress(),path->address())) ) { + if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { if (verb == Packet::VERB_OK) { Mutex::Lock _l(_paths_m); @@ -206,7 +207,7 @@ void Peer::received( #endif } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - attemptToContactAt(path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); + attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); path->sent(now); } } @@ -281,7 +282,7 @@ void Peer::received( if (count) { outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); outp.armor(_key,true,path->nextOutgoingCounter()); - path->send(RR,outp.data(),outp.size(),now); + path->send(RR,tPtr,outp.data(),outp.size(),now); } } } @@ -299,7 +300,7 @@ bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const return false; } -bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) +bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) { Mutex::Lock _l(_paths_m); @@ -316,7 +317,7 @@ bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceE } if (bestp >= 0) { - return _paths[bestp].path->send(RR,data,len,now); + return _paths[bestp].path->send(RR,tPtr,data,len,now); } else { return false; } @@ -345,7 +346,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) } } -void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) +void Peer::sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); @@ -387,35 +388,35 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u if (atAddress) { outp.armor(_key,false,counter); // false == don't encrypt full payload, but add MAC - RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,localAddr,atAddress,outp.data(),outp.size()); } else { - RR->sw->send(outp,false); // false == don't encrypt full payload, but add MAC + RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC } } -void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) +void Peer::attemptToContactAt(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) { if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,true,counter); - RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,localAddr,atAddress,outp.data(),outp.size()); } else { - sendHELLO(localAddr,atAddress,now,counter); + sendHELLO(tPtr,localAddr,atAddress,now,counter); } } -void Peer::tryMemorizedPath(uint64_t now) +void Peer::tryMemorizedPath(void *tPtr,uint64_t now) { if ((now - _lastTriedMemorizedPath) >= ZT_TRY_MEMORIZED_PATH_INTERVAL) { _lastTriedMemorizedPath = now; InetAddress mp; - if (RR->node->externalPathLookup(_id.address(),-1,mp)) - attemptToContactAt(InetAddress(),mp,now,true,0); + if (RR->node->externalPathLookup(tPtr,_id.address(),-1,mp)) + attemptToContactAt(tPtr,InetAddress(),mp,now,true,0); } } -bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) +bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); @@ -433,7 +434,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) if (bestp >= 0) { if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) { - attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false,_paths[bestp].path->nextOutgoingCounter()); + attemptToContactAt(tPtr,_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false,_paths[bestp].path->nextOutgoingCounter()); _paths[bestp].path->sent(now); } return true; @@ -452,12 +453,12 @@ bool Peer::hasActiveDirectPath(uint64_t now) const return false; } -void Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) +void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) { Mutex::Lock _l(_paths_m); for(unsigned int p=0;p<_numPaths;++p) { if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { - attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now,false,_paths[p].path->nextOutgoingCounter()); + attemptToContactAt(tPtr,_paths[p].path->localAddress(),_paths[p].path->address(),now,false,_paths[p].path->nextOutgoingCounter()); _paths[p].path->sent(now); _paths[p].lastReceive = 0; // path will not be used unless it speaks again } diff --git a/node/Peer.hpp b/node/Peer.hpp index 72040b1d..41836410 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -84,6 +84,7 @@ public: * This is called by the decode pipe when a packet is proven to be authentic * and appears to be valid. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param path Path over which packet was received * @param hops ZeroTier (not IP) hops * @param packetId Packet ID @@ -93,6 +94,7 @@ public: * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established */ void received( + void *tPtr, const SharedPtr &path, const unsigned int hops, const uint64_t packetId, @@ -125,13 +127,14 @@ public: /** * Send via best direct path * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param data Packet data * @param len Packet length * @param now Current time * @param forceEvenIfDead If true, send even if the path is not 'alive' * @return True if we actually sent something */ - bool sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); + bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); /** * Get the best current direct path @@ -147,41 +150,47 @@ public: * * No statistics or sent times are updated here. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param localAddr Local address * @param atAddress Destination address * @param now Current time * @param counter Outgoing packet counter */ - void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter); + void sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter); /** * Send ECHO (or HELLO for older peers) to this peer at the given address * * No statistics or sent times are updated here. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param localAddr Local address * @param atAddress Destination address * @param now Current time * @param sendFullHello If true, always send a full HELLO instead of just an ECHO * @param counter Outgoing packet counter */ - void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); + void attemptToContactAt(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); /** * Try a memorized or statically defined path if any are known * * Under the hood this is done periodically based on ZT_TRY_MEMORIZED_PATH_INTERVAL. + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param now Current time */ - void tryMemorizedPath(uint64_t now); + void tryMemorizedPath(void *tPtr,uint64_t now); /** * Send pings or keepalives depending on configured timeouts * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @param inetAddressFamily Keep this address family alive, or -1 for any * @return True if we have at least one direct path of the given family (or any if family is -1) */ - bool doPingAndKeepalive(uint64_t now,int inetAddressFamily); + bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); /** * @param now Current time @@ -195,11 +204,12 @@ public: * Resetting a path involves sending an ECHO to it and then deactivating * it until or unless it responds. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param scope IP scope * @param inetAddressFamily Family e.g. AF_INET * @param now Current time */ - void resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); + void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); /** * Get most recently active path addresses for IPv4 and/or IPv6 diff --git a/node/Revocation.cpp b/node/Revocation.cpp index 420476a4..bab5653c 100644 --- a/node/Revocation.cpp +++ b/node/Revocation.cpp @@ -25,13 +25,13 @@ namespace ZeroTier { -int Revocation::verify(const RuntimeEnvironment *RR) const +int Revocation::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } try { diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 93c55112..8b9ce6dd 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -113,9 +113,10 @@ public: * Verify this revocation's signature * * @param RR Runtime environment to provide for peer lookup, etc. + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template inline void serialize(Buffer &b,const bool forSign = false) const diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index e84b7b65..cba84cdc 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -40,15 +40,17 @@ namespace ZeroTier { class _ResetWithinScope { public: - _ResetWithinScope(uint64_t now,int inetAddressFamily,InetAddress::IpScope scope) : + _ResetWithinScope(void *tPtr,uint64_t now,int inetAddressFamily,InetAddress::IpScope scope) : _now(now), + _tPtr(tPtr), _family(inetAddressFamily), _scope(scope) {} - inline void operator()(Topology &t,const SharedPtr &p) { p->resetWithinScope(_scope,_family,_now); } + inline void operator()(Topology &t,const SharedPtr &p) { p->resetWithinScope(_tPtr,_scope,_family,_now); } private: uint64_t _now; + void *_tPtr; int _family; InetAddress::IpScope _scope; }; @@ -59,7 +61,7 @@ SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) : { } -void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) +void SelfAwareness::iam(void *tPtr,const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) { const InetAddress::IpScope scope = myPhysicalAddress.ipScope(); @@ -91,7 +93,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLoc } // Reset all paths within this scope and address family - _ResetWithinScope rset(now,myPhysicalAddress.ss_family,(InetAddress::IpScope)scope); + _ResetWithinScope rset(tPtr,now,myPhysicalAddress.ss_family,(InetAddress::IpScope)scope); RR->topology->eachPeer<_ResetWithinScope &>(rset); } else { // Otherwise just update DB to use to determine external surface info diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index 4bdafeb2..c1db0c84 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -47,7 +47,7 @@ public: * @param trusted True if this peer is trusted as an authority to inform us of external address changes * @param now Current time */ - void iam(const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); + void iam(void *tPtr,const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); /** * Clean up database periodically diff --git a/node/Switch.cpp b/node/Switch.cpp index aab2e7ff..62674472 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -64,7 +64,7 @@ Switch::Switch(const RuntimeEnvironment *renv) : { } -void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len) +void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len) { try { const uint64_t now = RR->node->now(); @@ -81,15 +81,15 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from const Address beaconAddr(reinterpret_cast(data) + 8,5); if (beaconAddr == RR->identity.address()) return; - if (!RR->node->shouldUsePathForZeroTierTraffic(beaconAddr,localAddr,fromAddr)) + if (!RR->node->shouldUsePathForZeroTierTraffic(tPtr,beaconAddr,localAddr,fromAddr)) return; - const SharedPtr peer(RR->topology->getPeer(beaconAddr)); + const SharedPtr peer(RR->topology->getPeer(tPtr,beaconAddr)); if (peer) { // we'll only respond to beacons from known peers if ((now - _lastBeaconResponse) >= 2500) { // limit rate of responses _lastBeaconResponse = now; Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NOP); outp.armor(peer->key(),true,path->nextOutgoingCounter()); - path->send(RR,outp.data(),outp.size(),now); + path->send(RR,tPtr,outp.data(),outp.size(),now); } } @@ -115,8 +115,8 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Note: we don't bother initiating NAT-t for fragments, since heads will set that off. // It wouldn't hurt anything, just redundant and unnecessary. - SharedPtr relayTo = RR->topology->getPeer(destination); - if ((!relayTo)||(!relayTo->sendDirect(fragment.data(),fragment.size(),now,false))) { + SharedPtr relayTo = RR->topology->getPeer(tPtr,destination); + if ((!relayTo)||(!relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,false))) { #ifdef ZT_ENABLE_CLUSTER if ((RR->cluster)&&(!isClusterFrontplane)) { RR->cluster->relayViaCluster(Address(),destination,fragment.data(),fragment.size(),false); @@ -127,7 +127,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Don't know peer or no direct path -- so relay via someone upstream relayTo = RR->topology->getUpstreamPeer(); if (relayTo) - relayTo->sendDirect(fragment.data(),fragment.size(),now,true); + relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,true); } } else { TRACE("dropped relay [fragment](%s) -> %s, max hops exceeded",fromAddr.toString().c_str(),destination.toString().c_str()); @@ -171,7 +171,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from for(unsigned int f=1;ffrag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); - if (rq->frag0.tryDecode(RR)) { + if (rq->frag0.tryDecode(RR,tPtr)) { rq->timestamp = 0; // packet decoded, free entry } else { rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something @@ -212,8 +212,8 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from packet.incrementHops(); #endif - SharedPtr relayTo = RR->topology->getPeer(destination); - if ((relayTo)&&(relayTo->sendDirect(packet.data(),packet.size(),now,false))) { + SharedPtr relayTo = RR->topology->getPeer(tPtr,destination); + if ((relayTo)&&(relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,false))) { if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) { // don't send RENDEZVOUS for cluster frontplane relays const InetAddress *hintToSource = (InetAddress *)0; const InetAddress *hintToDest = (InetAddress *)0; @@ -222,7 +222,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from InetAddress sourceV4,sourceV6; relayTo->getRendezvousAddresses(now,destV4,destV6); - const SharedPtr sourcePeer(RR->topology->getPeer(source)); + const SharedPtr sourcePeer(RR->topology->getPeer(tPtr,source)); if (sourcePeer) { sourcePeer->getRendezvousAddresses(now,sourceV4,sourceV6); if ((destV6)&&(sourceV6)) { @@ -249,7 +249,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from outp.append((uint8_t)4); outp.append(hintToSource->rawIpData(),4); } - send(outp,true); + send(tPtr,outp,true); } else { Packet outp(destination,RR->identity.address(),Packet::VERB_RENDEZVOUS); outp.append((uint8_t)0); @@ -262,7 +262,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from outp.append((uint8_t)4); outp.append(hintToDest->rawIpData(),4); } - send(outp,true); + send(tPtr,outp,true); } ++alt; } @@ -278,7 +278,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from #endif relayTo = RR->topology->getUpstreamPeer(&source,1,true); if (relayTo) - relayTo->sendDirect(packet.data(),packet.size(),now,true); + relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true); } } else { TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet.source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str()); @@ -321,7 +321,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from for(unsigned int f=1;ftotalFragments;++f) rq->frag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); - if (rq->frag0.tryDecode(RR)) { + if (rq->frag0.tryDecode(RR,tPtr)) { rq->timestamp = 0; // packet decoded, free entry } else { rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something @@ -334,7 +334,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } else { // Packet is unfragmented, so just process it IncomingPacket packet(data,len,path,now); - if (!packet.tryDecode(RR)) { + if (!packet.tryDecode(RR,tPtr)) { Mutex::Lock _l(_rxQueue_m); RXQueueEntry *rq = &(_rxQueue[ZT_RX_QUEUE_SIZE - 1]); unsigned long i = ZT_RX_QUEUE_SIZE - 1; @@ -362,7 +362,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } } -void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) +void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { if (!network->hasConfig()) return; @@ -474,7 +474,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c adv[42] = (checksum >> 8) & 0xff; adv[43] = checksum & 0xff; - RR->node->putFrame(network->id(),network->userPtr(),peerMac,from,ZT_ETHERTYPE_IPV6,0,adv,72); + RR->node->putFrame(tPtr,network->id(),network->userPtr(),peerMac,from,ZT_ETHERTYPE_IPV6,0,adv,72); return; // NDP emulation done. We have forged a "fake" reply, so no need to send actual NDP query. } // else no NDP emulation } // else no NDP emulation @@ -491,17 +491,18 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c * multicast addresses on bridge interfaces and subscribing each slave. * But in that case this does no harm, as the sets are just merged. */ if (fromBridged) - network->learnBridgedMulticastGroup(multicastGroup,RR->node->now()); + network->learnBridgedMulticastGroup(tPtr,multicastGroup,RR->node->now()); //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),multicastGroup.toString().c_str(),etherTypeName(etherType),len); // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. - if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } RR->mc->send( + tPtr, network->config().multicastLimit, RR->node->now(), network->id(), @@ -514,14 +515,14 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c len); } else if (to == network->mac()) { // Destination is this node, so just reinject it - RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,vlanId,data,len); + RR->node->putFrame(tPtr,network->id(),network->userPtr(),from,to,etherType,vlanId,data,len); } else if (to[0] == MAC::firstOctetForNetwork(network->id())) { // Destination is another ZeroTier peer on the same network Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this - SharedPtr toPeer(RR->topology->getPeer(toZT)); + SharedPtr toPeer(RR->topology->getPeer(tPtr,toZT)); - if (!network->filterOutgoingPacket(false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } @@ -536,7 +537,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(outp,true); + send(tPtr,outp,true); } else { Packet outp(toZT,RR->identity.address(),Packet::VERB_FRAME); outp.append(network->id()); @@ -544,7 +545,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(outp,true); + send(tPtr,outp,true); } //TRACE("%.16llx: UNICAST: %s -> %s etherType==%s(%.4x) vlanId==%u len==%u fromBridged==%d includeCom==%d",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),etherType,vlanId,len,(int)fromBridged,(int)includeCom); @@ -554,7 +555,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c // We filter with a NULL destination ZeroTier address first. Filtrations // for each ZT destination are also done below. This is the same rationale // and design as for multicast. - if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } @@ -592,7 +593,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } for(unsigned int b=0;bfilterOutgoingPacket(true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (network->filterOutgoingPacket(tPtr,true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId)) { Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(network->id()); outp.append((uint8_t)0x00); @@ -602,7 +603,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(outp,true); + send(tPtr,outp,true); } else { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); } @@ -610,20 +611,20 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } } -void Switch::send(Packet &packet,bool encrypt) +void Switch::send(void *tPtr,Packet &packet,bool encrypt) { if (packet.destination() == RR->identity.address()) { TRACE("BUG: caught attempt to send() to self, ignored"); return; } - if (!_trySend(packet,encrypt)) { + if (!_trySend(tPtr,packet,encrypt)) { Mutex::Lock _l(_txQueue_m); _txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt)); } } -void Switch::requestWhois(const Address &addr) +void Switch::requestWhois(void *tPtr,const Address &addr) { #ifdef ZT_TRACE if (addr == RR->identity.address()) { @@ -644,10 +645,10 @@ void Switch::requestWhois(const Address &addr) } } if (inserted) - _sendWhoisRequest(addr,(const Address *)0,0); + _sendWhoisRequest(tPtr,addr,(const Address *)0,0); } -void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) +void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) { { // cancel pending WHOIS since we now know this peer Mutex::Lock _l(_outstandingWhoisRequests_m); @@ -660,7 +661,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) while (i) { RXQueueEntry *rq = &(_rxQueue[--i]); if ((rq->timestamp)&&(rq->complete)) { - if (rq->frag0.tryDecode(RR)) + if (rq->frag0.tryDecode(RR,tPtr)) rq->timestamp = 0; } } @@ -670,7 +671,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { if (txi->dest == peer->address()) { - if (_trySend(txi->packet,txi->encrypt)) + if (_trySend(tPtr,txi->packet,txi->encrypt)) _txQueue.erase(txi++); else ++txi; } else ++txi; @@ -678,7 +679,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) } } -unsigned long Switch::doTimerTasks(uint64_t now) +unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) { unsigned long nextDelay = 0xffffffff; // ceiling delay, caller will cap to minimum @@ -695,7 +696,7 @@ unsigned long Switch::doTimerTasks(uint64_t now) _outstandingWhoisRequests.erase(*a); } else { r->lastSent = now; - r->peersConsulted[r->retries] = _sendWhoisRequest(*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0); + r->peersConsulted[r->retries] = _sendWhoisRequest(tPtr,*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0); TRACE("WHOIS %s (retry %u)",a->toString().c_str(),r->retries); ++r->retries; nextDelay = std::min(nextDelay,(unsigned long)ZT_WHOIS_RETRY_DELAY); @@ -709,7 +710,7 @@ unsigned long Switch::doTimerTasks(uint64_t now) { // Time out TX queue packets that never got WHOIS lookups or other info. Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { - if (_trySend(txi->packet,txi->encrypt)) + if (_trySend(tPtr,txi->packet,txi->encrypt)) _txQueue.erase(txi++); else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { TRACE("TX %s -> %s timed out",txi->packet.source().toString().c_str(),txi->packet.destination().toString().c_str()); @@ -743,19 +744,19 @@ bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address return false; } -Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) +Address Switch::_sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) { SharedPtr upstream(RR->topology->getUpstreamPeer(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); if (upstream) { Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS); addr.appendTo(outp); RR->node->expectReplyTo(outp.packetId()); - send(outp,true); + send(tPtr,outp,true); } return Address(); } -bool Switch::_trySend(Packet &packet,bool encrypt) +bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) { SharedPtr viaPath; const uint64_t now = RR->node->now(); @@ -769,7 +770,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) clusterMostRecentMemberId = RR->cluster->checkSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); #endif - const SharedPtr peer(RR->topology->getPeer(destination)); + const SharedPtr peer(RR->topology->getPeer(tPtr,destination)); if (peer) { /* First get the best path, and if it's dead (and this is not a root) * we attempt to re-activate that path but this packet will flow @@ -784,7 +785,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) if ((clusterMostRecentMemberId < 0)||(viaPath->lastIn() > clusterMostRecentTs)) { #endif if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); + peer->attemptToContactAt(tPtr,viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); viaPath->sent(now); } #ifdef ZT_ENABLE_CLUSTER @@ -801,7 +802,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #else if (!viaPath) { #endif - peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known + peer->tryMemorizedPath(tPtr,now); // periodically attempt memorized or statically defined paths, if any are known const SharedPtr relay(RR->topology->getUpstreamPeer()); if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { if (!(viaPath = peer->getBestPath(now,true))) @@ -816,7 +817,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #ifdef ZT_ENABLE_CLUSTER if (clusterMostRecentMemberId < 0) { #else - requestWhois(destination); + requestWhois(tPtr,destination); return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly #endif #ifdef ZT_ENABLE_CLUSTER @@ -844,9 +845,9 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #endif #ifdef ZT_ENABLE_CLUSTER - if ( ((viaPath)&&(viaPath->send(RR,packet.data(),chunkSize,now))) || ((clusterMostRecentMemberId >= 0)&&(RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,packet.data(),chunkSize))) ) { + if ( ((viaPath)&&(viaPath->send(RR,tPtr,packet.data(),chunkSize,now))) || ((clusterMostRecentMemberId >= 0)&&(RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,packet.data(),chunkSize))) ) { #else - if (viaPath->send(RR,packet.data(),chunkSize,now)) { + if (viaPath->send(RR,tPtr,packet.data(),chunkSize,now)) { #endif if (chunkSize < packet.size()) { // Too big for one packet, fragment the rest @@ -866,7 +867,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) else if (clusterMostRecentMemberId >= 0) RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,frag.data(),frag.size()); #else - viaPath->send(RR,frag.data(),frag.size(),now); + viaPath->send(RR,tPtr,frag.data(),frag.size(),now); #endif fragStart += chunkSize; remaining -= chunkSize; diff --git a/node/Switch.hpp b/node/Switch.hpp index 9245c036..ff350934 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -59,16 +59,18 @@ public: /** * Called when a packet is received from the real network * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param localAddr Local interface address * @param fromAddr Internet IP address of origin * @param data Packet data * @param len Packet length */ - void onRemotePacket(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len); + void onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len); /** * Called when a packet comes from a local Ethernet tap * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param network Which network's TAP did this packet come from? * @param from Originating MAC address * @param to Destination MAC address @@ -77,7 +79,7 @@ public: * @param data Ethernet payload * @param len Frame length */ - void onLocalEthernet(const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); + void onLocalEthernet(void *tPtr,const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); /** * Send a packet to a ZeroTier address (destination in packet) @@ -91,26 +93,29 @@ public: * Needless to say, the packet's source must be this node. Otherwise it * won't be encrypted right. (This is not used for relaying.) * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param packet Packet to send (buffer may be modified) * @param encrypt Encrypt packet payload? (always true except for HELLO) */ - void send(Packet &packet,bool encrypt); + void send(void *tPtr,Packet &packet,bool encrypt); /** * Request WHOIS on a given address * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param addr Address to look up */ - void requestWhois(const Address &addr); + void requestWhois(void *tPtr,const Address &addr); /** * Run any processes that are waiting for this peer's identity * * Called when we learn of a peer's identity from HELLO, OK(WHOIS), etc. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param peer New peer */ - void doAnythingWaitingForPeer(const SharedPtr &peer); + void doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer); /** * Perform retries and other periodic timer tasks @@ -118,15 +123,16 @@ public: * This can return a very long delay if there are no pending timer * tasks. The caller should cap this comparatively vs. other values. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @return Number of milliseconds until doTimerTasks() should be run again */ - unsigned long doTimerTasks(uint64_t now); + unsigned long doTimerTasks(void *tPtr,uint64_t now); private: bool _shouldUnite(const uint64_t now,const Address &source,const Address &destination); - Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); - bool _trySend(Packet &packet,bool encrypt); // packet is modified if return is true + Address _sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); + bool _trySend(void *tPtr,Packet &packet,bool encrypt); // packet is modified if return is true const RuntimeEnvironment *const RR; uint64_t _lastBeaconResponse; diff --git a/node/Tag.cpp b/node/Tag.cpp index eb4026bc..3f924da1 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -25,13 +25,13 @@ namespace ZeroTier { -int Tag::verify(const RuntimeEnvironment *RR) const +int Tag::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } try { diff --git a/node/Tag.hpp b/node/Tag.hpp index 146e8da9..38085906 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -105,9 +105,10 @@ public: * Check this tag's signature * * @param RR Runtime environment to allow identity lookup for signedBy + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or tag */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template inline void serialize(Buffer &b,const bool forSign = false) const diff --git a/node/Topology.cpp b/node/Topology.cpp index 21547cd2..a1d37332 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -55,19 +55,19 @@ namespace ZeroTier { #define ZT_DEFAULT_WORLD_LENGTH 634 static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x52,0x3c,0x32,0x50,0x1a,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0x4a,0xf7,0x86,0xa8,0x40,0xd6,0x52,0xea,0xae,0x9e,0x7a,0xbf,0x4c,0x97,0x66,0xab,0x2d,0x6f,0xaf,0xc9,0x2b,0x3a,0xff,0xed,0xd6,0x30,0x3e,0xc4,0x6a,0x65,0xf2,0xbd,0x83,0x52,0xf5,0x40,0xe9,0xcc,0x0d,0x6e,0x89,0x3f,0x9a,0xa0,0xb8,0xdf,0x42,0xd2,0x2f,0x84,0xe6,0x03,0x26,0x0f,0xa8,0xe3,0xcc,0x05,0x05,0x03,0xef,0x12,0x80,0x0d,0xce,0x3e,0xb6,0x58,0x3b,0x1f,0xa8,0xad,0xc7,0x25,0xf9,0x43,0x71,0xa7,0x5c,0x9a,0xc7,0xe1,0xa3,0xb8,0x88,0xd0,0x71,0x6c,0x94,0x99,0x73,0x41,0x0b,0x1b,0x48,0x84,0x02,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x0c,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9a,0x42,0xc5,0x21,0x27,0x09,0x06,0x2c,0x0f,0xf8,0x50,0x01,0x54,0x01,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x33,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x6b,0xaa,0xc5,0x0e,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x00,0x00,0x00,0x00,0x02,0x00,0xe0,0x01,0x27,0x09,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0xb7,0x40,0x01,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x0c,0x04,0x2d,0x20,0xc6,0x82,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x64,0x00,0x81,0xc3,0x54,0x00,0x00,0xff,0xfe,0x18,0x1d,0x61,0x27,0x09,0x04,0x2e,0x65,0xa0,0xf9,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x03,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x6a,0x30,0x01,0x27,0x09,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x68,0x00,0x83,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x64,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09}; -Topology::Topology(const RuntimeEnvironment *renv) : +Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : RR(renv), _trustedPathCount(0), _amRoot(false) { try { World cachedPlanet; - std::string buf(RR->node->dataStoreGet("planet")); + std::string buf(RR->node->dataStoreGet(tPtr,"planet")); if (buf.length() > 0) { Buffer dswtmp(buf.data(),(unsigned int)buf.length()); cachedPlanet.deserialize(dswtmp,0); } - addWorld(cachedPlanet,false); + addWorld(tPtr,cachedPlanet,false); } catch ( ... ) {} World defaultPlanet; @@ -75,10 +75,10 @@ Topology::Topology(const RuntimeEnvironment *renv) : Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top } - addWorld(defaultPlanet,false); + addWorld(tPtr,defaultPlanet,false); } -SharedPtr Topology::addPeer(const SharedPtr &peer) +SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) { #ifdef ZT_TRACE if ((!peer)||(peer->address() == RR->identity.address())) { @@ -98,12 +98,12 @@ SharedPtr Topology::addPeer(const SharedPtr &peer) np = hp; } - saveIdentity(np->identity()); + saveIdentity(tPtr,np->identity()); return np; } -SharedPtr Topology::getPeer(const Address &zta) +SharedPtr Topology::getPeer(void *tPtr,const Address &zta) { if (zta == RR->identity.address()) { TRACE("BUG: ignored attempt to getPeer() for self, returned NULL"); @@ -118,7 +118,7 @@ SharedPtr Topology::getPeer(const Address &zta) } try { - Identity id(_getIdentity(zta)); + Identity id(_getIdentity(tPtr,zta)); if (id) { SharedPtr np(new Peer(RR,RR->identity,id)); { @@ -134,7 +134,7 @@ SharedPtr Topology::getPeer(const Address &zta) return SharedPtr(); } -Identity Topology::getIdentity(const Address &zta) +Identity Topology::getIdentity(void *tPtr,const Address &zta) { if (zta == RR->identity.address()) { return RR->identity; @@ -144,15 +144,15 @@ Identity Topology::getIdentity(const Address &zta) if (ap) return (*ap)->identity(); } - return _getIdentity(zta); + return _getIdentity(tPtr,zta); } -void Topology::saveIdentity(const Identity &id) +void Topology::saveIdentity(void *tPtr,const Identity &id) { if (id) { char p[128]; Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)id.address().toInt()); - RR->node->dataStorePut(p,id.toString(false),false); + RR->node->dataStorePut(tPtr,p,id.toString(false),false); } } @@ -264,7 +264,7 @@ bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipa return false; } -bool Topology::addWorld(const World &newWorld,bool alwaysAcceptNew) +bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew) { if ((newWorld.type() != World::TYPE_PLANET)&&(newWorld.type() != World::TYPE_MOON)) return false; @@ -328,29 +328,29 @@ bool Topology::addWorld(const World &newWorld,bool alwaysAcceptNew) try { Buffer dswtmp; existing->serialize(dswtmp,false); - RR->node->dataStorePut(savePath,dswtmp.data(),dswtmp.size(),false); + RR->node->dataStorePut(tPtr,savePath,dswtmp.data(),dswtmp.size(),false); } catch ( ... ) { - RR->node->dataStoreDelete(savePath); + RR->node->dataStoreDelete(tPtr,savePath); } - _memoizeUpstreams(); + _memoizeUpstreams(tPtr); return true; } -void Topology::addMoon(const uint64_t id,const Address &seed) +void Topology::addMoon(void *tPtr,const uint64_t id,const Address &seed) { char savePath[64]; Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); try { - std::string moonBin(RR->node->dataStoreGet(savePath)); + std::string moonBin(RR->node->dataStoreGet(tPtr,savePath)); if (moonBin.length() > 1) { Buffer wtmp(moonBin.data(),(unsigned int)moonBin.length()); World w; w.deserialize(wtmp); if ((w.type() == World::TYPE_MOON)&&(w.id() == id)) { - addWorld(w,true); + addWorld(tPtr,w,true); return; } } @@ -363,7 +363,7 @@ void Topology::addMoon(const uint64_t id,const Address &seed) } } -void Topology::removeMoon(const uint64_t id) +void Topology::removeMoon(void *tPtr,const uint64_t id) { Mutex::Lock _l1(_upstreams_m); Mutex::Lock _l2(_peers_m); @@ -375,7 +375,7 @@ void Topology::removeMoon(const uint64_t id) } else { char savePath[64]; Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); - RR->node->dataStoreDelete(savePath); + RR->node->dataStoreDelete(tPtr,savePath); } } _moons.swap(nm); @@ -387,7 +387,7 @@ void Topology::removeMoon(const uint64_t id) } _moonSeeds.swap(cm); - _memoizeUpstreams(); + _memoizeUpstreams(tPtr); } void Topology::clean(uint64_t now) @@ -415,11 +415,11 @@ void Topology::clean(uint64_t now) } } -Identity Topology::_getIdentity(const Address &zta) +Identity Topology::_getIdentity(void *tPtr,const Address &zta) { char p[128]; Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)zta.toInt()); - std::string ids(RR->node->dataStoreGet(p)); + std::string ids(RR->node->dataStoreGet(tPtr,p)); if (ids.length() > 0) { try { return Identity(ids); @@ -428,7 +428,7 @@ Identity Topology::_getIdentity(const Address &zta) return Identity(); } -void Topology::_memoizeUpstreams() +void Topology::_memoizeUpstreams(void *tPtr) { // assumes _upstreams_m and _peers_m are locked _upstreamAddresses.clear(); @@ -442,7 +442,7 @@ void Topology::_memoizeUpstreams() SharedPtr &hp = _peers[i->identity.address()]; if (!hp) { hp = new Peer(RR,RR->identity,i->identity); - saveIdentity(i->identity); + saveIdentity(tPtr,i->identity); } } } @@ -456,7 +456,7 @@ void Topology::_memoizeUpstreams() SharedPtr &hp = _peers[i->identity.address()]; if (!hp) { hp = new Peer(RR,RR->identity,i->identity); - saveIdentity(i->identity); + saveIdentity(tPtr,i->identity); } } } diff --git a/node/Topology.hpp b/node/Topology.hpp index e21747c8..4870ab5e 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -50,7 +50,7 @@ class RuntimeEnvironment; class Topology { public: - Topology(const RuntimeEnvironment *renv); + Topology(const RuntimeEnvironment *renv,void *tPtr); /** * Add a peer to database @@ -58,18 +58,20 @@ public: * This will not replace existing peers. In that case the existing peer * record is returned. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param peer Peer to add * @return New or existing peer (should replace 'peer') */ - SharedPtr addPeer(const SharedPtr &peer); + SharedPtr addPeer(void *tPtr,const SharedPtr &peer); /** * Get a peer from its address * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param zta ZeroTier address of peer * @return Peer or NULL if not found */ - SharedPtr getPeer(const Address &zta); + SharedPtr getPeer(void *tPtr,const Address &zta); /** * Get a peer only if it is presently in memory (no disk cache) @@ -109,10 +111,11 @@ public: /** * Get the identity of a peer * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param zta ZeroTier address of peer * @return Identity or NULL Identity if not found */ - Identity getIdentity(const Address &zta); + Identity getIdentity(void *tPtr,const Address &zta); /** * Cache an identity @@ -120,9 +123,10 @@ public: * This is done automatically on addPeer(), and so is only useful for * cluster identity replication. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param id Identity to cache */ - void saveIdentity(const Identity &id); + void saveIdentity(void *tPtr,const Identity &id); /** * Get the current best upstream peer @@ -267,11 +271,12 @@ public: /** * Validate new world and update if newer and signature is okay * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param newWorld A new or updated planet or moon to learn * @param alwaysAcceptNew If true, always accept new moons even if we're not waiting for one * @return True if it was valid and newer than current (or totally new for moons) */ - bool addWorld(const World &newWorld,bool alwaysAcceptNew); + bool addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew); /** * Add a moon @@ -282,14 +287,15 @@ public: * @param id Moon ID * @param seed If non-NULL, an address of any member of the moon to contact */ - void addMoon(const uint64_t id,const Address &seed); + void addMoon(void *tPtr,const uint64_t id,const Address &seed); /** * Remove a moon * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param id Moon's world ID */ - void removeMoon(const uint64_t id); + void removeMoon(void *tPtr,const uint64_t id); /** * Clean and flush database @@ -420,8 +426,8 @@ public: } private: - Identity _getIdentity(const Address &zta); - void _memoizeUpstreams(); + Identity _getIdentity(void *tPtr,const Address &zta); + void _memoizeUpstreams(void *tPtr); const RuntimeEnvironment *const RR; diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index 0e1ada6b..62fabc48 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -71,7 +71,7 @@ BSDEthernetTap::BSDEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg) : _handler(handler), _arg(arg), @@ -460,8 +460,7 @@ void BSDEthernetTap::threadMain() to.setTo(getBuf,6); from.setTo(getBuf + 6,6); unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); - // TODO: VLAN support - _handler(_arg,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); } r = 0; diff --git a/osdep/BSDEthernetTap.hpp b/osdep/BSDEthernetTap.hpp index 1bb48d31..8c6314db 100644 --- a/osdep/BSDEthernetTap.hpp +++ b/osdep/BSDEthernetTap.hpp @@ -43,7 +43,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~BSDEthernetTap(); @@ -62,7 +62,7 @@ public: throw(); private: - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; uint64_t _nwid; Thread _thread; diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index e7fe657f..c4b978e7 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -62,7 +62,7 @@ LinuxEthernetTap::LinuxEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg) : _handler(handler), _arg(arg), @@ -470,7 +470,7 @@ void LinuxEthernetTap::threadMain() from.setTo(getBuf + 6,6); unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); // TODO: VLAN support - _handler(_arg,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); } r = 0; diff --git a/osdep/LinuxEthernetTap.hpp b/osdep/LinuxEthernetTap.hpp index 7dd7e01d..a2a00a79 100644 --- a/osdep/LinuxEthernetTap.hpp +++ b/osdep/LinuxEthernetTap.hpp @@ -44,7 +44,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~LinuxEthernetTap(); @@ -66,7 +66,7 @@ public: throw(); private: - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; uint64_t _nwid; Thread _thread; diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp index b3580929..35eac05a 100644 --- a/osdep/OSXEthernetTap.cpp +++ b/osdep/OSXEthernetTap.cpp @@ -314,7 +314,7 @@ OSXEthernetTap::OSXEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), void *arg) : _handler(handler), _arg(arg), @@ -646,7 +646,7 @@ void OSXEthernetTap::threadMain() from.setTo(getBuf + 6,6); unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); // TODO: VLAN support - _handler(_arg,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); } r = 0; diff --git a/osdep/OSXEthernetTap.hpp b/osdep/OSXEthernetTap.hpp index de48f9a4..5a96c210 100644 --- a/osdep/OSXEthernetTap.hpp +++ b/osdep/OSXEthernetTap.hpp @@ -48,7 +48,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~OSXEthernetTap(); @@ -67,7 +67,7 @@ public: throw(); private: - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; uint64_t _nwid; Thread _thread; diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index 8ee088bb..79b9d35e 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -456,7 +456,7 @@ WindowsEthernetTap::WindowsEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg) : _handler(handler), _arg(arg), @@ -1058,8 +1058,7 @@ void WindowsEthernetTap::threadMain() MAC from(tapReadBuf + 6,6); unsigned int etherType = ((((unsigned int)tapReadBuf[12]) & 0xff) << 8) | (((unsigned int)tapReadBuf[13]) & 0xff); try { - // TODO: decode vlans - _handler(_arg,_nwid,from,to,etherType,0,tapReadBuf + 14,bytesRead - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,tapReadBuf + 14,bytesRead - 14); } catch ( ... ) {} // handlers should not throw } } diff --git a/osdep/WindowsEthernetTap.hpp b/osdep/WindowsEthernetTap.hpp index 53bba3e9..f2cf73f3 100644 --- a/osdep/WindowsEthernetTap.hpp +++ b/osdep/WindowsEthernetTap.hpp @@ -87,7 +87,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~WindowsEthernetTap(); @@ -118,7 +118,7 @@ private: void _setRegistryIPv4Value(const char *regKey,const std::vector &value); void _syncIps(); - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; MAC _mac; uint64_t _nwid; diff --git a/service/OneService.cpp b/service/OneService.cpp index 22eefbb9..c07b3ba4 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -291,21 +291,21 @@ static void _moonToJson(nlohmann::json &mj,const World &world) class OneServiceImpl; -static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf); -static void SnodeEventCallback(ZT_Node *node,void *uptr,enum ZT_Event event,const void *metaData); -static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize); -static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure); -static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); -static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); -static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result); +static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf); +static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData); +static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize); +static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,const void *data,unsigned long len,int secure); +static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); +static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); +static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result); #ifdef ZT_ENABLE_CLUSTER static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len); static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z); #endif -static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); +static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); static int ShttpOnMessageBegin(http_parser *parser); static int ShttpOnUrl(http_parser *parser,const char *ptr,size_t length); @@ -573,7 +573,7 @@ public: cb.eventCallback = SnodeEventCallback; cb.pathCheckFunction = SnodePathCheckFunction; cb.pathLookupFunction = SnodePathLookupFunction; - _node = new Node(this,&cb,OSUtils::now()); + _node = new Node(this,(void *)0,&cb,OSUtils::now()); } // Read local configuration @@ -804,7 +804,7 @@ public: for(std::vector::iterator f(networksDotD.begin());f!=networksDotD.end();++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16)&&(f->substr(16) == ".conf")) - _node->join(Utils::hexStrToU64(f->substr(0,dot).c_str()),(void *)0); + _node->join(Utils::hexStrToU64(f->substr(0,dot).c_str()),(void *)0,(void *)0); } } { // Load existing moons @@ -812,7 +812,7 @@ public: for(std::vector::iterator f(moonsDotD.begin());f!=moonsDotD.end();++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16)&&(f->substr(16) == ".moon")) - _node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str()),0); + _node->orbit((void *)0,Utils::hexStrToU64(f->substr(0,dot).c_str()),0); } } @@ -877,7 +877,7 @@ public: uint64_t dl = _nextBackgroundTaskDeadline; if (dl <= now) { - _node->processBackgroundTasks(now,&_nextBackgroundTaskDeadline); + _node->processBackgroundTasks((void *)0,now,&_nextBackgroundTaskDeadline); dl = _nextBackgroundTaskDeadline; } @@ -892,7 +892,7 @@ public: std::vector added,removed; n->second.tap->scanMulticastGroups(added,removed); for(std::vector::iterator m(added.begin());m!=added.end();++m) - _node->multicastSubscribe(n->first,m->mac().toInt(),m->adi()); + _node->multicastSubscribe((void *)0,n->first,m->mac().toInt(),m->adi()); for(std::vector::iterator m(removed.begin());m!=removed.end();++m) _node->multicastUnsubscribe(n->first,m->mac().toInt(),m->adi()); } @@ -1306,7 +1306,7 @@ public: res["signature"] = json(); res["updatesMustBeSignedBy"] = json(); res["waiting"] = true; - _node->orbit(id,seed); + _node->orbit((void *)0,id,seed); scode = 200; } @@ -1315,7 +1315,7 @@ public: if (ps.size() == 2) { uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str()); - _node->join(wantnw,(void *)0); // does nothing if we are a member + _node->join(wantnw,(void *)0,(void *)0); // does nothing if we are a member ZT_VirtualNetworkList *nws = _node->networks(); if (nws) { for(unsigned long i=0;inetworkCount;++i) { @@ -1360,7 +1360,7 @@ public: if (ps[0] == "moon") { if (ps.size() == 2) { - _node->deorbit(Utils::hexStrToU64(ps[1].c_str())); + _node->deorbit((void *)0,Utils::hexStrToU64(ps[1].c_str())); res["result"] = true; scode = 200; } // else 404 @@ -1371,7 +1371,7 @@ public: uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str()); for(unsigned long i=0;inetworkCount;++i) { if (nws->networks[i].nwid == wantnw) { - _node->leave(wantnw,(void **)0); + _node->leave(wantnw,(void **)0,(void *)0); res["result"] = true; scode = 200; break; @@ -1693,6 +1693,7 @@ public: _lastDirectReceiveFromGlobal = OSUtils::now(); const ZT_ResultCode rc = _node->processWirePacket( + (void *)0, OSUtils::now(), reinterpret_cast(localAddr), (const struct sockaddr_storage *)from, // Phy<> uses sockaddr_storage, so it'll always be that big @@ -1845,6 +1846,7 @@ public: if (from) { InetAddress fakeTcpLocalInterfaceAddress((uint32_t)0xffffffff,0xffff); const ZT_ResultCode rc = _node->processWirePacket( + (void *)0, OSUtils::now(), reinterpret_cast(&fakeTcpLocalInterfaceAddress), reinterpret_cast(&from), @@ -2255,7 +2257,7 @@ public: inline void tapFrameHandler(uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { - _node->processVirtualNetworkFrame(OSUtils::now(),nwid,from.toInt(),to.toInt(),etherType,vlanId,data,len,&_nextBackgroundTaskDeadline); + _node->processVirtualNetworkFrame((void *)0,OSUtils::now(),nwid,from.toInt(),to.toInt(),etherType,vlanId,data,len,&_nextBackgroundTaskDeadline); } inline void onHttpRequestToServer(TcpConnection *tc) @@ -2426,21 +2428,21 @@ public: } }; -static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf) +static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf) { return reinterpret_cast(uptr)->nodeVirtualNetworkConfigFunction(nwid,nuptr,op,nwconf); } -static void SnodeEventCallback(ZT_Node *node,void *uptr,enum ZT_Event event,const void *metaData) +static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData) { reinterpret_cast(uptr)->nodeEventCallback(event,metaData); } -static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize) +static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize) { return reinterpret_cast(uptr)->nodeDataStoreGetFunction(name,buf,bufSize,readIndex,totalSize); } -static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure) +static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,const void *data,unsigned long len,int secure) { return reinterpret_cast(uptr)->nodeDataStorePutFunction(name,data,len,secure); } -static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) +static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) { return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); } -static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) +static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { reinterpret_cast(uptr)->nodeVirtualNetworkFrameFunction(nwid,nuptr,sourceMac,destMac,etherType,vlanId,data,len); } -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) { return reinterpret_cast(uptr)->nodePathCheckFunction(ztaddr,localAddr,remoteAddr); } -static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result) +static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result) { return reinterpret_cast(uptr)->nodePathLookupFunction(ztaddr,family,result); } #ifdef ZT_ENABLE_CLUSTER @@ -2458,7 +2460,7 @@ static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr, } #endif -static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) +static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { reinterpret_cast(uptr)->tapFrameHandler(nwid,from,to,etherType,vlanId,data,len); } static int ShttpOnMessageBegin(http_parser *parser) diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 7ecd42b1..7ec377cc 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -175,7 +175,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void std::string lj; lj.push_back((char)VERB_LATEST); lj.append(OSUtils::jsonDump(*latest)); - _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,lj.data(),(unsigned int)lj.length()); + _node.sendUserMessage((void *)0,origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,lj.data(),(unsigned int)lj.length()); if (_distLog) { fprintf(_distLog,"%.10llx GET_LATEST %u.%u.%u_%u platform %u arch %u vendor %u channel %s -> LATEST %u.%u.%u_%u" ZT_EOL_S,(unsigned long long)origin,rvMaj,rvMin,rvRev,rvBld,rvPlatform,rvArch,rvVendor,rvChannel.c_str(),bestVMaj,bestVMin,bestVRev,bestVBld); fflush(_distLog); @@ -205,7 +205,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append((uint8_t)VERB_GET_DATA); gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } @@ -229,7 +229,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void buf.append(reinterpret_cast(data) + 1,16); buf.append((uint32_t)idx); buf.append(d->second.bin.data() + idx,std::min((unsigned long)ZT_SOFTWARE_UPDATE_CHUNK_SIZE,(unsigned long)(d->second.bin.length() - idx))); - _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,buf.data(),buf.size()); + _node.sendUserMessage((void *)0,origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,buf.data(),buf.size()); //printf(">> DATA @%u\n",(unsigned int)idx); } } @@ -249,7 +249,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append((uint8_t)VERB_GET_DATA); gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } @@ -296,7 +296,7 @@ bool SoftwareUpdater::check(const uint64_t now) ZT_BUILD_ARCHITECTURE, (int)ZT_VENDOR_ZEROTIER, _channel.c_str()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,tmp,len); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,tmp,len); //printf(">> GET_LATEST\n"); } @@ -343,7 +343,7 @@ bool SoftwareUpdater::check(const uint64_t now) gd.append((uint8_t)VERB_GET_DATA); gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } -- cgit v1.2.3 From 57b5a33fbbbaceb7ac562145712a435fcd2b1a0a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 29 Mar 2017 12:32:59 -0700 Subject: Fix bug preventing default from being set to null if already set to an integer. --- controller/EmbeddedNetworkController.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index ce56e906..3be55d8e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1018,9 +1018,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json ntag = json::object(); const uint64_t tagId = OSUtils::jsonInt(tag["id"],0ULL); ntag["id"] = tagId; - if (tag.find("default") == tag.end()) - ntag["default"] = json(); - else ntag["default"] = OSUtils::jsonInt(tag["default"],0ULL); + json &dfl = tag["default"]; + if (dfl.is_null()) + ntag["default"] = dfl; + else ntag["default"] = OSUtils::jsonInt(dfl,0ULL); ntags[tagId] = ntag; } } -- cgit v1.2.3 From e5284771e472544ff6363ce773ee7f055fe575f1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 30 Mar 2017 09:54:04 -0700 Subject: Add ping/pong to API so controller supervisor in Central can do a full-path check of controller uptime, etc. --- controller/EmbeddedNetworkController.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3be55d8e..5bce7886 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1069,13 +1069,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } // else 404 - } else if (path[0] == "dbtest") { + } else if (path[0] == "ping") { json testRec; const uint64_t now = OSUtils::now(); testRec["clock"] = now; testRec["uptime"] = (now - _startTime); - _db.put("dbtest",testRec); + testRec["content"] = b; + _db.put("pong",testRec); + responseBody = OSUtils::jsonDump(testRec); + responseContentType = "application/json"; + return 200; } -- cgit v1.2.3 From eddbc7e757f26e59d6eeab7e31e31eb6c47dcf20 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 4 Apr 2017 08:07:38 -0700 Subject: Logic simplification, cleanup, and memory use improvements in Membership. Also fix an issue that may cause network instability in some cases. --- controller/EmbeddedNetworkController.cpp | 2 +- node/Capability.hpp | 5 +- node/CertificateOfMembership.hpp | 25 ++++--- node/CertificateOfOwnership.hpp | 5 +- node/CertificateOfRepresentation.hpp | 6 +- node/Membership.cpp | 48 ++++++------- node/Membership.hpp | 118 ++++++++++++++++++++----------- node/Network.cpp | 30 ++++---- node/Revocation.hpp | 41 +++++------ node/Tag.hpp | 19 ++--- 10 files changed, 175 insertions(+), 124 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5bce7886..a1a75c9e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -665,7 +665,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( // Member is being de-authorized, so spray Revocation objects to all online members if (!newAuth) { _clearNetworkMemberInfoCache(nwid); - Revocation rev(_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); + Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); Mutex::Lock _l(_lastRequestTime_m); for(std::map< std::pair,uint64_t >::iterator i(_lastRequestTime.begin());i!=_lastRequestTime.end();++i) { diff --git a/node/Capability.hpp b/node/Capability.hpp index 5ef6c994..454723ac 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -24,6 +24,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Utils.hpp" @@ -58,9 +59,11 @@ class RuntimeEnvironment; * handed off between nodes. Limited transferrability of capabilities is * a feature of true capability based security. */ -class Capability +class Capability : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_CAPABILITY; } + Capability() { memset(this,0,sizeof(Capability)); diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index ae976b50..dfccb138 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -27,6 +27,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "Buffer.hpp" #include "Address.hpp" #include "C25519.hpp" @@ -68,9 +69,11 @@ class RuntimeEnvironment; * This is a memcpy()'able structure and is safe (in a crash sense) to modify * without locks. */ -class CertificateOfMembership +class CertificateOfMembership : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COM; } + /** * Reserved qualifier IDs * @@ -155,18 +158,23 @@ public: /** * @return True if there's something here */ - inline operator bool() const throw() { return (_qualifierCount != 0); } + inline operator bool() const { return (_qualifierCount != 0); } + + /** + * @return Credential ID, always 0 for COMs + */ + inline uint32_t id() const { return 0; } /** * @return Timestamp for this cert and maximum delta for timestamp */ - inline std::pair timestamp() const + inline uint64_t timestamp() const { for(unsigned int i=0;i<_qualifierCount;++i) { if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP) - return std::pair(_qualifiers[i].value,_qualifiers[i].maxDelta); + return _qualifiers[i].value; } - return std::pair(0ULL,0ULL); + return 0; } /** @@ -258,12 +266,12 @@ public: /** * @return True if signed */ - inline bool isSigned() const throw() { return (_signedBy); } + inline bool isSigned() const { return (_signedBy); } /** * @return Address that signed this certificate or null address if none */ - inline const Address &signedBy() const throw() { return _signedBy; } + inline const Address &signedBy() const { return _signedBy; } template inline void serialize(Buffer &b) const @@ -321,7 +329,6 @@ public: } inline bool operator==(const CertificateOfMembership &c) const - throw() { if (_signedBy != c._signedBy) return false; @@ -335,7 +342,7 @@ public: } return (_signature == c._signature); } - inline bool operator!=(const CertificateOfMembership &c) const throw() { return (!(*this == c)); } + inline bool operator!=(const CertificateOfMembership &c) const { return (!(*this == c)); } private: struct _Qualifier diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 8c47582d..93be64dd 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -25,6 +25,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "C25519.hpp" #include "Address.hpp" #include "Identity.hpp" @@ -45,9 +46,11 @@ class RuntimeEnvironment; /** * Certificate indicating ownership of a network identifier */ -class CertificateOfOwnership +class CertificateOfOwnership : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COO; } + enum Thing { THING_NULL = 0, diff --git a/node/CertificateOfRepresentation.hpp b/node/CertificateOfRepresentation.hpp index 02e961c4..710ee577 100644 --- a/node/CertificateOfRepresentation.hpp +++ b/node/CertificateOfRepresentation.hpp @@ -20,6 +20,7 @@ #define ZT_CERTIFICATEOFREPRESENTATION_HPP #include "Constants.hpp" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Identity.hpp" @@ -47,14 +48,17 @@ namespace ZeroTier { * roots can shield nodes entirely and p2p connectivity behind them can * be disabled. This will be desirable for a number of use cases. */ -class CertificateOfRepresentation +class CertificateOfRepresentation : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COR; } + CertificateOfRepresentation() { memset(this,0,sizeof(CertificateOfRepresentation)); } + inline uint32_t id() const { return 0; } inline uint64_t timestamp() const { return _timestamp; } inline const Address &representative(const unsigned int i) const { return _reps[i]; } inline unsigned int repCount() const { return _repCount; } diff --git a/node/Membership.cpp b/node/Membership.cpp index ffe770c6..62c07314 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -39,6 +39,7 @@ Membership::Membership() : _remoteCaps(4), _remoteCoos(4) { + resetPushState(); } void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) @@ -48,18 +49,16 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u const Capability *sendCap; if (localCapabilityIndex >= 0) { sendCap = &(nconf.capabilities[localCapabilityIndex]); - if ( (_localCaps[localCapabilityIndex].id != sendCap->id()) || ((now - _localCaps[localCapabilityIndex].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCaps[localCapabilityIndex].lastPushed = now; - _localCaps[localCapabilityIndex].id = sendCap->id(); - } else sendCap = (const Capability *)0; + if ( ((now - _localCredLastPushed.cap[localCapabilityIndex]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) + _localCredLastPushed.cap[localCapabilityIndex] = now; + else sendCap = (const Capability *)0; } else sendCap = (const Capability *)0; const Tag *sendTags[ZT_MAX_NETWORK_TAGS]; unsigned int sendTagCount = 0; for(unsigned int t=0;t= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localTags[t].lastPushed = now; - _localTags[t].id = nconf.tags[t].id(); + if ( ((now - _localCredLastPushed.tag[t]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCredLastPushed.tag[t] = now; sendTags[sendTagCount++] = &(nconf.tags[t]); } } @@ -67,9 +66,8 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u const CertificateOfOwnership *sendCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; unsigned int sendCooCount = 0; for(unsigned int c=0;c= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCoos[c].lastPushed = now; - _localCoos[c].id = nconf.certificatesOfOwnership[c].id(); + if ( ((now - _localCredLastPushed.coo[c]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCredLastPushed.coo[c] = now; sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]); } } @@ -149,7 +147,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -// Template out addCredential() for most cred types to avoid copypasta +// Template out addCredential() for many cred types to avoid copypasta template static Membership::AddCredentialResult _addCredImpl(Hashtable &remoteCreds,const Hashtable &revocations,const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const C &cred) { @@ -165,7 +163,7 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot } } - const uint64_t *rt = revocations.get(Membership::revocationKey(C::credentialType(),cred.id())); + const uint64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id())); if ((rt)&&(*rt >= cred.timestamp())) { TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (timestamp below revocation threshold)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); return Membership::ADD_REJECTED; @@ -186,20 +184,9 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) -{ - return _addCredImpl(_remoteTags,_revocations,RR,tPtr,nconf,tag); -} - -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) -{ - return _addCredImpl(_remoteCaps,_revocations,RR,tPtr,nconf,cap); -} - -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) -{ - return _addCredImpl(_remoteCoos,_revocations,RR,tPtr,nconf,coo); -} +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) { return _addCredImpl(_remoteTags,_revocations,RR,tPtr,nconf,tag); } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) { return _addCredImpl(_remoteCaps,_revocations,RR,tPtr,nconf,cap); } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) { return _addCredImpl(_remoteCoos,_revocations,RR,tPtr,nconf,coo); } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev) { @@ -219,7 +206,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme case Credential::CREDENTIAL_TYPE_CAPABILITY: case Credential::CREDENTIAL_TYPE_TAG: case Credential::CREDENTIAL_TYPE_COO: - rt = &(_revocations[revocationKey(ct,rev.credentialId())]); + rt = &(_revocations[credentialKey(ct,rev.credentialId())]); if (*rt < rev.threshold()) { *rt = rev.threshold(); return ADD_ACCEPTED_NEW; @@ -234,4 +221,11 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } +void Membership::clean(const uint64_t now,const NetworkConfig &nconf) +{ + _cleanCredImpl(nconf,_remoteTags); + _cleanCredImpl(nconf,_remoteCaps); + _cleanCredImpl(nconf,_remoteCoos); +} + } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 4bd04ad6..22772859 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -47,23 +47,6 @@ class Network; */ class Membership { -private: - template - struct _RemoteCredential - { - _RemoteCredential() : lastReceived(0),revocationThreshold(0),credential() {} - uint64_t lastReceived; // last time we got this credential - uint64_t revocationThreshold; // credentials before this time are invalid - T credential; - }; - - struct _LocalCredentialPushState - { - _LocalCredentialPushState() : lastPushed(0),id(0) {} - uint64_t lastPushed; // last time we sent our own copy of this credential - uint32_t id; - }; - public: enum AddCredentialResult { @@ -92,19 +75,19 @@ public: void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** - * Check whether we should push MULTICAST_LIKEs to this peer + * Check whether we should push MULTICAST_LIKEs to this peer, and update last sent time if true * * @param now Current time * @return True if we should update multicasts */ - inline bool shouldLikeMulticasts(const uint64_t now) const { return ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD); } - - /** - * Set time we last updated multicasts for this peer - * - * @param now Current time - */ - inline void likingMulticasts(const uint64_t now) { _lastUpdatedMulticast = now; } + inline bool multicastLikeGate(const uint64_t now) + { + if ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD) { + _lastUpdatedMulticast = now; + return true; + } + return false; + } /** * Check whether the peer represented by this Membership should be allowed on this network at all @@ -128,11 +111,11 @@ public: * @return True if this peer has a certificate of ownership for the given resource */ template - inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) + inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) const { uint32_t *k = (uint32_t *)0; CertificateOfOwnership *v = (CertificateOfOwnership *)0; - Hashtable< uint32_t,CertificateOfOwnership >::Iterator i(_remoteCoos); + Hashtable< uint32_t,CertificateOfOwnership >::Iterator i(*(const_cast< Hashtable< uint32_t,CertificateOfOwnership> *>(&_remoteCoos))); while (i.next(k,v)) { if (_isCredentialTimestampValid(nconf,*v)&&(v->owns(r))) return true; @@ -179,13 +162,28 @@ public: AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); /** - * Generates a key for the internal revocation tracking hash table + * Clean internal databases of stale entries + * + * @param now Current time + * @param nconf Current network configuration + */ + void clean(const uint64_t now,const NetworkConfig &nconf); + + /** + * Reset last pushed time for local credentials * - * @param t Credential type - * @param i Credential ID - * @return Key for tracking revocations of this credential + * This is done when we update our network configuration and our credentials have changed + */ + inline void resetPushState() + { + _lastPushedCom = 0; + memset(&_localCredLastPushed,0,sizeof(_localCredLastPushed)); + } + + /** + * Generates a key for the internal use in indexing credentials by type and credential ID */ - static uint64_t revocationKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); } + static uint64_t credentialKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); } private: template @@ -193,12 +191,24 @@ private: { const uint64_t ts = remoteCredential.timestamp(); if (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) { - const uint64_t *threshold = _revocations.get(revocationKey(C::credentialType(),remoteCredential.id())); + const uint64_t *threshold = _revocations.get(credentialKey(C::credentialType(),remoteCredential.id())); return ((!threshold)||(ts > *threshold)); } return false; } + template + void _cleanCredImpl(const NetworkConfig &nconf,Hashtable &remoteCreds) + { + uint32_t *k = (uint32_t *)0; + C *v = (C *)0; + typename Hashtable::Iterator i(remoteCreds); + while (i.next(k,v)) { + if (!_isCredentialTimestampValid(nconf,*v)) + remoteCreds.erase(*k); + } + } + // Last time we pushed MULTICAST_LIKE(s) uint64_t _lastUpdatedMulticast; @@ -211,18 +221,46 @@ private: // Remote member's latest network COM CertificateOfMembership _com; - // Revocations + // Revocations by credentialKey() Hashtable< uint64_t,uint64_t > _revocations; - // Remote credentials and credential state + // Remote credentials that we have received from this member (and that are valid) Hashtable< uint32_t,Tag > _remoteTags; Hashtable< uint32_t,Capability > _remoteCaps; Hashtable< uint32_t,CertificateOfOwnership > _remoteCoos; - // Local credential push state tracking - _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; - _LocalCredentialPushState _localCaps[ZT_MAX_NETWORK_CAPABILITIES]; - _LocalCredentialPushState _localCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + // Time we last pushed our local credentials to this member + struct { + uint64_t tag[ZT_MAX_NETWORK_TAGS]; + uint64_t cap[ZT_MAX_NETWORK_CAPABILITIES]; + uint64_t coo[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + } _localCredLastPushed; + +public: + class CapabilityIterator + { + public: + CapabilityIterator(Membership &m,const NetworkConfig &nconf) : + _hti(m._remoteCaps), + _k((uint32_t *)0), + _c((Capability *)0), + _nconf(nconf) + { + } + + inline Capability *next() + { + if (_hti.next(_k,_c)) + return _c; + else return (Capability *)0; + } + + private: + Hashtable< uint32_t,Capability >::Iterator _hti; + uint32_t *_k; + Capability *_c; + const NetworkConfig &_nconf; + }; }; } // namespace ZeroTier diff --git a/node/Network.cpp b/node/Network.cpp index 0abfdf86..3c607b28 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -534,9 +534,9 @@ static _doZtFilterResult _doZtFilter( } if (inbound) { if (membership) { - if ((src)&&(membership->hasCertificateOfOwnershipFor(nconf,src))) + if ((src)&&(membership->hasCertificateOfOwnershipFor(nconf,src))) ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED; - if (membership->hasCertificateOfOwnershipFor(nconf,macSource)) + if (membership->hasCertificateOfOwnershipFor(nconf,macSource)) ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_MAC_AUTHENTICATED; } } else { @@ -1143,21 +1143,31 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD // _lock is NOT locked when this is called try { if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) - return 0; + return 0; // invalid config that is not for us or not for this network if (_config == nconf) return 1; // OK config, but duplicate of what we already have ZT_VirtualNetworkConfig ctmp; bool oldPortInitialized; - { + { // do things that require lock here, but unlock before calling callbacks Mutex::Lock _l(_lock); + _config = nconf; _lastConfigUpdate = RR->node->now(); _netconfFailure = NETCONF_FAILURE_NONE; + oldPortInitialized = _portInitialized; _portInitialized = true; + _externalConfig(&ctmp); + + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) + m->resetPushState(); } + _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); if (saveToDisk) { @@ -1299,10 +1309,9 @@ bool Network::gate(void *tPtr,const SharedPtr &peer) if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) { if (!m) m = &(_membership(peer->address())); - if (m->shouldLikeMulticasts(now)) { + if (m->multicastLikeGate(now)) { m->pushCredentials(RR,tPtr,now,peer->address(),_config,-1,false); _announceMulticastGroupsTo(tPtr,peer->address(),_allMulticastGroups()); - m->likingMulticasts(now); } return true; } @@ -1338,6 +1347,7 @@ void Network::clean() while (i.next(a,m)) { if (!RR->topology->getPeerNoCache(*a)) _memberships.erase(*a); + else m->clean(now,_config); } } } @@ -1546,8 +1556,7 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu } // Make sure that all "network anchors" have Membership records so we will - // push multicasts to them. Note that _membership() also does this but in a - // piecemeal on-demand fashion. + // push multicasts to them. const std::vector
anchors(_config.anchors()); for(std::vector
::const_iterator a(anchors.begin());a!=anchors.end();++a) _membership(*a); @@ -1559,11 +1568,8 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu Hashtable::Iterator i(_memberships); while (i.next(a,m)) { m->pushCredentials(RR,tPtr,now,*a,_config,-1,false); - if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { - if (!newMulticastGroup) - m->likingMulticasts(now); + if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config)) ) _announceMulticastGroupsTo(tPtr,*a,groups); - } } } } diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 8b9ce6dd..e5e013bd 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -26,6 +26,7 @@ #include "Constants.hpp" #include "../include/ZeroTierOne.h" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Utils.hpp" @@ -44,20 +45,10 @@ class RuntimeEnvironment; /** * Revocation certificate to instantaneously revoke a COM, capability, or tag */ -class Revocation +class Revocation : public Credential { public: - /** - * Credential type being revoked - */ - enum CredentialType - { - CREDENTIAL_TYPE_NULL = 0, - CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership - CREDENTIAL_TYPE_CAPABILITY = 2, - CREDENTIAL_TYPE_TAG = 3, - CREDENTIAL_TYPE_COO = 4 // CertificateOfOwnership - }; + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_REVOCATION; } Revocation() { @@ -73,23 +64,23 @@ public: * @param tgt Target node whose credential(s) are being revoked * @param ct Credential type being revoked */ - Revocation(const uint64_t i,const uint64_t nwid,const uint64_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const CredentialType ct) : + Revocation(const uint32_t i,const uint64_t nwid,const uint32_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const Credential::Type ct) : _id(i), - _networkId(nwid), _credentialId(cid), + _networkId(nwid), _threshold(thr), _flags(fl), _target(tgt), _signedBy(), _type(ct) {} - inline uint64_t id() const { return _id; } + inline uint32_t id() const { return _id; } + inline uint32_t credentialId() const { return _credentialId; } inline uint64_t networkId() const { return _networkId; } - inline uint64_t credentialId() const { return _credentialId; } inline uint64_t threshold() const { return _threshold; } inline const Address &target() const { return _target; } inline const Address &signer() const { return _signedBy; } - inline CredentialType type() const { return _type; } + inline Credential::Type type() const { return _type; } inline bool fastPropagate() const { return ((_flags & ZT_REVOCATION_FLAG_FAST_PROPAGATE) != 0); } @@ -123,8 +114,10 @@ public: { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + b.append((uint32_t)0); // 4 unused bytes, currently set to 0 b.append(_id); b.append(_networkId); + b.append((uint32_t)0); // 4 unused bytes, currently set to 0 b.append(_credentialId); b.append(_threshold); b.append(_flags); @@ -151,14 +144,16 @@ public: unsigned int p = startAt; - _id = b.template at(p); p += 8; + p += 4; // 4 bytes, currently unused + _id = b.template at(p); p += 4; _networkId = b.template at(p); p += 8; - _credentialId = b.template at(p); p += 8; + p += 4; // 4 bytes, currently unused + _credentialId = b.template at(p); p += 4; _threshold = b.template at(p); p += 8; _flags = b.template at(p); p += 8; _target.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; - _type = (CredentialType)b[p++]; + _type = (Credential::Type)b[p++]; if (b[p++] == 1) { if (b.template at(p) == ZT_C25519_SIGNATURE_LEN) { @@ -178,14 +173,14 @@ public: } private: - uint64_t _id; + uint32_t _id; + uint32_t _credentialId; uint64_t _networkId; - uint64_t _credentialId; uint64_t _threshold; uint64_t _flags; Address _target; Address _signedBy; - CredentialType _type; + Credential::Type _type; C25519::Signature _signature; }; diff --git a/node/Tag.hpp b/node/Tag.hpp index 38085906..1f7f6835 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -25,6 +25,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "C25519.hpp" #include "Address.hpp" #include "Identity.hpp" @@ -51,9 +52,11 @@ class RuntimeEnvironment; * Unlike capabilities tags are signed only by the issuer and are never * transferrable. */ -class Tag +class Tag : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_TAG; } + Tag() { memset(this,0,sizeof(Tag)); @@ -67,19 +70,19 @@ public: * @param value Tag value */ Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : - _networkId(nwid), - _ts(ts), _id(id), _value(value), + _networkId(nwid), + _ts(ts), _issuedTo(issuedTo), _signedBy() { } - inline uint64_t networkId() const { return _networkId; } - inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } + inline uint64_t networkId() const { return _networkId; } + inline uint64_t timestamp() const { return _ts; } inline const Address &issuedTo() const { return _issuedTo; } inline const Address &signedBy() const { return _signedBy; } @@ -115,11 +118,9 @@ public: { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - // These are the same between Tag and Capability b.append(_networkId); b.append(_ts); b.append(_id); - b.append(_value); _issuedTo.appendTo(b); @@ -187,10 +188,10 @@ public: }; private: - uint64_t _networkId; - uint64_t _ts; uint32_t _id; uint32_t _value; + uint64_t _networkId; + uint64_t _ts; Address _issuedTo; Address _signedBy; C25519::Signature _signature; -- cgit v1.2.3 From cd050b3423ede9c21e53db3a47cdad7ccf5bcb65 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 4 Apr 2017 08:39:19 -0700 Subject: Performance improvement in controller. --- controller/EmbeddedNetworkController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5bce7886..841060d6 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1076,8 +1076,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( testRec["clock"] = now; testRec["uptime"] = (now - _startTime); testRec["content"] = b; - _db.put("pong",testRec); responseBody = OSUtils::jsonDump(testRec); + _db.writeRaw("pong",responseBody); responseContentType = "application/json"; return 200; -- cgit v1.2.3 From f6d92eb737507e6c56cf59aa7b4c4fce679e23cd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Apr 2017 13:48:19 -0700 Subject: JSONDB fix. --- controller/EmbeddedNetworkController.cpp | 3 ++- controller/JSONDB.cpp | 3 ++- osdep/OSUtils.cpp | 6 +++--- osdep/OSUtils.hpp | 5 ++--- 4 files changed, 9 insertions(+), 8 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b7b740a0..0884deda 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1129,7 +1129,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( } else { Mutex::Lock _l(_db_m); - std::string pfx("network/"); pfx.append(nwids); + std::string pfx("network/"); + pfx.append(nwids); _db.filter(pfx,[](const std::string &n,const json &obj) { return false; // delete }); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 8b6de9b4..756664eb 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -99,8 +99,9 @@ void JSONDB::erase(const std::string &n) void JSONDB::_reload(const std::string &p,const std::string &b) { - std::vector dl(OSUtils::listDirectory(p.c_str())); + std::vector dl(OSUtils::listDirectory(p.c_str(),true)); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { + printf("%s\n",di->c_str()); if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { this->get(b + di->substr(0,di->length() - 5)); } else { diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 33e143da..fd5efed0 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -73,7 +73,7 @@ bool OSUtils::redirectUnixOutputs(const char *stdoutPath,const char *stderrPath) } #endif // __UNIX_LIKE__ -std::vector OSUtils::listDirectory(const char *path) +std::vector OSUtils::listDirectory(const char *path,bool includeDirectories) { std::vector r; @@ -82,7 +82,7 @@ std::vector OSUtils::listDirectory(const char *path) WIN32_FIND_DATAA ffd; if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { do { - if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))&&((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)) + if ( (strcmp(ffd.cFileName,".")) && (strcmp(ffd.cFileName,"..")) && (((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)||(((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)&&(includeDirectories))) ) r.push_back(std::string(ffd.cFileName)); } while (FindNextFileA(hFind,&ffd)); FindClose(hFind); @@ -98,7 +98,7 @@ std::vector OSUtils::listDirectory(const char *path) if (readdir_r(d,&de,&dptr)) break; if (dptr) { - if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type != DT_DIR)) + if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&((dptr->d_type != DT_DIR)||(includeDirectories))) r.push_back(std::string(dptr->d_name)); } else break; } diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 2e007ef2..b84d5d2d 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -104,12 +104,11 @@ public: /** * List a directory's contents * - * This returns only files, not sub-directories. - * * @param path Path to list + * @param includeDirectories If true, include directories as well as files * @return Names of files in directory (without path prepended) */ - static std::vector listDirectory(const char *path); + static std::vector listDirectory(const char *path,bool includeDirectories = false); /** * Clean a directory of files whose last modified time is older than this -- cgit v1.2.3 From bc61357a44cf4906dda2b30c4474ae891982e620 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Apr 2017 17:37:44 -0700 Subject: HTTP backend support for JSONDB --- controller/EmbeddedNetworkController.cpp | 2 - controller/JSONDB.cpp | 161 ++++++++++++++++++++++--------- controller/JSONDB.hpp | 12 +-- osdep/Http.hpp | 33 +++++++ service/OneService.cpp | 24 ++++- 5 files changed, 175 insertions(+), 57 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0884deda..597bc9c9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -434,8 +434,6 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa _db(dbPath), _node(node) { - OSUtils::mkdir(dbPath); - OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions } EmbeddedNetworkController::~EmbeddedNetworkController() diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 756664eb..afc0631d 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -18,43 +18,67 @@ #include "JSONDB.hpp" +#define ZT_JSONDB_HTTP_TIMEOUT 60000 + namespace ZeroTier { static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); +static const std::map _ZT_JSONDB_GET_HEADERS; + +JSONDB::JSONDB(const std::string &basePath) : + _basePath(basePath) +{ + if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { + // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. + // Typically it's used with 127.0.0.1 anyway. + std::string hn = _basePath.substr(7); + std::size_t hnend = hn.find_first_of('/'); + if (hnend != std::string::npos) + hn = hn.substr(0,hnend); + std::size_t hnsep = hn.find_last_of(':'); + if (hnsep != std::string::npos) + hn[hnsep] = '/'; + _httpAddr.fromString(hn); + if (hnend != std::string::npos) + _basePath = _basePath.substr(7 + hnend); + if (_basePath.length() == 0) + _basePath = "/"; + if (_basePath[0] != '/') + _basePath = std::string("/") + _basePath; + } else { + OSUtils::mkdir(_basePath.c_str()); + OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions + } + _reload(_basePath,std::string()); +} bool JSONDB::writeRaw(const std::string &n,const std::string &obj) { if (!_isValidObjectName(n)) return false; - - const std::string path(_genPath(n,true)); - if (!path.length()) - return false; - - const std::string buf(obj); - if (!OSUtils::writeFile(path.c_str(),buf)) - return false; - - return true; + if (_httpAddr) { + std::map headers; + std::string body; + std::map reqHeaders; + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); + reqHeaders["Content-Length"] = tmp; + reqHeaders["Content-Type"] = "application/json"; + const unsigned int sc = Http::PUT(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),obj.length(),headers,body); + return (sc == 200); + } else { + const std::string path(_genPath(n,true)); + if (!path.length()) + return false; + return OSUtils::writeFile(path.c_str(),obj); + } } bool JSONDB::put(const std::string &n,const nlohmann::json &obj) { - if (!_isValidObjectName(n)) - return false; - - const std::string path(_genPath(n,true)); - if (!path.length()) - return false; - - const std::string buf(OSUtils::jsonDump(obj)); - if (!OSUtils::writeFile(path.c_str(),buf)) - return false; - - _E &e = _db[n]; - e.obj = obj; - - return true; + const bool r = writeRaw(n,OSUtils::jsonDump(obj)); + _db[n].obj = obj; + return r; } const nlohmann::json &JSONDB::get(const std::string &n) @@ -66,22 +90,28 @@ const nlohmann::json &JSONDB::get(const std::string &n) if (e != _db.end()) return e->second.obj; - const std::string path(_genPath(n,false)); - if (!path.length()) - return _EMPTY_JSON; std::string buf; - if (!OSUtils::readFile(path.c_str(),buf)) - return _EMPTY_JSON; + if (_httpAddr) { + std::map headers; + const unsigned int sc = Http::GET(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,buf); + if (sc != 200) + return _EMPTY_JSON; + } else { + const std::string path(_genPath(n,false)); + if (!path.length()) + return _EMPTY_JSON; + if (!OSUtils::readFile(path.c_str(),buf)) + return _EMPTY_JSON; + } - _E &e2 = _db[n]; try { + _E &e2 = _db[n]; e2.obj = OSUtils::jsonParse(buf); + return e2.obj; } catch ( ... ) { - e2.obj = _EMPTY_JSON; - buf = "{}"; + _db.erase(n); + return _EMPTY_JSON; } - - return e2.obj; } void JSONDB::erase(const std::string &n) @@ -89,23 +119,50 @@ void JSONDB::erase(const std::string &n) if (!_isValidObjectName(n)) return; - std::string path(_genPath(n,true)); - if (!path.length()) - return; + if (_httpAddr) { + std::string body; + std::map headers; + Http::DEL(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + } else { + std::string path(_genPath(n,true)); + if (!path.length()) + return; + OSUtils::rm(path.c_str()); + } - OSUtils::rm(path.c_str()); _db.erase(n); } void JSONDB::_reload(const std::string &p,const std::string &b) { - std::vector dl(OSUtils::listDirectory(p.c_str(),true)); - for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { - printf("%s\n",di->c_str()); - if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { - this->get(b + di->substr(0,di->length() - 5)); - } else { - this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); + if (_httpAddr) { + std::string body; + std::map headers; + const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + if (sc == 200) { + try { + nlohmann::json dbImg(OSUtils::jsonParse(body)); + std::string tmp; + if (dbImg.is_object()) { + for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { + if (i.value().is_object()) { + tmp = i.key(); + _db[tmp].obj = i.value(); + } + } + } + } catch ( ... ) { + // TODO: report error? + } + } + } else { + std::vector dl(OSUtils::listDirectory(p.c_str(),true)); + for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { + if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { + this->get(b + di->substr(0,di->length() - 5)); + } else { + this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); + } } } } @@ -130,15 +187,23 @@ std::string JSONDB::_genPath(const std::string &n,bool create) if (pt.size() == 0) return std::string(); + char sep; + if (_httpAddr) { + sep = '/'; + create = false; + } else { + sep = ZT_PATH_SEPARATOR; + } + std::string p(_basePath); if (create) OSUtils::mkdir(p.c_str()); for(unsigned long i=0,j=(unsigned long)(pt.size()-1);i _db; }; diff --git a/osdep/Http.hpp b/osdep/Http.hpp index 1ecf4eec..e7d4d03e 100644 --- a/osdep/Http.hpp +++ b/osdep/Http.hpp @@ -135,6 +135,39 @@ public: responseBody); } + /** + * Make HTTP PUT request + * + * It is the responsibility of the caller to set all headers. With PUT, the + * Content-Length and Content-Type headers must be set or the PUT will not + * work. + * + * @return HTTP status code or 0 on error (responseBody will contain error message) + */ + static inline unsigned int PUT( + unsigned long maxResponseSize, + unsigned long timeout, + const struct sockaddr *remoteAddress, + const char *path, + const std::map &requestHeaders, + const void *postData, + unsigned long postDataLength, + std::map &responseHeaders, + std::string &responseBody) + { + return _do( + "PUT", + maxResponseSize, + timeout, + remoteAddress, + path, + requestHeaders, + postData, + postDataLength, + responseHeaders, + responseBody); + } + private: static unsigned int _do( const char *method, diff --git a/service/OneService.cpp b/service/OneService.cpp index c07b3ba4..5f2adfe3 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -381,6 +381,7 @@ public: const std::string _homePath; std::string _authToken; + std::string _controllerDbPath; EmbeddedNetworkController *_controller; Phy _phy; Node *_node; @@ -482,6 +483,7 @@ public: OneServiceImpl(const char *hp,unsigned int port) : _homePath((hp) ? hp : ".") + ,_controllerDbPath(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH) ,_controller((EmbeddedNetworkController *)0) ,_phy(this,false,true) ,_node((Node *)0) @@ -747,7 +749,7 @@ public: for(int i=0;i<3;++i) _portsBE[i] = Utils::hton((uint16_t)_ports[i]); - _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str()); + _controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str()); _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER @@ -1522,6 +1524,26 @@ public: _allowManagementFrom.push_back(nw); } } + + json &controllerDbHttpHost = settings["controllerDbHttpHost"]; + json &controllerDbHttpPort = settings["controllerDbHttpPort"]; + json &controllerDbHttpPath = settings["controllerDbHttpPath"]; + if ((controllerDbHttpHost.is_string())&&(controllerDbHttpPort.is_number())) { + _controllerDbPath = "http://"; + _controllerDbPath.append(controllerDbHttpHost); + char dbp[128]; + Utils::snprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort); + _controllerDbPath.push_back(':'); + _controllerDbPath.append(dbp); + if (controllerDbHttpPath.is_string()) { + std::string p = controllerDbHttpPath; + if ((p.length() == 0)||(p[0] != '/')) + _controllerDbPath.push_back('/'); + _controllerDbPath.append(p); + } else { + _controllerDbPath.push_back('/'); + } + } } // Checks if a managed IP or route target is allowed -- cgit v1.2.3 From ba0d73d102e09b6d60f1d677b1e5472a25c08d42 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 20 Apr 2017 10:21:40 -0700 Subject: Windows build fixes. --- controller/JSONDB.cpp | 2 +- service/OneService.cpp | 3 ++- windows/ZeroTierOne/ZeroTierOne.vcxproj | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index afc0631d..007e0fec 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -64,7 +64,7 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) Utils::snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); reqHeaders["Content-Length"] = tmp; reqHeaders["Content-Type"] = "application/json"; - const unsigned int sc = Http::PUT(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),obj.length(),headers,body); + const unsigned int sc = Http::PUT(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); return (sc == 200); } else { const std::string path(_genPath(n,true)); diff --git a/service/OneService.cpp b/service/OneService.cpp index 5f2adfe3..a9185eea 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1530,7 +1530,8 @@ public: json &controllerDbHttpPath = settings["controllerDbHttpPath"]; if ((controllerDbHttpHost.is_string())&&(controllerDbHttpPort.is_number())) { _controllerDbPath = "http://"; - _controllerDbPath.append(controllerDbHttpHost); + std::string h = controllerDbHttpHost; + _controllerDbPath.append(h); char dbp[128]; Utils::snprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort); _controllerDbPath.push_back(':'); diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 96de5d2b..9715f3d9 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -360,12 +360,13 @@ STATICLIB;ZT_SALSA20_SSE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;WIN32;NOMINMAX;ZT_SOFTWARE_UPDATE_DEFAULT="apply";ZT_BUILD_PLATFORM=2;ZT_BUILD_ARCHITECTURE=1;%(PreprocessorDefinitions) MultiThreaded - NoExtensions + StreamingSIMDExtensions2 true AnySuitable Speed true 4996 + Guard true -- cgit v1.2.3 From f4feccc6265cc480b84c85f897b225714072d4ec Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 24 Apr 2017 09:09:04 -0700 Subject: Do not serve controller requests until init is done. --- controller/JSONDB.cpp | 20 +++++++++++++------- controller/JSONDB.hpp | 15 ++++++++------- 2 files changed, 21 insertions(+), 14 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 007e0fec..d3e76fc1 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -26,7 +26,8 @@ static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); static const std::map _ZT_JSONDB_GET_HEADERS; JSONDB::JSONDB(const std::string &basePath) : - _basePath(basePath) + _basePath(basePath), + _ready(false) { if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. @@ -49,7 +50,7 @@ JSONDB::JSONDB(const std::string &basePath) : OSUtils::mkdir(_basePath.c_str()); OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions } - _reload(_basePath,std::string()); + _ready = _reload(_basePath,std::string()); } bool JSONDB::writeRaw(const std::string &n,const std::string &obj) @@ -83,9 +84,13 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) const nlohmann::json &JSONDB::get(const std::string &n) { + while (!_ready) { + Thread::sleep(250); + _ready = _reload(_basePath,std::string()); + } + if (!_isValidObjectName(n)) return _EMPTY_JSON; - std::map::iterator e(_db.find(n)); if (e != _db.end()) return e->second.obj; @@ -133,7 +138,7 @@ void JSONDB::erase(const std::string &n) _db.erase(n); } -void JSONDB::_reload(const std::string &p,const std::string &b) +bool JSONDB::_reload(const std::string &p,const std::string &b) { if (_httpAddr) { std::string body; @@ -150,11 +155,11 @@ void JSONDB::_reload(const std::string &p,const std::string &b) _db[tmp].obj = i.value(); } } + return true; } - } catch ( ... ) { - // TODO: report error? - } + } catch ( ... ) {} // invalid JSON, so maybe incomplete request } + return false; } else { std::vector dl(OSUtils::listDirectory(p.c_str(),true)); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { @@ -164,6 +169,7 @@ void JSONDB::_reload(const std::string &p,const std::string &b) this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); } } + return true; } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index c19112ed..beafbaf5 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -36,6 +36,7 @@ #include "../ext/json/json.hpp" #include "../osdep/OSUtils.hpp" #include "../osdep/Http.hpp" +#include "../osdep/Thread.hpp" namespace ZeroTier { @@ -47,12 +48,6 @@ class JSONDB public: JSONDB(const std::string &basePath); - inline void reload() - { - _db.clear(); - _reload(_basePath,std::string()); - } - bool writeRaw(const std::string &n,const std::string &obj); bool put(const std::string &n,const nlohmann::json &obj); @@ -79,6 +74,11 @@ public: template inline void filter(const std::string &prefix,F func) { + while (!_ready) { + Thread::sleep(250); + _ready = _reload(_basePath,std::string()); + } + for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { if (!func(i->first,get(i->first))) { @@ -94,7 +94,7 @@ public: inline bool operator!=(const JSONDB &db) const { return (!(*this == db)); } private: - void _reload(const std::string &p,const std::string &b); + bool _reload(const std::string &p,const std::string &b); bool _isValidObjectName(const std::string &n); std::string _genPath(const std::string &n,bool create); @@ -108,6 +108,7 @@ private: InetAddress _httpAddr; std::string _basePath; std::map _db; + volatile bool _ready; }; } // namespace ZeroTier -- cgit v1.2.3 From cafbe44dde55e57115cc654610172556dff19bec Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 24 Apr 2017 19:16:36 -0700 Subject: Controller optimizations -- make locking more fine-grained, use true hardware concurrency, etc. --- controller/EmbeddedNetworkController.cpp | 193 ++++++++++++------------------- controller/EmbeddedNetworkController.hpp | 7 +- controller/JSONDB.cpp | 36 ++++-- controller/JSONDB.hpp | 15 +-- selftest.cpp | 2 + 5 files changed, 106 insertions(+), 147 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 597bc9c9..3b901afe 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include "../include/ZeroTierOne.h" #include "../node/Constants.hpp" @@ -430,7 +431,6 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _startTime(OSUtils::now()), - _threadsStarted(false), _db(dbPath), _node(node) { @@ -439,11 +439,11 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa EmbeddedNetworkController::~EmbeddedNetworkController() { Mutex::Lock _l(_threads_m); - if (_threadsStarted) { - for(int i=0;i<(ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT*2);++i) + if (_threads.size() > 0) { + for(unsigned long i=0;i<(((unsigned long)_threads.size())*2);++i) _queue.post((_RQEntry *)0); - for(int i=0;i::iterator i(_threads.begin());i!=_threads.end();++i) + Thread::join(*i); } } @@ -465,11 +465,13 @@ void EmbeddedNetworkController::request( { Mutex::Lock _l(_threads_m); - if (!_threadsStarted) { - for(int i=0;i= 4) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - json member; - { - Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString()); - } + json member(_db.get("network",nwids,"member",Address(address).toString())); if (!member.size()) return 404; - _addMemberNonPersistedFields(member,OSUtils::now()); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; - - return 200; } else { - - Mutex::Lock _l(_db_m); - responseBody = "{"; _db.filter((std::string("network/") + nwids + "/member/"),[&responseBody](const std::string &n,const json &member) { if ((member.is_object())&&(member.size() > 0)) { @@ -540,9 +527,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( }); responseBody.push_back('}'); responseContentType = "application/json"; - - return 200; } + return 200; } // else 404 @@ -560,14 +546,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } else if (path.size() == 1) { std::set networkIds; - { - Mutex::Lock _l(_db_m); - _db.filter("network/",[&networkIds](const std::string &n,const json &obj) { - if (n.length() == (16 + 8)) - networkIds.insert(n.substr(8)); - return true; // do not delete - }); - } + _db.filter("network/",[&networkIds](const std::string &n,const json &obj) { + if (n.length() == (16 + 8)) + networkIds.insert(n.substr(8)); + return true; // do not delete + }); responseBody.push_back('['); for(std::set::iterator i(networkIds.begin());i!=networkIds.end();++i) { @@ -634,11 +617,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); - json member; - { - Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString()); - } + json member(_db.get("network",nwids,"member",Address(address).toString())); json origMember(member); // for detecting changes _initMember(member); @@ -735,10 +714,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["lastModified"] = now; json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",Address(address).toString(),member); - } + _db.put("network",nwids,"member",Address(address).toString(),member); _pushMemberUpdate(now,nwid,member); } @@ -806,31 +782,26 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } else { // POST to network ID - json network; - { - Mutex::Lock _l(_db_m); - - // Magic ID ending with ______ picks a random unused network ID - if (path[1].substr(10) == "______") { - nwid = 0; - uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; - uint64_t nwidPostfix = 0; - for(unsigned long k=0;k<100000;++k) { // sanity limit on trials - Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); - uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); - if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (_db.get("network",nwids).size() <= 0) { - nwid = tryNwid; - break; - } + // Magic ID ending with ______ picks a random unused network ID + if (path[1].substr(10) == "______") { + nwid = 0; + uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; + uint64_t nwidPostfix = 0; + for(unsigned long k=0;k<100000;++k) { // sanity limit on trials + Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); + uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); + if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); + if (_db.get("network",nwids).size() <= 0) { + nwid = tryNwid; + break; } - if (!nwid) - return 503; } - - network = _db.get("network",nwids); + if (!nwid) + return 503; } + json network(_db.get("network",nwids)); + json origNetwork(network); // for detecting changes _initNetwork(network); @@ -1044,10 +1015,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); network["lastModified"] = now; - { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,network); - } + _db.put("network",nwids,network); // Send an update to all members of the network _db.filter((std::string("network/") + nwids + "/member/"),[this,&now,&nwid](const std::string &n,const json &obj) { @@ -1101,11 +1069,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network; - { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids); - } + json network(_db.get("network",nwids)); if (!network.size()) return 404; @@ -1113,8 +1077,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - Mutex::Lock _l(_db_m); - json member = _db.get("network",nwids,"member",Address(address).toString()); _db.erase("network",nwids,"member",Address(address).toString()); @@ -1125,8 +1087,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( return 200; } } else { - Mutex::Lock _l(_db_m); - std::string pfx("network/"); pfx.append(nwids); _db.filter(pfx,[](const std::string &n,const json &obj) { @@ -1226,7 +1186,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str(), ((double)report->receivedFromLinkQuality / (double)ZT_PATH_LINK_QUALITY_MAX)); - Mutex::Lock _l(self->_db_m); self->_db.writeRaw(id,std::string(tmp)); } @@ -1252,13 +1211,8 @@ void EmbeddedNetworkController::_request( char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network; - json member; - { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids); - member = _db.get("network",nwids,"member",identity.address().toString()); - } + json network(_db.get("network",nwids)); + json member(_db.get("network",nwids,"member",identity.address().toString())); if (!network.size()) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); @@ -1403,7 +1357,6 @@ void EmbeddedNetworkController::_request( if (!authorizedBy) { if (origMember != member) { member["lastModified"] = now; - Mutex::Lock _l(_db_m); _db.put("network",nwids,"member",identity.address().toString(),member); } _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); @@ -1759,7 +1712,6 @@ void EmbeddedNetworkController::_request( if (member != origMember) { member["lastModified"] = now; - Mutex::Lock _l(_db_m); _db.put("network",nwids,"member",identity.address().toString(),member); } @@ -1780,45 +1732,42 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid } } - { - Mutex::Lock _l(_db_m); - _db.filter(pfx,[&nmi,&now](const std::string &n,const json &member) { - try { - if (OSUtils::jsonBool(member["authorized"],false)) { - ++nmi.authorizedMemberCount; - - if (member.count("recentLog")) { - const json &mlog = member["recentLog"]; - if ((mlog.is_array())&&(mlog.size() > 0)) { - const json &mlog1 = mlog[0]; - if (mlog1.is_object()) { - if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) - ++nmi.activeMemberCount; - } + _db.filter(pfx,[&nmi,&now](const std::string &n,const json &member) { + try { + if (OSUtils::jsonBool(member["authorized"],false)) { + ++nmi.authorizedMemberCount; + + if (member.count("recentLog")) { + const json &mlog = member["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + const json &mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) + ++nmi.activeMemberCount; } } + } - if (OSUtils::jsonBool(member["activeBridge"],false)) { - nmi.activeBridges.insert(Address(Utils::hexStrToU64(OSUtils::jsonString(member["id"],"0000000000").c_str()))); - } + if (OSUtils::jsonBool(member["activeBridge"],false)) { + nmi.activeBridges.insert(Address(Utils::hexStrToU64(OSUtils::jsonString(member["id"],"0000000000").c_str()))); + } - if (member.count("ipAssignments")) { - const json &mips = member["ipAssignments"]; - if (mips.is_array()) { - for(unsigned long i=0;i _queue; - Thread _threads[ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT]; - bool _threadsStarted; + std::vector _threads; Mutex _threads_m; std::map _nmiCache; Mutex _nmiCache_m; JSONDB _db; - Mutex _db_m; Node *const _node; std::string _path; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index d3e76fc1..dd8e3968 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -78,22 +78,29 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) bool JSONDB::put(const std::string &n,const nlohmann::json &obj) { const bool r = writeRaw(n,OSUtils::jsonDump(obj)); - _db[n].obj = obj; + { + Mutex::Lock _l(_db_m); + _db[n].obj = obj; + } return r; } -const nlohmann::json &JSONDB::get(const std::string &n) +nlohmann::json JSONDB::get(const std::string &n) { - while (!_ready) { - Thread::sleep(250); - _ready = _reload(_basePath,std::string()); - } + { + Mutex::Lock _l(_db_m); - if (!_isValidObjectName(n)) - return _EMPTY_JSON; - std::map::iterator e(_db.find(n)); - if (e != _db.end()) - return e->second.obj; + while (!_ready) { + Thread::sleep(250); + _ready = _reload(_basePath,std::string()); + } + + if (!_isValidObjectName(n)) + return _EMPTY_JSON; + std::map::iterator e(_db.find(n)); + if (e != _db.end()) + return e->second.obj; + } std::string buf; if (_httpAddr) { @@ -110,6 +117,7 @@ const nlohmann::json &JSONDB::get(const std::string &n) } try { + Mutex::Lock _l(_db_m); _E &e2 = _db[n]; e2.obj = OSUtils::jsonParse(buf); return e2.obj; @@ -135,11 +143,15 @@ void JSONDB::erase(const std::string &n) OSUtils::rm(path.c_str()); } - _db.erase(n); + { + Mutex::Lock _l(_db_m); + _db.erase(n); + } } bool JSONDB::_reload(const std::string &p,const std::string &b) { + // Assumes _db_m is locked if (_httpAddr) { std::string body; std::map headers; diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index beafbaf5..2d3a5224 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -51,18 +51,16 @@ public: bool writeRaw(const std::string &n,const std::string &obj); bool put(const std::string &n,const nlohmann::json &obj); - inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); } inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); } inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); } - const nlohmann::json &get(const std::string &n); - - inline const nlohmann::json &get(const std::string &n1,const std::string &n2) { return this->get((n1 + "/" + n2)); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3) { return this->get((n1 + "/" + n2 + "/" + n3)); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4)); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5)); } + nlohmann::json get(const std::string &n); + inline nlohmann::json get(const std::string &n1,const std::string &n2) { return this->get((n1 + "/" + n2)); } + inline nlohmann::json get(const std::string &n1,const std::string &n2,const std::string &n3) { return this->get((n1 + "/" + n2 + "/" + n3)); } + inline nlohmann::json get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4)); } + inline nlohmann::json get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5)); } void erase(const std::string &n); @@ -74,6 +72,8 @@ public: template inline void filter(const std::string &prefix,F func) { + Mutex::Lock _l(_db_m); + while (!_ready) { Thread::sleep(250); _ready = _reload(_basePath,std::string()); @@ -108,6 +108,7 @@ private: InetAddress _httpAddr; std::string _basePath; std::map _db; + Mutex _db_m; volatile bool _ready; }; diff --git a/selftest.cpp b/selftest.cpp index 91d304a6..33e65f2c 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "node/Constants.hpp" #include "node/Hashtable.hpp" @@ -1114,6 +1115,7 @@ int main(int argc,char **argv) */ std::cout << "[info] sizeof(void *) == " << sizeof(void *) << std::endl; + std::cout << "[info] hardware concurrency == " << std::thread::hardware_concurrency() << std::endl; std::cout << "[info] sizeof(NetworkConfig) == " << sizeof(ZeroTier::NetworkConfig) << std::endl; srand((unsigned int)time(0)); -- cgit v1.2.3 From 4f2a77976965f19640416afca24367d3037820b8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 24 Apr 2017 20:51:02 -0700 Subject: JSONDB performance improvements, threading fix. --- controller/EmbeddedNetworkController.cpp | 59 +++++++++++++++++--------------- controller/EmbeddedNetworkController.hpp | 1 + controller/JSONDB.cpp | 57 ++++++++++++++++-------------- controller/JSONDB.hpp | 31 ++++++++--------- node/Node.cpp | 3 +- osdep/Thread.hpp | 42 +++++++++-------------- 6 files changed, 96 insertions(+), 97 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3b901afe..84906849 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -431,6 +431,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _startTime(OSUtils::now()), + _running(true), _db(dbPath), _node(node) { @@ -438,12 +439,19 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa EmbeddedNetworkController::~EmbeddedNetworkController() { - Mutex::Lock _l(_threads_m); - if (_threads.size() > 0) { - for(unsigned long i=0;i<(((unsigned long)_threads.size())*2);++i) + _running = false; + std::vector t; + { + Mutex::Lock _l(_threads_m); + t = _threads; + } + if (t.size() > 0) { + for(unsigned long i=0,j=(unsigned long)(t.size() * 4);i::iterator i(_threads.begin());i!=_threads.end();++i) + /* + for(std::vector::iterator i(t.begin());i!=t.end();++i) Thread::join(*i); + */ } } @@ -1111,23 +1119,23 @@ void EmbeddedNetworkController::threadMain() throw() { uint64_t lastCircuitTestCheck = 0; - for(;;) { - _RQEntry *const qe = _queue.get(); // waits on next request - if (!qe) break; // enqueue a NULL to terminate threads + _RQEntry *qe = (_RQEntry *)0; + while ((_running)&&((qe = _queue.get()))) { try { _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); } catch ( ... ) {} delete qe; - - uint64_t now = OSUtils::now(); - if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - lastCircuitTestCheck = now; - Mutex::Lock _l(_tests_m); - for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { - if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - _node->circuitTestEnd(&(*i)); - _tests.erase(i++); - } else ++i; + if (_running) { + uint64_t now = OSUtils::now(); + if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + lastCircuitTestCheck = now; + Mutex::Lock _l(_tests_m); + for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { + if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + _node->circuitTestEnd(&(*i)); + _tests.erase(i++); + } else ++i; + } } } } @@ -1723,13 +1731,11 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid char pfx[256]; Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid); - { - Mutex::Lock _l(_nmiCache_m); - std::map::iterator c(_nmiCache.find(nwid)); - if ((c != _nmiCache.end())&&((now - c->second.nmiTimestamp) < 1000)) { // a short duration cache but limits CPU use on big networks - nmi = c->second; - return; - } + Mutex::Lock _l(_nmiCache_m); + std::map::iterator c(_nmiCache.find(nwid)); + if ((c != _nmiCache.end())&&((now - c->second.nmiTimestamp) < 1000)) { // a short duration cache but limits CPU use on big networks + nmi = c->second; + return; } _db.filter(pfx,[&nmi,&now](const std::string &n,const json &member) { @@ -1770,10 +1776,7 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid }); nmi.nmiTimestamp = now; - { - Mutex::Lock _l(_nmiCache_m); - _nmiCache[nwid] = nmi; - } + _nmiCache[nwid] = nmi; } void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member) diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 906f4345..04f52c7d 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -178,6 +178,7 @@ private: const uint64_t _startTime; + volatile bool _running; BlockingQueue<_RQEntry *> _queue; std::vector _threads; Mutex _threads_m; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index dd8e3968..509250b9 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -50,7 +50,7 @@ JSONDB::JSONDB(const std::string &basePath) : OSUtils::mkdir(_basePath.c_str()); OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions } - _ready = _reload(_basePath,std::string()); + _reload(_basePath,std::string()); } bool JSONDB::writeRaw(const std::string &n,const std::string &obj) @@ -87,16 +87,16 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) nlohmann::json JSONDB::get(const std::string &n) { - { - Mutex::Lock _l(_db_m); + while (!_ready) { + Thread::sleep(250); + _reload(_basePath,std::string()); + } - while (!_ready) { - Thread::sleep(250); - _ready = _reload(_basePath,std::string()); - } + if (!_isValidObjectName(n)) + return _EMPTY_JSON; - if (!_isValidObjectName(n)) - return _EMPTY_JSON; + { + Mutex::Lock _l(_db_m); std::map::iterator e(_db.find(n)); if (e != _db.end()) return e->second.obj; @@ -116,14 +116,16 @@ nlohmann::json JSONDB::get(const std::string &n) return _EMPTY_JSON; } - try { + { Mutex::Lock _l(_db_m); - _E &e2 = _db[n]; - e2.obj = OSUtils::jsonParse(buf); - return e2.obj; - } catch ( ... ) { - _db.erase(n); - return _EMPTY_JSON; + try { + _E &e2 = _db[n]; + e2.obj = OSUtils::jsonParse(buf); + return e2.obj; + } catch ( ... ) { + _db.erase(n); + return _EMPTY_JSON; + } } } @@ -131,7 +133,15 @@ void JSONDB::erase(const std::string &n) { if (!_isValidObjectName(n)) return; + _erase(n); + { + Mutex::Lock _l(_db_m); + _db.erase(n); + } +} +void JSONDB::_erase(const std::string &n) +{ if (_httpAddr) { std::string body; std::map headers; @@ -142,17 +152,12 @@ void JSONDB::erase(const std::string &n) return; OSUtils::rm(path.c_str()); } - - { - Mutex::Lock _l(_db_m); - _db.erase(n); - } } -bool JSONDB::_reload(const std::string &p,const std::string &b) +void JSONDB::_reload(const std::string &p,const std::string &b) { - // Assumes _db_m is locked if (_httpAddr) { + Mutex::Lock _l(_db_m); std::string body; std::map headers; const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); @@ -161,18 +166,19 @@ bool JSONDB::_reload(const std::string &p,const std::string &b) nlohmann::json dbImg(OSUtils::jsonParse(body)); std::string tmp; if (dbImg.is_object()) { + _db.clear(); for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { if (i.value().is_object()) { tmp = i.key(); _db[tmp].obj = i.value(); } } - return true; + _ready = true; } } catch ( ... ) {} // invalid JSON, so maybe incomplete request } - return false; } else { + _ready = true; std::vector dl(OSUtils::listDirectory(p.c_str(),true)); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { @@ -181,7 +187,6 @@ bool JSONDB::_reload(const std::string &p,const std::string &b) this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); } } - return true; } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 2d3a5224..a045d1b4 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -72,29 +72,28 @@ public: template inline void filter(const std::string &prefix,F func) { - Mutex::Lock _l(_db_m); - while (!_ready) { Thread::sleep(250); - _ready = _reload(_basePath,std::string()); + _reload(_basePath,std::string()); } - - for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { - if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { - if (!func(i->first,get(i->first))) { - std::map::iterator i2(i); ++i2; - this->erase(i->first); - i = i2; - } else ++i; - } else break; + { + Mutex::Lock _l(_db_m); + for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { + if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { + if (!func(i->first,i->second.obj)) { + this->_erase(i->first); + _db.erase(i++); + } else { + ++i; + } + } else break; + } } } - inline bool operator==(const JSONDB &db) const { return ((_basePath == db._basePath)&&(_db == db._db)); } - inline bool operator!=(const JSONDB &db) const { return (!(*this == db)); } - private: - bool _reload(const std::string &p,const std::string &b); + void _erase(const std::string &n); + void _reload(const std::string &p,const std::string &b); bool _isValidObjectName(const std::string &n); std::string _genPath(const std::string &n,bool create); diff --git a/node/Node.cpp b/node/Node.cpp index 2b3f7996..ccbe9411 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -490,7 +490,8 @@ int Node::sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *d void Node::setNetconfMaster(void *networkControllerInstance) { RR->localNetworkController = reinterpret_cast(networkControllerInstance); - RR->localNetworkController->init(RR->identity,this); + if (networkControllerInstance) + RR->localNetworkController->init(RR->identity,this); } ZT_ResultCode Node::circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) diff --git a/osdep/Thread.hpp b/osdep/Thread.hpp index 227c2cfe..5423a8ab 100644 --- a/osdep/Thread.hpp +++ b/osdep/Thread.hpp @@ -46,7 +46,6 @@ class Thread { public: Thread() - throw() { _th = NULL; _tid = 0; @@ -54,7 +53,6 @@ public: template static inline Thread start(C *instance) - throw(std::runtime_error) { Thread t; t._th = CreateThread(NULL,0,&___zt_threadMain,(LPVOID)instance,0,&t._tid); @@ -88,7 +86,7 @@ public: CancelSynchronousIo(t._th); } - inline operator bool() const throw() { return (_th != NULL); } + inline operator bool() const { return (_th != NULL); } private: HANDLE _th; @@ -123,33 +121,18 @@ class Thread { public: Thread() - throw() { - memset(&_tid,0,sizeof(_tid)); - pthread_attr_init(&_tattr); - // This corrects for systems with abnormally small defaults (musl) and also - // shrinks the stack on systems with large defaults to save a bit of memory. - pthread_attr_setstacksize(&_tattr,ZT_THREAD_MIN_STACK_SIZE); - _started = false; - } - - ~Thread() - { - pthread_attr_destroy(&_tattr); + memset(this,0,sizeof(Thread)); } Thread(const Thread &t) - throw() { - memcpy(&_tid,&(t._tid),sizeof(_tid)); - _started = t._started; + memcpy(this,&t,sizeof(Thread)); } inline Thread &operator=(const Thread &t) - throw() { - memcpy(&_tid,&(t._tid),sizeof(_tid)); - _started = t._started; + memcpy(this,&t,sizeof(Thread)); return *this; } @@ -163,12 +146,20 @@ public: */ template static inline Thread start(C *instance) - throw(std::runtime_error) { Thread t; - t._started = true; - if (pthread_create(&t._tid,&t._tattr,&___zt_threadMain,instance)) + pthread_attr_t tattr; + pthread_attr_init(&tattr); + // This corrects for systems with abnormally small defaults (musl) and also + // shrinks the stack on systems with large defaults to save a bit of memory. + pthread_attr_setstacksize(&tattr,ZT_THREAD_MIN_STACK_SIZE); + if (pthread_create(&t._tid,&tattr,&___zt_threadMain,instance)) { + pthread_attr_destroy(&tattr); throw std::runtime_error("pthread_create() failed, unable to create thread"); + } else { + t._started = true; + pthread_attr_destroy(&tattr); + } return t; } @@ -190,11 +181,10 @@ public: */ static inline void sleep(unsigned long ms) { usleep(ms * 1000); } - inline operator bool() const throw() { return (_started); } + inline operator bool() const { return (_started); } private: pthread_t _tid; - pthread_attr_t _tattr; volatile bool _started; }; -- cgit v1.2.3 From 4e77365e8d8d49a7329d65b2d5f0508a7f12a097 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 25 Apr 2017 16:17:54 -0700 Subject: Remove a little cruft. --- controller/JSONDB.cpp | 21 --------------------- controller/JSONDB.hpp | 1 - 2 files changed, 22 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 509250b9..1055b036 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -55,8 +55,6 @@ JSONDB::JSONDB(const std::string &basePath) : bool JSONDB::writeRaw(const std::string &n,const std::string &obj) { - if (!_isValidObjectName(n)) - return false; if (_httpAddr) { std::map headers; std::string body; @@ -92,9 +90,6 @@ nlohmann::json JSONDB::get(const std::string &n) _reload(_basePath,std::string()); } - if (!_isValidObjectName(n)) - return _EMPTY_JSON; - { Mutex::Lock _l(_db_m); std::map::iterator e(_db.find(n)); @@ -131,8 +126,6 @@ nlohmann::json JSONDB::get(const std::string &n) void JSONDB::erase(const std::string &n) { - if (!_isValidObjectName(n)) - return; _erase(n); { Mutex::Lock _l(_db_m); @@ -190,20 +183,6 @@ void JSONDB::_reload(const std::string &p,const std::string &b) } } -bool JSONDB::_isValidObjectName(const std::string &n) -{ - if (n.length() == 0) - return false; - const char *p = n.c_str(); - char c; - // For security reasons we should not allow dots, backslashes, or other path characters or potential path characters. - while ((c = *(p++))) { - if (!( ((c >= 'a')&&(c <= 'z')) || ((c >= 'A')&&(c <= 'Z')) || ((c >= '0')&&(c <= '9')) || (c == '/') || (c == '_') || (c == '~') || (c == '-') )) - return false; - } - return true; -} - std::string JSONDB::_genPath(const std::string &n,bool create) { std::vector pt(OSUtils::split(n.c_str(),"/","","")); diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index a045d1b4..09667cf0 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -94,7 +94,6 @@ public: private: void _erase(const std::string &n); void _reload(const std::string &p,const std::string &b); - bool _isValidObjectName(const std::string &n); std::string _genPath(const std::string &n,bool create); struct _E -- cgit v1.2.3 From 12055789351861140740e658f608decc5fb077f2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 26 Apr 2017 06:48:08 -0700 Subject: Big cleanup of controller code, should help performance. --- controller/EmbeddedNetworkController.cpp | 177 ++++++-------------- controller/EmbeddedNetworkController.hpp | 26 +-- controller/JSONDB.cpp | 276 ++++++++++++++++++++++++------- controller/JSONDB.hpp | 163 +++++++++++++----- 4 files changed, 391 insertions(+), 251 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 84906849..be53f2b8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -59,9 +59,6 @@ using json = nlohmann::json; // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 -// Nodes are considered active if they've queried in less than this long -#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD (ZT_NETWORK_AUTOCONF_DELAY * 2) - namespace ZeroTier { static json _renderRule(ZT_VirtualNetworkRule &rule) @@ -474,9 +471,11 @@ void EmbeddedNetworkController::request( { Mutex::Lock _l(_threads_m); if (_threads.size() == 0) { - long hwc = (long)std::thread::hardware_concurrency(); - if (hwc <= 0) + long hwc = (long)(std::thread::hardware_concurrency() / 2); + if (hwc < 1) hwc = 1; + else if (hwc > 16) + hwc = 16; for(long i=0;i= 3) { @@ -516,22 +515,21 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( if (path.size() >= 4) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - json member(_db.get("network",nwids,"member",Address(address).toString())); - if (!member.size()) + json member; + if (!_db.getNetworkMember(nwid,address,member)) return 404; _addMemberNonPersistedFields(member,OSUtils::now()); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; } else { responseBody = "{"; - _db.filter((std::string("network/") + nwids + "/member/"),[&responseBody](const std::string &n,const json &member) { + _db.eachMember(nwid,[&responseBody](uint64_t networkId,uint64_t nodeId,const json &member) { if ((member.is_object())&&(member.size() > 0)) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(OSUtils::jsonString(member["id"],"0")); responseBody.append("\":"); responseBody.append(OSUtils::jsonString(member["revision"],"0")); } - return true; // never delete }); responseBody.push_back('}'); responseContentType = "application/json"; @@ -543,9 +541,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } else { const uint64_t now = OSUtils::now(); - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); - _addNetworkNonPersistedFields(network,now,nmi); + JSONDB::NetworkSummaryInfo ns; + _db.getNetworkSummaryInfo(nwid,ns); + _addNetworkNonPersistedFields(network,now,ns); responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; return 200; @@ -553,21 +551,20 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } } else if (path.size() == 1) { - std::set networkIds; - _db.filter("network/",[&networkIds](const std::string &n,const json &obj) { - if (n.length() == (16 + 8)) - networkIds.insert(n.substr(8)); - return true; // do not delete - }); + std::vector networkIds(_db.networkIds()); + std::sort(networkIds.begin(),networkIds.end()); + char tmp[64]; responseBody.push_back('['); - for(std::set::iterator i(networkIds.begin());i!=networkIds.end();++i) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\""); + for(std::vector::const_iterator i(networkIds.begin());i!=networkIds.end();++i) { + if (responseBody.length() > 1) + responseBody.push_back(','); + Utils::snprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); + responseBody.append(tmp); } responseBody.push_back(']'); responseContentType = "application/json"; + return 200; } // else 404 @@ -625,7 +622,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); - json member(_db.get("network",nwids,"member",Address(address).toString())); + json member; + _db.getNetworkMember(nwid,address,member); json origMember(member); // for detecting changes _initMember(member); @@ -649,7 +647,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( // Member is being de-authorized, so spray Revocation objects to all online members if (!newAuth) { - _clearNetworkMemberInfoCache(nwid); Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); Mutex::Lock _l(_lastRequestTime_m); @@ -722,7 +719,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["lastModified"] = now; json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - _db.put("network",nwids,"member",Address(address).toString(),member); + _db.saveNetworkMember(nwid,address,member); _pushMemberUpdate(now,nwid,member); } @@ -799,8 +796,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (_db.get("network",nwids).size() <= 0) { + if (!_db.hasNetwork(tryNwid)) { nwid = tryNwid; break; } @@ -808,8 +804,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!nwid) return 503; } - json network(_db.get("network",nwids)); + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + json network; + _db.getNetwork(nwid,network); json origNetwork(network); // for detecting changes _initNetwork(network); @@ -1023,18 +1021,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); network["lastModified"] = now; - _db.put("network",nwids,network); + _db.saveNetwork(nwid,network); // Send an update to all members of the network - _db.filter((std::string("network/") + nwids + "/member/"),[this,&now,&nwid](const std::string &n,const json &obj) { - _pushMemberUpdate(now,nwid,obj); - return true; // do not delete + _db.eachMember(nwid,[this,&now,&nwid](uint64_t networkId,uint64_t nodeId,const json &obj) { + this->_pushMemberUpdate(now,nwid,obj); }); } - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); - _addNetworkNonPersistedFields(network,now,nmi); + JSONDB::NetworkSummaryInfo ns; + _db.getNetworkSummaryInfo(nwid,ns); + _addNetworkNonPersistedFields(network,now,ns); responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; @@ -1074,20 +1071,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( if (path[0] == "network") { if ((path.size() >= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network(_db.get("network",nwids)); - if (!network.size()) - return 404; - if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - json member = _db.get("network",nwids,"member",Address(address).toString()); - _db.erase("network",nwids,"member",Address(address).toString()); - + json member = _db.eraseNetworkMember(nwid,address); if (!member.size()) return 404; responseBody = OSUtils::jsonDump(member); @@ -1095,15 +1083,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( return 200; } } else { - std::string pfx("network/"); - pfx.append(nwids); - _db.filter(pfx,[](const std::string &n,const json &obj) { - return false; // delete - }); - - Mutex::Lock _l2(_nmiCache_m); - _nmiCache.erase(nwid); - + json network = _db.eraseNetwork(nwid); + if (!network.size()) + return 404; responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; return 200; @@ -1143,7 +1125,7 @@ void EmbeddedNetworkController::threadMain() void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) { - char tmp[1024],id[128]; + char tmp[2048],id[128]; EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); if ((!test)||(!report)||(!test->credentialNetworkId)) return; // sanity check @@ -1152,6 +1134,7 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes Utils::snprintf(id,sizeof(id),"network/%.16llx/test/%.16llx-%.16llx-%.10llx-%.10llx",test->credentialNetworkId,test->testId,now,report->upstream,report->current); Utils::snprintf(tmp,sizeof(tmp), "{\"id\": \"%s\"," + "\"objtype\": \"circuit_test\"," "\"timestamp\": %llu," "\"networkId\": \"%.16llx\"," "\"testId\": \"%.16llx\"," @@ -1219,16 +1202,14 @@ void EmbeddedNetworkController::_request( char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network(_db.get("network",nwids)); - json member(_db.get("network",nwids,"member",identity.address().toString())); - - if (!network.size()) { + json network,member; + JSONDB::NetworkSummaryInfo ns; + if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; } const bool newMember = (member.size() == 0); - json origMember(member); // for detecting modification later _initMember(member); @@ -1365,7 +1346,7 @@ void EmbeddedNetworkController::_request( if (!authorizedBy) { if (origMember != member) { member["lastModified"] = now; - _db.put("network",nwids,"member",identity.address().toString(),member); + _db.saveNetworkMember(nwid,identity.address().toInt(),member); } _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); return; @@ -1376,15 +1357,13 @@ void EmbeddedNetworkController::_request( // ------------------------------------------------------------------------- NetworkConfig nc; - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; - if (now > nmi.mostRecentDeauthTime) { + if (now > ns.mostRecentDeauthTime) { // If we recently de-authorized a member, shrink credential TTL/max delta to // be below the threshold required to exclude it. Cap this to a min/max to // prevent jitter or absurdly large values. - const uint64_t deauthWindow = now - nmi.mostRecentDeauthTime; + const uint64_t deauthWindow = now - ns.mostRecentDeauthTime; if (deauthWindow < ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA) { credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; } else if (deauthWindow < (ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA + 5000ULL)) { @@ -1403,9 +1382,8 @@ void EmbeddedNetworkController::_request( Utils::scopy(nc.name,sizeof(nc.name),OSUtils::jsonString(network["name"],"").c_str()); nc.multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); - for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) { + for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); - } json &v4AssignMode = network["v4AssignMode"]; json &v6AssignMode = network["v6AssignMode"]; @@ -1629,14 +1607,13 @@ void EmbeddedNetworkController::_request( } // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip6))) { + if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip6)) ) { ipAssignments.push_back(ip6.toIpString()); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) nc.staticIps[nc.staticIpCount++] = ip6; haveManagedIpv6AutoAssignment = true; - _clearNetworkMemberInfoCache(nwid); // clear cache to prevent IP assignment duplication on many rapid assigns break; } } @@ -1682,7 +1659,7 @@ void EmbeddedNetworkController::_request( // If it's routed, then try to claim and assign it and if successful end loop const InetAddress ip4(Utils::hton(ip),0); - if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { + if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip4)) ) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { @@ -1692,7 +1669,6 @@ void EmbeddedNetworkController::_request( v4ip->sin_addr.s_addr = Utils::hton(ip); } haveManagedIpv4AutoAssignment = true; - _clearNetworkMemberInfoCache(nwid); // clear cache to prevent IP assignment duplication on many rapid assigns break; } } @@ -1720,65 +1696,12 @@ void EmbeddedNetworkController::_request( if (member != origMember) { member["lastModified"] = now; - _db.put("network",nwids,"member",identity.address().toString(),member); + _db.saveNetworkMember(nwid,identity.address().toInt(),member); } _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } -void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) -{ - char pfx[256]; - Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid); - - Mutex::Lock _l(_nmiCache_m); - std::map::iterator c(_nmiCache.find(nwid)); - if ((c != _nmiCache.end())&&((now - c->second.nmiTimestamp) < 1000)) { // a short duration cache but limits CPU use on big networks - nmi = c->second; - return; - } - - _db.filter(pfx,[&nmi,&now](const std::string &n,const json &member) { - try { - if (OSUtils::jsonBool(member["authorized"],false)) { - ++nmi.authorizedMemberCount; - - if (member.count("recentLog")) { - const json &mlog = member["recentLog"]; - if ((mlog.is_array())&&(mlog.size() > 0)) { - const json &mlog1 = mlog[0]; - if (mlog1.is_object()) { - if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) - ++nmi.activeMemberCount; - } - } - } - - if (OSUtils::jsonBool(member["activeBridge"],false)) { - nmi.activeBridges.insert(Address(Utils::hexStrToU64(OSUtils::jsonString(member["id"],"0000000000").c_str()))); - } - - if (member.count("ipAssignments")) { - const json &mips = member["ipAssignments"]; - if (mips.is_array()) { - for(unsigned long i=0;i metaData; }; - // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places - struct _NetworkMemberInfo - { - _NetworkMemberInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} - std::set
activeBridges; - std::set allocatedIps; - unsigned long authorizedMemberCount; - unsigned long activeMemberCount; - unsigned long totalMemberCount; - uint64_t mostRecentDeauthTime; - uint64_t nmiTimestamp; // time this NMI structure was computed - }; - static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); - void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); - inline void _clearNetworkMemberInfoCache(const uint64_t nwid) { Mutex::Lock _l(_nmiCache_m); _nmiCache.erase(nwid); } void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); // These init objects with default and static/informational fields @@ -164,12 +149,12 @@ private: } network["objtype"] = "network"; } - inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const _NetworkMemberInfo &nmi) + inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const JSONDB::NetworkSummaryInfo &ns) { network["clock"] = now; - network["authorizedMemberCount"] = nmi.authorizedMemberCount; - network["activeMemberCount"] = nmi.activeMemberCount; - network["totalMemberCount"] = nmi.totalMemberCount; + network["authorizedMemberCount"] = ns.authorizedMemberCount; + network["activeMemberCount"] = ns.activeMemberCount; + network["totalMemberCount"] = ns.totalMemberCount; } inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now) { @@ -183,9 +168,6 @@ private: std::vector _threads; Mutex _threads_m; - std::map _nmiCache; - Mutex _nmiCache_m; - JSONDB _db; Node *const _node; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 1055b036..c8a31ab4 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -26,12 +26,11 @@ static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); static const std::map _ZT_JSONDB_GET_HEADERS; JSONDB::JSONDB(const std::string &basePath) : - _basePath(basePath), - _ready(false) + _basePath(basePath) { if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. - // Typically it's used with 127.0.0.1 anyway. + // Typically it's just used with 127.0.0.1 anyway. std::string hn = _basePath.substr(7); std::size_t hnend = hn.find_first_of('/'); if (hnend != std::string::npos) @@ -50,7 +49,32 @@ JSONDB::JSONDB(const std::string &basePath) : OSUtils::mkdir(_basePath.c_str()); OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions } - _reload(_basePath,std::string()); + + unsigned int cnt = 0; + while (!_load(_basePath)) { + if ((++cnt & 7) == 0) + fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); + Thread::sleep(250); + } + + for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) + _recomputeSummaryInfo(n->first); +} + +JSONDB::~JSONDB() +{ + { + Mutex::Lock _l(_networks_m); + _networks.clear(); + } + { + Mutex::Lock _l(_summaryThread_m); + if (_summaryThread) { + _updateSummaryInfoQueue.post(0); + _updateSummaryInfoQueue.post(0); + Thread::join(_summaryThread); + } + } } bool JSONDB::writeRaw(const std::string &n,const std::string &obj) @@ -73,84 +97,173 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) } } -bool JSONDB::put(const std::string &n,const nlohmann::json &obj) +void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) { - const bool r = writeRaw(n,OSUtils::jsonDump(obj)); + char n[256]; + Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + writeRaw(n,OSUtils::jsonDump(networkConfig)); { - Mutex::Lock _l(_db_m); - _db[n].obj = obj; + Mutex::Lock _l(_networks_m); + _networks[networkId].config = networkConfig; } - return r; + //_recomputeSummaryInfo(networkId); } -nlohmann::json JSONDB::get(const std::string &n) +void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig) { - while (!_ready) { - Thread::sleep(250); - _reload(_basePath,std::string()); + char n[256]; + Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + writeRaw(n,OSUtils::jsonDump(memberConfig)); + { + Mutex::Lock _l(_networks_m); + _networks[networkId].members[nodeId] = memberConfig; } + _recomputeSummaryInfo(networkId); +} - { - Mutex::Lock _l(_db_m); - std::map::iterator e(_db.find(n)); - if (e != _db.end()) - return e->second.obj; +nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) +{ + if (!_httpAddr) { // Member deletion is done by Central in harnessed mode, and deleting the cache network entry also deletes all members + std::vector memberIds; + { + Mutex::Lock _l(_networks_m); + std::unordered_map::iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return _EMPTY_JSON; + for(std::unordered_map::iterator m(i->second.members.begin());m!=i->second.members.end();++m) + memberIds.push_back(m->first); + } + for(std::vector::iterator m(memberIds.begin());m!=memberIds.end();++m) + eraseNetworkMember(networkId,*m,false); } - std::string buf; + char n[256]; + Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + if (_httpAddr) { - std::map headers; - const unsigned int sc = Http::GET(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,buf); - if (sc != 200) - return _EMPTY_JSON; + // Deletion is currently done by Central in harnessed mode + //std::map headers; + //std::string body; + //Http::DEL(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); } else { const std::string path(_genPath(n,false)); - if (!path.length()) - return _EMPTY_JSON; - if (!OSUtils::readFile(path.c_str(),buf)) - return _EMPTY_JSON; + if (path.length()) + OSUtils::rm(path.c_str()); } { - Mutex::Lock _l(_db_m); - try { - _E &e2 = _db[n]; - e2.obj = OSUtils::jsonParse(buf); - return e2.obj; - } catch ( ... ) { - _db.erase(n); - return _EMPTY_JSON; - } + Mutex::Lock _l(_networks_m); + std::unordered_map::iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return _EMPTY_JSON; // sanity check, shouldn't happen + nlohmann::json tmp(i->second.config); + _networks.erase(i); + return tmp; } } -void JSONDB::erase(const std::string &n) +nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo) { - _erase(n); + char n[256]; + Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + + if (_httpAddr) { + // Deletion is currently done by the caller in Central harnessed mode + //std::map headers; + //std::string body; + //Http::DEL(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + } else { + const std::string path(_genPath(n,false)); + if (path.length()) + OSUtils::rm(path.c_str()); + } + { - Mutex::Lock _l(_db_m); - _db.erase(n); + Mutex::Lock _l(_networks_m); + std::unordered_map::iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return _EMPTY_JSON; + std::unordered_map::iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return _EMPTY_JSON; + nlohmann::json tmp(j->second); + i->second.members.erase(j); + if (recomputeSummaryInfo) + _recomputeSummaryInfo(networkId); + return tmp; } } -void JSONDB::_erase(const std::string &n) +void JSONDB::threadMain() + throw() { - if (_httpAddr) { - std::string body; - std::map headers; - Http::DEL(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); - } else { - std::string path(_genPath(n,true)); - if (!path.length()) - return; - OSUtils::rm(path.c_str()); + uint64_t networkId = 0; + while ((networkId = _updateSummaryInfoQueue.get()) != 0) { + const uint64_t now = OSUtils::now(); + { + Mutex::Lock _l(_networks_m); + std::unordered_map::iterator n(_networks.find(networkId)); + if (n != _networks.end()) { + NetworkSummaryInfo &ns = n->second.summaryInfo; + ns.activeBridges.clear(); + ns.allocatedIps.clear(); + ns.authorizedMemberCount = 0; + ns.activeMemberCount = 0; + ns.totalMemberCount = 0; + ns.mostRecentDeauthTime = 0; + + for(std::unordered_map::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { + try { + if (OSUtils::jsonBool(m->second["authorized"],false)) { + ++ns.authorizedMemberCount; + + try { + const nlohmann::json &mlog = m->second["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + const nlohmann::json &mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < (ZT_NETWORK_AUTOCONF_DELAY * 2)) + ++ns.activeMemberCount; + } + } + } catch ( ... ) {} + + try { + if (OSUtils::jsonBool(m->second["activeBridge"],false)) + ns.activeBridges.push_back(Address(m->first)); + } catch ( ... ) {} + + try { + const nlohmann::json &mips = m->second["ipAssignments"]; + if (mips.is_array()) { + for(unsigned long i=0;isecond["lastDeauthorizedTime"],0ULL)); + } catch ( ... ) {} + } + ++ns.totalMemberCount; + } catch ( ... ) {} + } + + std::sort(ns.activeBridges.begin(),ns.activeBridges.end()); + std::sort(ns.allocatedIps.begin(),ns.allocatedIps.end()); + + n->second.summaryInfoLastComputed = now; + } + } } } -void JSONDB::_reload(const std::string &p,const std::string &b) +bool JSONDB::_load(const std::string &p) { if (_httpAddr) { - Mutex::Lock _l(_db_m); std::string body; std::map headers; const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); @@ -159,30 +272,73 @@ void JSONDB::_reload(const std::string &p,const std::string &b) nlohmann::json dbImg(OSUtils::jsonParse(body)); std::string tmp; if (dbImg.is_object()) { - _db.clear(); + Mutex::Lock _l(_networks_m); for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { - if (i.value().is_object()) { - tmp = i.key(); - _db[tmp].obj = i.value(); + nlohmann::json &j = i.value(); + if (j.is_object()) { + std::string id(OSUtils::jsonString(j["id"],"0")); + std::string objtype(OSUtils::jsonString(j["objtype"],"")); + + if ((id.length() == 16)&&(objtype == "network")) { + const uint64_t nwid = Utils::hexStrToU64(id.c_str()); + if (nwid) + _networks[nwid].config = j; + } else if ((id.length() == 10)&&(objtype == "member")) { + const uint64_t mid = Utils::hexStrToU64(id.c_str()); + const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); + if ((mid)&&(nwid)) + _networks[nwid].members[mid] = j; + } } } - _ready = true; + return true; } } catch ( ... ) {} // invalid JSON, so maybe incomplete request } + return false; } else { - _ready = true; std::vector dl(OSUtils::listDirectory(p.c_str(),true)); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { - this->get(b + di->substr(0,di->length() - 5)); + std::string buf; + if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) { + try { + nlohmann::json j(OSUtils::jsonParse(buf)); + std::string id(OSUtils::jsonString(j["id"],"0")); + std::string objtype(OSUtils::jsonString(j["objtype"],"")); + + if ((id.length() == 16)&&(objtype == "network")) { + const uint64_t nwid = Utils::strToU64(id.c_str()); + if (nwid) { + Mutex::Lock _l(_networks_m); + _networks[nwid].config = j; + } + } else if ((id.length() == 10)&&(objtype == "member")) { + const uint64_t mid = Utils::strToU64(id.c_str()); + const uint64_t nwid = Utils::strToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); + if ((mid)&&(nwid)) { + Mutex::Lock _l(_networks_m); + _networks[nwid].members[mid] = j; + } + } + } catch ( ... ) {} + } } else { - this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); + this->_load((p + ZT_PATH_SEPARATOR_S + *di)); } } + return true; } } +void JSONDB::_recomputeSummaryInfo(const uint64_t networkId) +{ + Mutex::Lock _l(_summaryThread_m); + if (!_summaryThread) + _summaryThread = Thread::start(this); + _updateSummaryInfoQueue.post(networkId); +} + std::string JSONDB::_genPath(const std::string &n,bool create) { std::vector pt(OSUtils::split(n.c_str(),"/","","")); diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 09667cf0..0883bd4b 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "../node/Constants.hpp" #include "../node/Utils.hpp" @@ -37,6 +38,7 @@ #include "../osdep/OSUtils.hpp" #include "../osdep/Http.hpp" #include "../osdep/Thread.hpp" +#include "../osdep/BlockingQueue.hpp" namespace ZeroTier { @@ -46,68 +48,145 @@ namespace ZeroTier { class JSONDB { public: + struct NetworkSummaryInfo + { + NetworkSummaryInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} + std::vector
activeBridges; + std::vector allocatedIps; + unsigned long authorizedMemberCount; + unsigned long activeMemberCount; + unsigned long totalMemberCount; + uint64_t mostRecentDeauthTime; + }; + JSONDB(const std::string &basePath); + ~JSONDB(); bool writeRaw(const std::string &n,const std::string &obj); - bool put(const std::string &n,const nlohmann::json &obj); - inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } - inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); } - inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); } - inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); } + inline bool hasNetwork(const uint64_t networkId) const + { + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + return (i != _networks.end()); + } + + inline bool getNetwork(const uint64_t networkId,nlohmann::json &config) const + { + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + config = i->second.config; + return true; + } + + inline bool getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const + { + for(;;) { + { + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + if (i->second.summaryInfoLastComputed) { + ns = i->second.summaryInfo; + return true; + } + } + Thread::sleep(100); // wait for this to be done the first time, which happens when we start + } + } + + /** + * @return Bit mask: 0 == none, 1 == network only, 3 == network and member + */ + inline int getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const + { + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return 0; + networkConfig = i->second.config; + ns = i->second.summaryInfo; + std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return 1; + memberConfig = j->second; + return 3; + } + + inline bool getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const + { + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return false; + memberConfig = j->second; + return true; + } + + void saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig); - nlohmann::json get(const std::string &n); - inline nlohmann::json get(const std::string &n1,const std::string &n2) { return this->get((n1 + "/" + n2)); } - inline nlohmann::json get(const std::string &n1,const std::string &n2,const std::string &n3) { return this->get((n1 + "/" + n2 + "/" + n3)); } - inline nlohmann::json get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4)); } - inline nlohmann::json get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5)); } + void saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig); - void erase(const std::string &n); + nlohmann::json eraseNetwork(const uint64_t networkId); - inline void erase(const std::string &n1,const std::string &n2) { this->erase(n1 + "/" + n2); } - inline void erase(const std::string &n1,const std::string &n2,const std::string &n3) { this->erase(n1 + "/" + n2 + "/" + n3); } - inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4); } - inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5); } + nlohmann::json eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo = true); + + std::vector networkIds() const + { + std::vector r; + Mutex::Lock _l(_networks_m); + for(std::unordered_map::const_iterator n(_networks.begin());n!=_networks.end();++n) + r.push_back(n->first); + return r; + } template - inline void filter(const std::string &prefix,F func) + inline void eachMember(const uint64_t networkId,F func) { - while (!_ready) { - Thread::sleep(250); - _reload(_basePath,std::string()); - } - { - Mutex::Lock _l(_db_m); - for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { - if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { - if (!func(i->first,i->second.obj)) { - this->_erase(i->first); - _db.erase(i++); - } else { - ++i; - } - } else break; + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i != _networks.end()) { + for(std::unordered_map::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { + try { + func(networkId,m->first,m->second); + } catch ( ... ) {} } } } + void threadMain() + throw(); + private: - void _erase(const std::string &n); - void _reload(const std::string &p,const std::string &b); + bool _load(const std::string &p); + void _recomputeSummaryInfo(const uint64_t networkId); std::string _genPath(const std::string &n,bool create); - struct _E + std::string _basePath; + InetAddress _httpAddr; + + BlockingQueue _updateSummaryInfoQueue; + + Thread _summaryThread; + Mutex _summaryThread_m; + + struct _NW { - nlohmann::json obj; - inline bool operator==(const _E &e) const { return (obj == e.obj); } - inline bool operator!=(const _E &e) const { return (obj != e.obj); } + _NW() : summaryInfoLastComputed(0) {} + nlohmann::json config; + NetworkSummaryInfo summaryInfo; + uint64_t summaryInfoLastComputed; + std::unordered_map members; }; - InetAddress _httpAddr; - std::string _basePath; - std::map _db; - Mutex _db_m; - volatile bool _ready; + std::unordered_map _networks; + Mutex _networks_m; }; } // namespace ZeroTier -- cgit v1.2.3 From 7c184cf9919fb9d01b8279397e7b20fa756dc981 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 26 Apr 2017 10:35:59 -0700 Subject: Another performance improvement to controller. --- controller/JSONDB.cpp | 35 +++++++++++++++++++++++++---------- controller/JSONDB.hpp | 5 ++--- 2 files changed, 27 insertions(+), 13 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index c8a31ab4..9b0dd836 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -26,7 +26,8 @@ static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); static const std::map _ZT_JSONDB_GET_HEADERS; JSONDB::JSONDB(const std::string &basePath) : - _basePath(basePath) + _basePath(basePath), + _summaryThreadRun(true) { if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. @@ -67,14 +68,14 @@ JSONDB::~JSONDB() Mutex::Lock _l(_networks_m); _networks.clear(); } + Thread t; { Mutex::Lock _l(_summaryThread_m); - if (_summaryThread) { - _updateSummaryInfoQueue.post(0); - _updateSummaryInfoQueue.post(0); - Thread::join(_summaryThread); - } + _summaryThreadRun = false; + t = _summaryThread; } + if (t) + Thread::join(t); } bool JSONDB::writeRaw(const std::string &n,const std::string &obj) @@ -197,10 +198,21 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ void JSONDB::threadMain() throw() { - uint64_t networkId = 0; - while ((networkId = _updateSummaryInfoQueue.get()) != 0) { - const uint64_t now = OSUtils::now(); + std::vector todo; + while (_summaryThreadRun) { + Thread::sleep(10); + { + Mutex::Lock _l(_summaryThread_m); + if (_summaryThreadToDo.empty()) + continue; + else _summaryThreadToDo.swap(todo); + } + + const uint64_t now = OSUtils::now(); + for(std::vector::iterator ii(todo.begin());ii!=todo.end();++ii) { + const uint64_t networkId = *ii; + Mutex::Lock _l(_networks_m); std::unordered_map::iterator n(_networks.find(networkId)); if (n != _networks.end()) { @@ -258,6 +270,8 @@ void JSONDB::threadMain() n->second.summaryInfoLastComputed = now; } } + + todo.clear(); } } @@ -334,9 +348,10 @@ bool JSONDB::_load(const std::string &p) void JSONDB::_recomputeSummaryInfo(const uint64_t networkId) { Mutex::Lock _l(_summaryThread_m); + if (std::find(_summaryThreadToDo.begin(),_summaryThreadToDo.end(),networkId) == _summaryThreadToDo.end()) + _summaryThreadToDo.push_back(networkId); if (!_summaryThread) _summaryThread = Thread::start(this); - _updateSummaryInfoQueue.post(networkId); } std::string JSONDB::_genPath(const std::string &n,bool create) diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 0883bd4b..55ead4ea 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -38,7 +38,6 @@ #include "../osdep/OSUtils.hpp" #include "../osdep/Http.hpp" #include "../osdep/Thread.hpp" -#include "../osdep/BlockingQueue.hpp" namespace ZeroTier { @@ -171,9 +170,9 @@ private: std::string _basePath; InetAddress _httpAddr; - BlockingQueue _updateSummaryInfoQueue; - Thread _summaryThread; + std::vector _summaryThreadToDo; + volatile bool _summaryThreadRun; Mutex _summaryThread_m; struct _NW -- cgit v1.2.3 From e8ab6adf8902dce98b4721e2f4a5382e54bcbcbc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 26 Apr 2017 12:17:43 -0700 Subject: Deadlock fix. --- controller/JSONDB.cpp | 11 ++++++++++- controller/JSONDB.hpp | 19 ++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 9b0dd836..a2fe7f2a 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -60,6 +60,15 @@ JSONDB::JSONDB(const std::string &basePath) : for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) _recomputeSummaryInfo(n->first); + for(;;) { + _summaryThread_m.lock(); + if (_summaryThreadToDo.empty()) { + _summaryThread_m.unlock(); + break; + } + _summaryThread_m.unlock(); + Thread::sleep(50); + } } JSONDB::~JSONDB() @@ -107,7 +116,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC Mutex::Lock _l(_networks_m); _networks[networkId].config = networkConfig; } - //_recomputeSummaryInfo(networkId); + _recomputeSummaryInfo(networkId); } void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig) diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 55ead4ea..f89ff6c9 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -82,19 +82,12 @@ public: inline bool getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const { - for(;;) { - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return false; - if (i->second.summaryInfoLastComputed) { - ns = i->second.summaryInfo; - return true; - } - } - Thread::sleep(100); // wait for this to be done the first time, which happens when we start - } + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + ns = i->second.summaryInfo; + return true; } /** -- cgit v1.2.3 From 9e80db0fd169de19d5d343e8b6998c8ace4aeb22 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 27 Apr 2017 00:59:36 -0700 Subject: Cleanup, fix a valgrind error, stack use reduction. --- controller/EmbeddedNetworkController.cpp | 134 +- controller/EmbeddedNetworkController.hpp | 3 +- controller/JSONDB.cpp | 57 +- controller/JSONDB.hpp | 55 +- ext/json/LICENSE.MIT | 17 +- ext/json/README.md | 420 +- ext/json/json.hpp | 6976 +++++++++++++++++++----------- make-linux.mk | 9 +- node/CertificateOfOwnership.hpp | 2 + root-watcher/schema.sql | 3 +- selftest.cpp | 47 - 11 files changed, 4910 insertions(+), 2813 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index be53f2b8..a41a4a94 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2015 ZeroTier, Inc-> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -30,9 +30,9 @@ #include #include #include -#include #include #include +#include #include "../include/ZeroTierOne.h" #include "../node/Constants.hpp" @@ -1017,7 +1017,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy - if (network != origNetwork) { + if (true) { json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); network["lastModified"] = now; @@ -1235,8 +1235,9 @@ void EmbeddedNetworkController::_request( } // These are always the same, but make sure they are set - member["id"] = identity.address().toString(); - member["address"] = member["id"]; + const std::string addrs(identity.address().toString()); + member["id"] = addrs; + member["address"] = addrs; member["nwid"] = nwids; // Determine whether and how member is authorized @@ -1356,8 +1357,6 @@ void EmbeddedNetworkController::_request( // If we made it this far, they are authorized. // ------------------------------------------------------------------------- - NetworkConfig nc; - uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; if (now > ns.mostRecentDeauthTime) { // If we recently de-authorized a member, shrink credential TTL/max delta to @@ -1371,19 +1370,21 @@ void EmbeddedNetworkController::_request( } } - nc.networkId = nwid; - nc.type = OSUtils::jsonBool(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; - nc.timestamp = now; - nc.credentialTimeMaxDelta = credentialtmd; - nc.revision = OSUtils::jsonInt(network["revision"],0ULL); - nc.issuedTo = identity.address(); - if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (OSUtils::jsonBool(network["allowPassiveBridging"],false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),OSUtils::jsonString(network["name"],"").c_str()); - nc.multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); + std::auto_ptr nc(new NetworkConfig()); + + nc->networkId = nwid; + nc->type = OSUtils::jsonBool(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc->timestamp = now; + nc->credentialTimeMaxDelta = credentialtmd; + nc->revision = OSUtils::jsonInt(network["revision"],0ULL); + nc->issuedTo = identity.address(); + if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (OSUtils::jsonBool(network["allowPassiveBridging"],false)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc->name,sizeof(nc->name),OSUtils::jsonString(network["name"],"").c_str()); + nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) - nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + nc->addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); json &v4AssignMode = network["v4AssignMode"]; json &v6AssignMode = network["v6AssignMode"]; @@ -1399,15 +1400,15 @@ void EmbeddedNetworkController::_request( // Old versions with no rules engine support get an allow everything rule. // Since rules are enforced bidirectionally, newer versions *will* still // enforce rules on the inbound side. - nc.ruleCount = 1; - nc.rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; + nc->ruleCount = 1; + nc->rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; } else { if (rules.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + if (nc->ruleCount >= ZT_MAX_NETWORK_RULES) break; - if (_parseRule(rules[i],nc.rules[nc.ruleCount])) - ++nc.ruleCount; + if (_parseRule(rules[i],nc->rules[nc->ruleCount])) + ++nc->ruleCount; } } @@ -1451,10 +1452,10 @@ void EmbeddedNetworkController::_request( ++caprc; } } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) - ++nc.capabilityCount; - if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + nc->capabilities[nc->capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); + if (nc->capabilities[nc->capabilityCount].sign(_signingId,identity.address())) + ++nc->capabilityCount; + if (nc->capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) break; } } @@ -1485,17 +1486,17 @@ void EmbeddedNetworkController::_request( } } for(std::map< uint32_t,uint32_t >::const_iterator t(memberTagsById.begin());t!=memberTagsById.end();++t) { - if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + if (nc->tagCount >= ZT_MAX_NETWORK_TAGS) break; - nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(_signingId)) - ++nc.tagCount; + nc->tags[nc->tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); + if (nc->tags[nc->tagCount].sign(_signingId)) + ++nc->tagCount; } } if (routes.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) + if (nc->routeCount >= ZT_MAX_NETWORK_ROUTES) break; json &route = routes[i]; json &target = route["target"]; @@ -1505,11 +1506,11 @@ void EmbeddedNetworkController::_request( InetAddress v; if (via.is_string()) v.fromString(via.get()); if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + ZT_VirtualNetworkRoute *r = &(nc->routes[nc->routeCount]); *(reinterpret_cast(&(r->target))) = t; if (v.ss_family == t.ss_family) *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; + ++nc->routeCount; } } } @@ -1518,13 +1519,13 @@ void EmbeddedNetworkController::_request( const bool noAutoAssignIps = OSUtils::jsonBool(member["noAutoAssignIps"],false); if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { - if ((OSUtils::jsonBool(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + if ((OSUtils::jsonBool(v6AssignMode["rfc4193"],false))&&(nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc->staticIps[nc->staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); + nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } - if ((OSUtils::jsonBool(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + if ((OSUtils::jsonBool(v6AssignMode["6plane"],false))&&(nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc->staticIps[nc->staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); + nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } } @@ -1542,15 +1543,15 @@ void EmbeddedNetworkController::_request( // this route, ignoring the netmask bits field of the assigned IP itself. Using that was worthless and a source // of user error / poor UX. int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + for(unsigned int rk=0;rkrouteCount;++rk) { + if ( (!nc->routes[rk].via.ss_family) && (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip)) ) + routedNetmaskBits = reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); } if (routedNetmaskBits > 0) { - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { ip.setPort(routedNetmaskBits); - nc.staticIps[nc.staticIpCount++] = ip; + nc->staticIps[nc->staticIpCount++] = ip; } if (ip.ss_family == AF_INET) haveManagedIpv4AutoAssignment = true; @@ -1601,9 +1602,9 @@ void EmbeddedNetworkController::_request( // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + for(unsigned int rk=0;rkrouteCount;++rk) { + if ( (!nc->routes[rk].via.ss_family) && (nc->routes[rk].target.ss_family == AF_INET6) && (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip6)) ) + routedNetmaskBits = reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); } // If it's routed, then try to claim and assign it and if successful end loop @@ -1611,8 +1612,8 @@ void EmbeddedNetworkController::_request( ipAssignments.push_back(ip6.toIpString()); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) - nc.staticIps[nc.staticIpCount++] = ip6; + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) + nc->staticIps[nc->staticIpCount++] = ip6; haveManagedIpv6AutoAssignment = true; break; } @@ -1646,10 +1647,10 @@ void EmbeddedNetworkController::_request( // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = -1; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); + for(unsigned int rk=0;rkrouteCount;++rk) { + if (nc->routes[rk].target.ss_family == AF_INET) { + uint32_t targetIp = Utils::ntoh((uint32_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_addr.s_addr)); + int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_port)); if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { routedNetmaskBits = targetBits; break; @@ -1662,8 +1663,8 @@ void EmbeddedNetworkController::_request( if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip4)) ) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + struct sockaddr_in *const v4ip = reinterpret_cast(&(nc->staticIps[nc->staticIpCount++])); v4ip->sin_family = AF_INET; v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); v4ip->sin_addr.s_addr = Utils::hton(ip); @@ -1678,17 +1679,17 @@ void EmbeddedNetworkController::_request( } // Issue a certificate of ownership for all static IPs - if (nc.staticIpCount) { - nc.certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); - for(unsigned int i=0;istaticIpCount) { + nc->certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); + for(unsigned int i=0;istaticIpCount;++i) + nc->certificatesOfOwnership[0].addThing(nc->staticIps[i]); + nc->certificatesOfOwnership[0].sign(_signingId); + nc->certificateOfOwnershipCount = 1; } CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); if (com.sign(_signingId)) { - nc.com = com; + nc->com = com; } else { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR); return; @@ -1699,7 +1700,7 @@ void EmbeddedNetworkController::_request( _db.saveNetworkMember(nwid,identity.address().toInt(),member); } - _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); + _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member) @@ -1716,11 +1717,10 @@ void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,con online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); } if (online) { - Dictionary *metaData = new Dictionary(mdstr.c_str()); + Dictionary metaData(mdstr.c_str()); try { - this->request(nwid,InetAddress(),0,id,*metaData); + this->request(nwid,InetAddress(),0,id,metaData); } catch ( ... ) {} - delete metaData; } } } catch ( ... ) {} diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 4dada88e..8a220139 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -34,6 +34,7 @@ #include "../node/Utils.hpp" #include "../node/Address.hpp" #include "../node/InetAddress.hpp" +#include "../node/NonCopyable.hpp" #include "../osdep/OSUtils.hpp" #include "../osdep/Thread.hpp" @@ -50,7 +51,7 @@ namespace ZeroTier { class Node; -class EmbeddedNetworkController : public NetworkController +class EmbeddedNetworkController : public NetworkController,NonCopyable { public: /** diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index a2fe7f2a..01eb50cd 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -107,9 +107,64 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) } } +bool JSONDB::hasNetwork(const uint64_t networkId) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + return (i != _networks.end()); +} + +bool JSONDB::getNetwork(const uint64_t networkId,nlohmann::json &config) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + config = i->second.config; + return true; +} + +bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + ns = i->second.summaryInfo; + return true; +} + +int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return 0; + std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return 1; + networkConfig = i->second.config; + memberConfig = j->second; + ns = i->second.summaryInfo; + return 3; +} + +bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return false; + memberConfig = j->second; + return true; +} + void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) { - char n[256]; + char n[64]; Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig)); { diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index f89ff6c9..8db5cc48 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -63,63 +63,18 @@ public: bool writeRaw(const std::string &n,const std::string &obj); - inline bool hasNetwork(const uint64_t networkId) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - return (i != _networks.end()); - } + bool hasNetwork(const uint64_t networkId) const; - inline bool getNetwork(const uint64_t networkId,nlohmann::json &config) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return false; - config = i->second.config; - return true; - } + bool getNetwork(const uint64_t networkId,nlohmann::json &config) const; - inline bool getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return false; - ns = i->second.summaryInfo; - return true; - } + bool getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const; /** * @return Bit mask: 0 == none, 1 == network only, 3 == network and member */ - inline int getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return 0; - networkConfig = i->second.config; - ns = i->second.summaryInfo; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); - if (j == i->second.members.end()) - return 1; - memberConfig = j->second; - return 3; - } + int getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const; - inline bool getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return false; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); - if (j == i->second.members.end()) - return false; - memberConfig = j->second; - return true; - } + bool getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const; void saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig); diff --git a/ext/json/LICENSE.MIT b/ext/json/LICENSE.MIT index e2ac4891..00599afe 100644 --- a/ext/json/LICENSE.MIT +++ b/ext/json/LICENSE.MIT @@ -1,14 +1,13 @@ -The library is licensed under the MIT License -: +MIT License -Copyright (c) 2013-2016 Niels Lohmann +Copyright (c) 2013-2017 Niels Lohmann -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. diff --git a/ext/json/README.md b/ext/json/README.md index 4bcbe97f..fc6dde9b 100644 --- a/ext/json/README.md +++ b/ext/json/README.md @@ -3,13 +3,34 @@ [![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) [![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk/branch/develop?svg=true)](https://ci.appveyor.com/project/nlohmann/json) [![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) -[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/fsf5FqYe6GoX68W6) +[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/f3732b3327e34358a0e9d1fe9f661f08)](https://www.codacy.com/app/nlohmann/json?utm_source=github.com&utm_medium=referral&utm_content=nlohmann/json&utm_campaign=Badge_Grade) +[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/nv9fOg0XVVhWmFFy) [![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) [![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) [![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/289/badge)](https://bestpractices.coreinfrastructure.org/projects/289) +- [Design goals](#design-goals) +- [Integration](#integration) +- [Examples](#examples) + - [JSON as first-class data type](#json-as-first-class-data-type) + - [Serialization / Deserialization](#serialization--deserialization) + - [STL-like access](#stl-like-access) + - [Conversion from STL containers](#conversion-from-stl-containers) + - [JSON Pointer and JSON Patch](#json-pointer-and-json-patch) + - [Implicit conversions](#implicit-conversions) + - [Conversions to/from arbitrary types](#arbitrary-types-conversions) + - [Binary formats (CBOR and MessagePack)](#binary-formats-cbor-and-messagepack) +- [Supported compilers](#supported-compilers) +- [License](#license) +- [Thanks](#thanks) +- [Used third-party tools](#used-third-party-tools) +- [Projects using JSON for Modern C++](#projects-using-json-for-modern-c) +- [Notes](#notes) +- [Execute unit tests](#execute-unit-tests) + ## Design goals There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: @@ -24,7 +45,7 @@ Other aspects were not so important to us: - **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. -- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). +- **Speed**. There are certainly [faster JSON libraries](https://github.com/miloyip/nativejson-benchmark#parsing-time) out there. However, if your goal is to speed up your development by adding JSON support with a single header, then this library is the way to go. If you know how to use a `std::vector` or `std::map`, you are already set. See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. @@ -44,9 +65,15 @@ to the files you want to use JSON objects. That's it. Do not forget to set the n :beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. +:warning: [Version 3.0.0](https://github.com/nlohmann/json/wiki/Road-toward-3.0.0) is currently under development. Branch `develop` is used for the ongoing work and is probably **unstable**. Please use the `master` branch for the last stable version 2.1.1. + ## Examples +Beside the examples below, you may want to check the [documentation](https://nlohmann.github.io/json/) where each function contains a separate code example (e.g., check out [`emplace()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a602f275f0359ab181221384989810604.html#a602f275f0359ab181221384989810604)). All [example files](https://github.com/nlohmann/json/tree/develop/doc/examples) can be compiled and executed on their own (e.g., file [emplace.cpp](https://github.com/nlohmann/json/blob/develop/doc/examples/emplace.cpp)). + +### JSON as first-class data type + Here are some examples to give you an idea how to use the class. Assume you want to create the JSON object @@ -129,6 +156,8 @@ json array_not_object = { json::array({"currency", "USD"}), json::array({"value" ### Serialization / Deserialization +#### To/from strings + You can create an object (deserialization) by appending `_json` to a string literal: ```cpp @@ -142,8 +171,14 @@ auto j2 = R"( "pi": 3.141 } )"_json; +``` + +Note that without appending the `_json` suffix, the passed string literal is not parsed, but just used as JSON string value. That is, `json j = "{ \"happy\": true, \"pi\": 3.141 }"` would just store the string `"{ "happy": true, "pi": 3.141 }"` rather than parsing the actual object. -// or explicitly +The above example can also be expressed explicitly using `json::parse()`: + +```cpp +// parse explicitly auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); ``` @@ -162,6 +197,8 @@ std::cout << j.dump(4) << std::endl; // } ``` +#### To/from streams (e.g. files, string streams) + You can also use streams to serialize and deserialize: ```cpp @@ -176,10 +213,37 @@ std::cout << j; std::cout << std::setw(4) << j << std::endl; ``` -These operators work for any subclasses of `std::istream` or `std::ostream`. +These operators work for any subclasses of `std::istream` or `std::ostream`. Here is the same example with files: + +```cpp +// read a JSON file +std::ifstream i("file.json"); +json j; +i >> j; + +// write prettified JSON to another file +std::ofstream o("pretty.json"); +o << std::setw(4) << j << std::endl; +``` Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. +#### Read from iterator range + +You can also read JSON from an iterator range; that is, from any container accessible by iterators whose content is stored as contiguous byte sequence, for instance a `std::vector`: + +```cpp +std::vector v = {'t', 'r', 'u', 'e'}; +json j = json::parse(v.begin(), v.end()); +``` + +You may leave the iterators for the range [begin, end): + +```cpp +std::vector v = {'t', 'r', 'u', 'e'}; +json j = json::parse(v); +``` + ### STL-like access @@ -192,6 +256,9 @@ j.push_back("foo"); j.push_back(1); j.push_back(true); +// also use emplace_back +j.emplace_back(1.78); + // iterate the array for (json::iterator it = j.begin(); it != j.end(); ++it) { std::cout << *it << '\n'; @@ -207,6 +274,9 @@ const std::string tmp = j[0]; j[1] = 42; bool foo = j.at(2); +// comparison +j == "[\"foo\", 1, true]"_json; // true + // other stuff j.size(); // 3 entries j.empty(); // false @@ -221,15 +291,15 @@ j.is_object(); j.is_array(); j.is_string(); -// comparison -j == "[\"foo\", 1, true]"_json; // true - // create an object json o; o["foo"] = 23; o["bar"] = false; o["baz"] = 3.141; +// also use emplace +o.emplace("weather", "sunny"); + // special iterator member functions for objects for (json::iterator it = o.begin(); it != o.end(); ++it) { std::cout << it.key() << " : " << it.value() << "\n"; @@ -383,6 +453,252 @@ int vi = jn.get(); // etc. ``` +### Arbitrary types conversions + +Every type can be serialized in JSON, not just STL-containers and scalar types. Usually, you would do something along those lines: + +```cpp +namespace ns { + // a simple struct to model a person + struct person { + std::string name; + std::string address; + int age; + }; +} + +ns::person p = {"Ned Flanders", "744 Evergreen Terrace", 60}; + +// convert to JSON: copy each value into the JSON object +json j; +j["name"] = p.name; +j["address"] = p.address; +j["age"] = p.age; + +// ... + +// convert from JSON: copy each value from the JSON object +ns::person p { + j["name"].get(), + j["address"].get(), + j["age"].get() +}; +``` + +It works, but that's quite a lot of boilerplate... Fortunately, there's a better way: + +```cpp +// create a person +ns::person p {"Ned Flanders", "744 Evergreen Terrace", 60}; + +// conversion: person -> json +json j = p; + +std::cout << j << std::endl; +// {"address":"744 Evergreen Terrace","age":60,"name":"Ned Flanders"} + +// conversion: json -> person +ns::person p2 = j; + +// that's it +assert(p == p2); +``` + +#### Basic usage + +To make this work with one of your types, you only need to provide two functions: + +```cpp +using nlohmann::json; + +namespace ns { + void to_json(json& j, const person& p) { + j = json{{"name", p.name}, {"address", p.address}, {"age", p.age}}; + } + + void from_json(const json& j, person& p) { + p.name = j.at("name").get(); + p.address = j.at("address").get(); + p.age = j.at("age").get(); + } +} // namespace ns +``` + +That's all! When calling the `json` constructor with your type, your custom `to_json` method will be automatically called. +Likewise, when calling `get()`, the `from_json` method will be called. + +Some important things: + +* Those methods **MUST** be in your type's namespace (which can be the global namespace), or the library will not be able to locate them (in this example, they are in namespace `ns`, where `person` is defined). +* When using `get()`, `your_type` **MUST** be [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). (There is a way to bypass this requirement described later.) +* In function `from_json`, use function [`at()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a93403e803947b86f4da2d1fb3345cf2c.html#a93403e803947b86f4da2d1fb3345cf2c) to access the object values rather than `operator[]`. In case a key does not exists, `at` throws an exception that you can handle, whereas `operator[]` exhibits undefined behavior. + +#### How do I convert third-party types? + +This requires a bit more advanced technique. But first, let's see how this conversion mechanism works: + +The library uses **JSON Serializers** to convert types to json. +The default serializer for `nlohmann::json` is `nlohmann::adl_serializer` (ADL means [Argument-Dependent Lookup](http://en.cppreference.com/w/cpp/language/adl)). + +It is implemented like this (simplified): + +```cpp +template +struct adl_serializer { + static void to_json(json& j, const T& value) { + // calls the "to_json" method in T's namespace + } + + static void from_json(const json& j, T& value) { + // same thing, but with the "from_json" method + } +}; +``` + +This serializer works fine when you have control over the type's namespace. However, what about `boost::optional`, or `std::filesystem::path` (C++17)? Hijacking the `boost` namespace is pretty bad, and it's illegal to add something other than template specializations to `std`... + +To solve this, you need to add a specialization of `adl_serializer` to the `nlohmann` namespace, here's an example: + +```cpp +// partial specialization (full specialization works too) +namespace nlohmann { + template + struct adl_serializer> { + static void to_json(json& j, const boost::optional& opt) { + if (opt == boost::none) { + j = nullptr; + } else { + j = *opt; // this will call adl_serializer::to_json which will + // find the free function to_json in T's namespace! + } + } + + static void from_json(const json& j, boost::optional& opt) { + if (j.is_null()) { + opt = boost::none; + } else { + opt = j.get(); // same as above, but with + // adl_serializer::from_json + } + } + }; +} +``` + +#### How can I use `get()` for non-default constructible/non-copyable types? + +There is a way, if your type is [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible). You will need to specialize the `adl_serializer` as well, but with a special `from_json` overload: + +```cpp +struct move_only_type { + move_only_type() = delete; + move_only_type(int ii): i(ii) {} + move_only_type(const move_only_type&) = delete; + move_only_type(move_only_type&&) = default; + + int i; +}; + +namespace nlohmann { + template <> + struct adl_serializer { + // note: the return type is no longer 'void', and the method only takes + // one argument + static move_only_type from_json(const json& j) { + return {j.get()}; + } + + // Here's the catch! You must provide a to_json method! Otherwise you + // will not be able to convert move_only_type to json, since you fully + // specialized adl_serializer on that type + static void to_json(json& j, move_only_type t) { + j = t.i; + } + }; +} +``` + +#### Can I write my own serializer? (Advanced use) + +Yes. You might want to take a look at [`unit-udt.cpp`](https://github.com/nlohmann/json/blob/develop/test/src/unit-udt.cpp) in the test suite, to see a few examples. + +If you write your own serializer, you'll need to do a few things: + +* use a different `basic_json` alias than `nlohmann::json` (the last template parameter of `basic_json` is the `JSONSerializer`) +* use your `basic_json` alias (or a template parameter) in all your `to_json`/`from_json` methods +* use `nlohmann::to_json` and `nlohmann::from_json` when you need ADL + +Here is an example, without simplifications, that only accepts types with a size <= 32, and uses ADL. + +```cpp +// You should use void as a second template argument +// if you don't need compile-time checks on T +template::type> +struct less_than_32_serializer { + template + static void to_json(BasicJsonType& j, T value) { + // we want to use ADL, and call the correct to_json overload + using nlohmann::to_json; // this method is called by adl_serializer, + // this is where the magic happens + to_json(j, value); + } + + template + static void from_json(const BasicJsonType& j, T& value) { + // same thing here + using nlohmann::from_json; + from_json(j, value); + } +}; +``` + +Be **very** careful when reimplementing your serializer, you can stack overflow if you don't pay attention: + +```cpp +template +struct bad_serializer +{ + template + static void to_json(BasicJsonType& j, const T& value) { + // this calls BasicJsonType::json_serializer::to_json(j, value); + // if BasicJsonType::json_serializer == bad_serializer ... oops! + j = value; + } + + template + static void to_json(const BasicJsonType& j, T& value) { + // this calls BasicJsonType::json_serializer::from_json(j, value); + // if BasicJsonType::json_serializer == bad_serializer ... oops! + value = j.template get(); // oops! + } +}; +``` + +### Binary formats (CBOR and MessagePack) + +Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [CBOR](http://cbor.io) (Concise Binary Object Representation) and [MessagePack](http://msgpack.org) to efficiently encode JSON values to byte vectors and to decode such vectors. + +```cpp +// create a JSON value +json j = R"({"compact": true, "schema": 0})"_json; + +// serialize to CBOR +std::vector v_cbor = json::to_cbor(j); + +// 0xa2, 0x67, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xf5, 0x66, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 + +// roundtrip +json j_from_cbor = json::from_cbor(v_cbor); + +// serialize to MessagePack +std::vector v_msgpack = json::to_msgpack(j); + +// 0x82, 0xa7, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xc3, 0xa6, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 + +// roundtrip +json j_from_msgpack = json::from_msgpack(v_msgpack); +``` + ## Supported compilers @@ -391,6 +707,7 @@ Though it's 2016 already, the support for C++11 is still a bit sparse. Currently - GCC 4.9 - 6.0 (and possibly later) - Clang 3.4 - 3.9 (and possibly later) - Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) +- Microsoft Visual C++ 2017 / Build Tools 15.1.548.43366 (and possibly later) I would be happy to learn about other compilers/versions. @@ -423,16 +740,14 @@ The following compilers are currently used in continuous integration at [Travis] | Clang 3.7.1 | Ubuntu 14.04.4 LTS | clang version 3.7.1 (tags/RELEASE_371/final) | | Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | | Clang 3.8.1 | Ubuntu 14.04.4 LTS | clang version 3.8.1 (tags/RELEASE_381/final) | -| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | -| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | -| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | | Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | -| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | -| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | | Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | -| Clang Xcode 8.0 | Darwin Kernel Version 15.6.0 (OSX 10.11.6) | Apple LLVM version 8.0.0 (clang-800.0.38) | +| Clang Xcode 8.0 | Darwin Kernel Version 15.6.0 | Apple LLVM version 8.0.0 (clang-800.0.38) | +| Clang Xcode 8.1 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.2 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.3 | Darwin Kernel Version 16.5.0 (macOS 10.12.4) | Apple LLVM version 8.1.0 (clang-802.0.38) | | Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | - +| Visual Studio 2017 | Windows Server 2016 | Microsoft (R) Build Engine version 15.1.548.43366 | ## License @@ -440,7 +755,7 @@ The following compilers are currently used in continuous integration at [Travis] The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): -Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) +Copyright © 2013-2017 [Niels Lohmann](http://nlohmann.me) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: @@ -465,7 +780,7 @@ I deeply appreciate the help of the following people. - [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. - [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. - [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. -- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. +- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. In particular, he pushed forward the implementation of user-defined types. - [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. - [dariomt](https://github.com/dariomt) fixed some typos in the examples. - [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. @@ -493,14 +808,70 @@ I deeply appreciate the help of the following people. - [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. - [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. - [Thomas Braun](https://github.com/t-b) fixed a warning in a test case. -- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). +- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). He also implemented the magic behind the serialization/deserialization of user-defined types. - [Stefan](https://github.com/5tefan) fixed a minor issue in the documentation. - [Vasil Dimov](https://github.com/vasild) fixed the documentation regarding conversions from `std::multiset`. - [ChristophJud](https://github.com/ChristophJud) overworked the CMake files to ease project inclusion. -- [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable. +- [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable and added Visual Studio 17 to the build matrix. - [Denis Andrejew](https://github.com/seeekr) fixed a grammar issue in the README file. - -Thanks a lot for helping out! +- [Pierre-Antoine Lacaze](https://github.com/palacaze) found a subtle bug in the `dump()` function. +- [TurpentineDistillery](https://github.com/TurpentineDistillery) pointed to [`std::locale::classic()`](http://en.cppreference.com/w/cpp/locale/locale/classic) to avoid too much locale joggling, found some nice performance improvements in the parser, improved the benchmarking code, and realized locale-independent number parsing and printing. +- [cgzones](https://github.com/cgzones) had an idea how to fix the Coverity scan. +- [Jared Grubb](https://github.com/jaredgrubb) silenced a nasty documentation warning. +- [Yixin Zhang](https://github.com/qwename) fixed an integer overflow check. +- [Bosswestfalen](https://github.com/Bosswestfalen) merged two iterator classes into a smaller one. +- [Daniel599](https://github.com/Daniel599) helped to get Travis execute the tests with Clang's sanitizers. +- [Jonathan Lee](https://github.com/vjon) fixed an example in the README file. +- [gnzlbg](https://github.com/gnzlbg) supported the implementation of user-defined types. +- [Alexej Harm](https://github.com/qis) helped to get the user-defined types working with Visual Studio. +- [Jared Grubb](https://github.com/jaredgrubb) supported the implementation of user-defined types. +- [EnricoBilla](https://github.com/EnricoBilla) noted a typo in an example. +- [Martin Hořeňovský](https://github.com/horenmar) found a way for a 2x speedup for the compilation time of the test suite. +- [ukhegg](https://github.com/ukhegg) found proposed an improvement for the examples section. +- [rswanson-ihi](https://github.com/rswanson-ihi) noted a typo in the README. +- [Mihai Stan](https://github.com/stanmihai4) fixed a bug in the comparison with `nullptr`s. +- [Tushar Maheshwari](https://github.com/tusharpm) added [cotire](https://github.com/sakra/cotire) support to speed up the compilation. +- [TedLyngmo](https://github.com/TedLyngmo) noted a typo in the README, removed unnecessary bit arithmetic, and fixed some `-Weffc++` warnings. +- [Krzysztof Woś](https://github.com/krzysztofwos) made exceptions more visible. +- [ftillier](https://github.com/ftillier) fixed a compiler warning. +- [tinloaf](https://github.com/tinloaf) made sure all pushed warnings are properly popped. +- [Fytch](https://github.com/Fytch) found a bug in the documentation. + +Thanks a lot for helping out! Please [let me know](mailto:mail@nlohmann.me) if I forgot someone. + + +## Used third-party tools + +The library itself contains of a single header file licensed under the MIT license. However, it is built, tested, documented, and whatnot using a lot of third-party tools and services. Thanks a lot! + +- [**American fuzzy lop**](http://lcamtuf.coredump.cx/afl/) for fuzz testing +- [**AppVeyor**](https://www.appveyor.com) for [continuous integration](https://ci.appveyor.com/project/nlohmann/json) on Windows +- [**Artistic Style**](http://astyle.sourceforge.net) for automatic source code identation +- [**benchpress**](https://github.com/sbs-ableton/benchpress) to benchmark the code +- [**Catch**](https://github.com/philsquared/Catch) for the unit tests +- [**Clang**](http://clang.llvm.org) for compilation with code sanitizers +- [**Cmake**](https://cmake.org) for build automation +- [**Codacity**](https://www.codacy.com) for further [code analysis](https://www.codacy.com/app/nlohmann/json) +- [**cotire**](https://github.com/sakra/cotire) to speed of compilation +- [**Coveralls**](https://coveralls.io) to measure [code coverage](https://coveralls.io/github/nlohmann/json) +- [**Coverity Scan**](https://scan.coverity.com) for [static analysis](https://scan.coverity.com/projects/nlohmann-json) +- [**cppcheck**](http://cppcheck.sourceforge.net) for static analysis +- [**cxxopts**](https://github.com/jarro2783/cxxopts) to let benchpress parse command-line parameters +- [**Doxygen**](http://www.stack.nl/~dimitri/doxygen/) to generate [documentation](https://nlohmann.github.io/json/) +- [**git-update-ghpages**](https://github.com/rstacruz/git-update-ghpages) to upload the documentation to gh-pages +- [**Github Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md) +- [**libFuzzer**](http://llvm.org/docs/LibFuzzer.html) to implement fuzz testing for OSS-Fuzz +- [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library +- [**re2c**](http://re2c.org) to generate an automaton for the lexical analysis +- [**send_to_wandbox**](https://github.com/nlohmann/json/blob/develop/doc/scripts/send_to_wandbox.py) to send code examples to [Wandbox](http://melpon.org/wandbox) +- [**Travis**](https://travis-ci.org) for [continuous integration](https://travis-ci.org/nlohmann/json) on Linux and macOS +- [**Valgrind**](http://valgrind.org) to check for correct memory management +- [**Wandbox**](http://melpon.org/wandbox) for [online examples](http://melpon.org/wandbox/permlink/4NEU6ZZMoM9lpIex) + + +## Projects using JSON for Modern C++ + +The library is currently used in Apple macOS Sierra and iOS 10. I am not sure what they are using the library for, but I am happy that it runs on so many devices. ## Notes @@ -512,6 +883,10 @@ Thanks a lot for helping out! - Other encodings such as Latin-1, UTF-16, or UTF-32 are not supported and will yield parse errors. - [Unicode noncharacters](http://www.unicode.org/faq/private_use.html#nonchar1) will not be replaced by the library. - Invalid surrogates (e.g., incomplete pairs such as `\uDEAD`) will yield parse errors. + - The strings stored in the library are UTF-8 encoded. When using the default string type (`std::string`), note that its length/size functions return the number of stored bytes rather than the number of characters or glyphs. +- The code can be compiled without C++ **runtime type identification** features; that is, you can use the `-fno-rtti` compiler flag. +- **Exceptions** are used widely within the library. They can, however, be switched off with either using the compiler flag `-fno-exceptions` or by defining the symbol `JSON_NOEXCEPTION`. In this case, exceptions are replaced by an `abort()` call. +- By default, the library does not preserve the **insertion order of object elements**. This is standards-compliant, as the [JSON standard](https://tools.ietf.org/html/rfc7159.html) defines objects as "an unordered collection of zero or more name/value pairs". If you do want to preserve the insertion order, you can specialize the object type with containers like [`tsl::ordered_map`](https://github.com/Tessil/ordered-map) or [`nlohmann::fifo_map`](https://github.com/nlohmann/fifo_map). ## Execute unit tests @@ -519,10 +894,11 @@ Thanks a lot for helping out! To compile and run the tests, you need to execute ```sh -$ make check +$ make json_unit -Ctest +$ ./test/json_unit "*" =============================================================================== -All tests passed (8905491 assertions in 36 test cases) +All tests passed (11203022 assertions in 48 test cases) ``` Alternatively, you can use [CMake](https://cmake.org) and run diff --git a/ext/json/json.hpp b/ext/json/json.hpp index 9d48e7a6..8a8b876a 100644 --- a/ext/json/json.hpp +++ b/ext/json/json.hpp @@ -1,7 +1,7 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 2.0.10 +| | |__ | | | | | | version 2.1.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License . @@ -29,42 +29,39 @@ SOFTWARE. #ifndef NLOHMANN_JSON_HPP #define NLOHMANN_JSON_HPP -#include // all_of, for_each, transform +#include // all_of, copy, fill, find, for_each, none_of, remove, reverse, transform #include // array #include // assert -#include // isdigit #include // and, not, or -#include // isfinite, ldexp, signbit +#include // lconv, localeconv +#include // isfinite, labs, ldexp, signbit #include // nullptr_t, ptrdiff_t, size_t #include // int64_t, uint64_t -#include // strtod, strtof, strtold, strtoul +#include // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull #include // strlen +#include // forward_list #include // function, hash, less #include // initializer_list -#include // setw #include // istream, ostream -#include // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator +#include // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator #include // numeric_limits #include // locale #include // map #include // addressof, allocator, allocator_traits, unique_ptr #include // accumulate #include // stringstream -#include // domain_error, invalid_argument, out_of_range #include // getline, stoi, string, to_string -#include // add_pointer, enable_if, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_floating_point, is_integral, is_nothrow_move_assignable, std::is_nothrow_move_constructible, std::is_pointer, std::is_reference, std::is_same, remove_const, remove_pointer, remove_reference +#include // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type #include // declval, forward, make_pair, move, pair, swap #include // vector // exclude unsupported compilers #if defined(__clang__) - #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) - #if CLANG_VERSION < 30400 + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" #endif #elif defined(__GNUC__) - #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - #if GCC_VERSION < 40900 + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" #endif #endif @@ -90,6 +87,17 @@ SOFTWARE. #define JSON_DEPRECATED #endif +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) +#endif + /*! @brief namespace for Niels Lohmann @see https://github.com/nlohmann @@ -98,1466 +106,2175 @@ SOFTWARE. namespace nlohmann { - /*! @brief unnamed namespace with internal helper functions -@since version 1.0.0 + +This namespace collects some functions that could not be defined inside the +@ref basic_json class. + +@since version 2.1.0 */ -namespace +namespace detail { +//////////////// +// exceptions // +//////////////// + /*! -@brief Helper to determine whether there's a key_type for T. +@brief general exception of the @ref basic_json class -Thus helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. +Extension of std::exception objects with a member @a id for exception ids. -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0, overworked in version 2.0.6 +@note To have nothrow-copy-constructible exceptions, we internally use + std::runtime_error which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. + +@since version 3.0.0 */ -template -struct has_mapped_type +class exception : public std::exception { + public: + /// returns the explanatory string + virtual const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) + : id(id_), m(what_arg) + {} + + static std::string name(const std::string& ename, int id) + { + return "[json.exception." + ename + "." + std::to_string(id) + "] "; + } + private: - template - static int detect(U&&); + /// an exception object as storage for error messages + std::runtime_error m; +}; - static void detect(...); +/*! +@brief exception indicating a parse error + +This excpetion is thrown by the library when a parse error occurs. Parse +errors can occur during the deserialization of JSON text as well as when +using JSON Patch. + +Member @a byte holds the byte index of the last read character in the input +file. + +@note For an input with n bytes, 1 is the index of the first character + and n+1 is the index of the terminating null byte or the end of + file. This also holds true when reading a byte vector (CBOR or + MessagePack). + +Exceptions have ids 1xx. + +name / id | example massage | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number wihtout a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.111 | parse error: bad input stream | Parsing CBOR or MessagePack from an input stream where the [`badbit` or `failbit`](http://en.cppreference.com/w/cpp/io/ios_base/iostate) is set. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. + +@since version 3.0.0 +*/ +class parse_error : public exception +{ public: - static constexpr bool value = - std::is_integral()))>::value; -}; + /*! + @brief create a parse error exception + @param[in] id the id of the exception + @param[in] byte_ the byte index where the error occured (or 0 if + the position cannot be determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id, size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id) + "parse error" + + (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id, byte_, w.c_str()); + } -} + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character + and n+1 is the index of the terminating null byte or the end of + file. This also holds true when reading a byte vector (CBOR or + MessagePack). + */ + const size_t byte; + + private: + parse_error(int id_, size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) + {} +}; /*! -@brief a class to store JSON values +@brief exception indicating errors with iterators + +Exceptions have ids 2xx. + +name / id | example massage | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compated, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@since version 3.0.0 +*/ +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id) + what_arg; + return invalid_iterator(id, w.c_str()); + } -@tparam ObjectType type for JSON objects (`std::map` by default; will be used -in @ref object_t) -@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used -in @ref array_t) -@tparam StringType type for JSON strings and object keys (`std::string` by -default; will be used in @ref string_t) -@tparam BooleanType type for JSON booleans (`bool` by default; will be used -in @ref boolean_t) -@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by -default; will be used in @ref number_integer_t) -@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c -`uint64_t` by default; will be used in @ref number_unsigned_t) -@tparam NumberFloatType type for JSON floating-point numbers (`double` by -default; will be used in @ref number_float_t) -@tparam AllocatorType type of the allocator to use (`std::allocator` by -default) + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; -@requirement The class satisfies the following concept requirements: -- Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): - JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): - A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): - A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): - A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): - A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): - JSON values can be destructed. -- Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): - JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): - All non-static data members are private and standard layout types, the class - has no virtual functions or (virtual) base classes. -- Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): - JSON values can be compared with `==`, see @ref - operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): - JSON values can be compared with `<`, see @ref - operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): - Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of - other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): - JSON values can be compared against `std::nullptr_t` objects which are used - to model the `null` value. -- Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): - JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); - JSON values can be used like STL containers and provide reverse iterator - access. +/*! +@brief exception indicating executing a member function with a wrong type + +Exceptions have ids 3xx. + +name / id | example massage | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. + +@since version 3.0.0 +*/ +class type_error : public exception +{ + public: + static type_error create(int id, const std::string& what_arg) + { + std::string w = exception::name("type_error", id) + what_arg; + return type_error(id, w.c_str()); + } -@invariant The member variables @a m_value and @a m_type have the following -relationship: -- If `m_type == value_t::object`, then `m_value.object != nullptr`. -- If `m_type == value_t::array`, then `m_value.array != nullptr`. -- If `m_type == value_t::string`, then `m_value.string != nullptr`. -The invariants are checked by member function assert_invariant(). + private: + type_error(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; -@internal -@note ObjectType trick from http://stackoverflow.com/a/9860911 -@endinternal +/*! +@brief exception indicating access out of the defined range -@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange -Format](http://rfc7159.net/rfc7159) +Exceptions have ids 4xx. -@since version 1.0.0 +name / id | example massage | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -@nosubgrouping +@since version 3.0.0 */ -template < - template class ObjectType = std::map, - template class ArrayType = std::vector, - class StringType = std::string, - class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator - > -class basic_json +class out_of_range : public exception { + public: + static out_of_range create(int id, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id) + what_arg; + return out_of_range(id, w.c_str()); + } + private: - /// workaround type for MSVC - using basic_json_t = basic_json; + out_of_range(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; - public: - // forward declarations - template class iter_impl; - template class json_reverse_iterator; - class json_pointer; +/*! +@brief exception indicating other errors - ///////////////////// - // container types // - ///////////////////// +Exceptions have ids 5xx. - /// @name container types - /// The canonic container types to use @ref basic_json like any other STL - /// container. - /// @{ +name / id | example massage | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. - /// the type of elements in a basic_json container - using value_type = basic_json; +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id, const std::string& what_arg) + { + std::string w = exception::name("other_error", id) + what_arg; + return other_error(id, w.c_str()); + } - /// the type of an element reference - using reference = value_type&; - /// the type of an element const reference - using const_reference = const value_type&; + private: + other_error(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; - /// a type to represent differences between iterators - using difference_type = std::ptrdiff_t; - /// a type to represent container sizes - using size_type = std::size_t; - /// the allocator type - using allocator_type = AllocatorType; - /// the type of an element pointer - using pointer = typename std::allocator_traits::pointer; - /// the type of an element const pointer - using const_pointer = typename std::allocator_traits::const_pointer; +/////////////////////////// +// JSON type enumeration // +/////////////////////////// - /// an iterator for a basic_json container - using iterator = iter_impl; - /// a const iterator for a basic_json container - using const_iterator = iter_impl; - /// a reverse iterator for a basic_json container - using reverse_iterator = json_reverse_iterator; - /// a const reverse iterator for a basic_json container - using const_reverse_iterator = json_reverse_iterator; +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type - /// @} +@since version 1.0.0 +*/ +enum class value_t : uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function +}; +/*! +@brief comparison operator for JSON types - /*! - @brief returns the allocator associated with the container - */ - static allocator_type get_allocator() +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string +- furthermore, each type is not smaller than itself + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) { - return allocator_type(); + return false; } + return order[static_cast(lhs)] < + order[static_cast(rhs)]; +} - /////////////////////////// - // JSON value data types // - /////////////////////////// - /// @name JSON value data types - /// The data types to store a JSON value. These types are derived from - /// the template arguments passed to class @ref basic_json. - /// @{ +///////////// +// helpers // +///////////// - /*! - @brief a type for an object +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: - > An object is an unordered collection of zero or more name/value pairs, - > where a name is a string and a value is a string, number, boolean, null, - > object, or array. +template +using uncvref_t = typename std::remove_cv::type>::type; - To store objects in C++, a type is defined by the template parameters - described below. +/* +Implementation of two C++17 constructs: conjunction, negation. This is needed +to avoid evaluating all the traits in a condition - @tparam ObjectType the container to store objects (e.g., `std::map` or - `std::unordered_map`) - @tparam StringType the type of the keys or names (e.g., `std::string`). - The comparison function `std::less` is used to order elements - inside the container. - @tparam AllocatorType the allocator to use for objects (e.g., - `std::allocator`) +For example: not std::is_same::value and has_value_type::value +will not compile when T = void (on MSVC at least). Whereas +conjunction>, has_value_type>::value will +stop evaluating if negation<...>::value == false - #### Default type +Please note that those constructs must be used with caution, since symbols can +become very long quickly (which can slow down compilation and cause MSVC +internal compiler errors). Only use it when you have to (see example ahead). +*/ +template struct conjunction : std::true_type {}; +template struct conjunction : B1 {}; +template +struct conjunction : std::conditional, B1>::type {}; - With the default values for @a ObjectType (`std::map`), @a StringType - (`std::string`), and @a AllocatorType (`std::allocator`), the default - value for @a object_t is: +template struct negation : std::integral_constant < bool, !B::value > {}; - @code {.cpp} - std::map< - std::string, // key_type - basic_json, // value_type - std::less, // key_compare - std::allocator> // allocator_type - > - @endcode +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; - #### Behavior - The choice of @a object_t influences the behavior of the JSON class. With - the default type, objects have the following behavior: +////////////////// +// constructors // +////////////////// - - When all names are unique, objects will be interoperable in the sense - that all software implementations receiving that object will agree on - the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. - - Internally, name/value pairs are stored in lexicographical order of the - names. Objects will also be serialized (see @ref dump) in this order. - For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored - and serialized as `{"a": 2, "b": 1}`. - - When comparing objects, the order of the name/value pairs is irrelevant. - This makes objects interoperable in the sense that they will not be - affected by these differences. For instance, `{"b": 1, "a": 2}` and - `{"a": 2, "b": 1}` will be treated as equal. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the object's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON object. +template struct external_constructor; - #### Storage - - Objects are stored as pointers in a @ref basic_json type. That is, for any - access to object values, a pointer of type `object_t*` must be - dereferenced. +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; - @sa @ref array_t -- type for an array value +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } +}; - @since version 1.0.0 +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + j.m_type = value_t::number_float; + j.m_value = val; + j.assert_invariant(); + } +}; - @note The order name/value pairs are added to the object is *not* - preserved by the library. Therefore, iterating an object may return - name/value pairs in a different order than they were originally stored. In - fact, keys will be traversed in alphabetical order as `std::map` with - `std::less` is used by default. Please note this behavior conforms to [RFC - 7159](http://rfc7159.net/rfc7159), because any order implements the - specified "unordered" nature of JSON objects. - */ - using object_t = ObjectType, - AllocatorType>>; +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; - /*! - @brief a type for an array +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: - > An array is an ordered sequence of zero or more values. +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_type = value_t::array; + j.m_value = arr; + j.assert_invariant(); + } - To store objects in C++, a type is defined by the template parameters - explained below. + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + j.m_type = value_t::array; + j.m_value.array = j.template create(begin(arr), end(arr)); + j.assert_invariant(); + } - @tparam ArrayType container type to store arrays (e.g., `std::vector` or - `std::list`) - @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + template + static void construct(BasicJsonType& j, const std::vector& arr) + { + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->reserve(arr.size()); + for (bool x : arr) + { + j.m_value.array->push_back(x); + } + j.assert_invariant(); + } +}; - #### Default type +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_type = value_t::object; + j.m_value = obj; + j.assert_invariant(); + } - With the default values for @a ArrayType (`std::vector`) and @a - AllocatorType (`std::allocator`), the default value for @a array_t is: + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; - @code {.cpp} - std::vector< - basic_json, // value_type - std::allocator // allocator_type - > - @endcode + j.m_type = value_t::object; + j.m_value.object = j.template create(begin(obj), end(obj)); + j.assert_invariant(); + } +}; - #### Limits - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. +//////////////////////// +// has_/is_ functions // +//////////////////////// - In this class, the array's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON array. +/*! +@brief Helper to determine whether there's a key_type for T. - #### Storage +This helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. - Arrays are stored as pointers in a @ref basic_json type. That is, for any - access to array values, a pointer of type `array_t*` must be dereferenced. +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0, overworked in version 2.0.6 +*/ +#define NLOHMANN_JSON_HAS_HELPER(type) \ + template struct has_##type { \ + private: \ + template \ + static int detect(U &&); \ + static void detect(...); \ + public: \ + static constexpr bool value = \ + std::is_integral()))>::value; \ + } - @sa @ref object_t -- type for an object value +NLOHMANN_JSON_HAS_HELPER(mapped_type); +NLOHMANN_JSON_HAS_HELPER(key_type); +NLOHMANN_JSON_HAS_HELPER(value_type); +NLOHMANN_JSON_HAS_HELPER(iterator); - @since version 1.0.0 - */ - using array_t = ArrayType>; +#undef NLOHMANN_JSON_HAS_HELPER - /*! - @brief a type for a string - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: - > A string is a sequence of zero or more Unicode characters. +template +struct is_compatible_object_type_impl : std::false_type {}; - To store objects in C++, a type is defined by the template parameter - described below. Unicode values are split by the JSON class into - byte-sized characters during deserialization. +template +struct is_compatible_object_type_impl +{ + static constexpr auto value = + std::is_constructible::value and + std::is_constructible::value; +}; - @tparam StringType the container to store strings (e.g., `std::string`). - Note this container is used for keys/names in objects, see @ref object_t. +template +struct is_compatible_object_type +{ + static auto constexpr value = is_compatible_object_type_impl < + conjunction>, + has_mapped_type, + has_key_type>::value, + typename BasicJsonType::object_t, CompatibleObjectType >::value; +}; - #### Default type +template +struct is_basic_json_nested_type +{ + static auto constexpr value = std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value; +}; - With the default values for @a StringType (`std::string`), the default - value for @a string_t is: +template +struct is_compatible_array_type +{ + static auto constexpr value = + conjunction>, + negation>, + negation>, + negation>, + has_value_type, + has_iterator>::value; +}; - @code {.cpp} - std::string - @endcode +template +struct is_compatible_integer_type_impl : std::false_type {}; - #### String comparison +template +struct is_compatible_integer_type_impl +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + std::is_constructible::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; - [RFC 7159](http://rfc7159.net/rfc7159) states: - > Software implementations are typically required to test names of object - > members for equality. Implementations that transform the textual - > representation into sequences of Unicode code units and then perform the - > comparison numerically, code unit by code unit, are interoperable in the - > sense that implementations will agree in all cases on equality or - > inequality of two strings. For example, implementations that compare - > strings with escaped characters unconverted may incorrectly find that - > `"a\\b"` and `"a\u005Cb"` are not equal. +template +struct is_compatible_integer_type +{ + static constexpr auto value = + is_compatible_integer_type_impl < + std::is_integral::value and + not std::is_same::value, + RealIntegerType, CompatibleNumberIntegerType > ::value; +}; - This implementation is interoperable as it does compare strings code unit - by code unit. - #### Storage +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json +{ + private: + // also check the return type of from_json + template::from_json( + std::declval(), std::declval()))>::value>> + static int detect(U&&); + static void detect(...); - String values are stored as pointers in a @ref basic_json type. That is, - for any access to string values, a pointer of type `string_t*` must be - dereferenced. + public: + static constexpr bool value = std::is_integral>()))>::value; +}; - @since version 1.0.0 - */ - using string_t = StringType; +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json +{ + private: + template < + typename U, + typename = enable_if_t::from_json(std::declval()))>::value >> + static int detect(U&&); + static void detect(...); - /*! - @brief a type for a boolean + public: + static constexpr bool value = std::is_integral>()))>::value; +}; - [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a - type which differentiates the two literals `true` and `false`. +// This trait checks if BasicJsonType::json_serializer::to_json exists +template +struct has_to_json +{ + private: + template::to_json( + std::declval(), std::declval()))> + static int detect(U&&); + static void detect(...); - To store objects in C++, a type is defined by the template parameter @a - BooleanType which chooses the type to use. + public: + static constexpr bool value = std::is_integral>()))>::value; +}; - #### Default type - With the default values for @a BooleanType (`bool`), the default value for - @a boolean_t is: +///////////// +// to_json // +///////////// - @code {.cpp} - bool - @endcode +template::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor::construct(j, b); +} - #### Storage +template::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor::construct(j, s); +} - Boolean values are stored directly inside a @ref basic_json type. +template::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} - @since version 1.0.0 - */ - using boolean_t = BooleanType; +template < + typename BasicJsonType, typename CompatibleNumberUnsignedType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} - /*! - @brief a type for a number (integer) +template < + typename BasicJsonType, typename CompatibleNumberIntegerType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. +template::value, int> = 0> +void to_json(BasicJsonType& j, EnumType e) noexcept +{ + using underlying_type = typename std::underlying_type::type; + external_constructor::construct(j, static_cast(e)); +} - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. +template +void to_json(BasicJsonType& j, const std::vector& e) +{ + external_constructor::construct(j, e); +} - To store integer numbers in C++, a type is defined by the template - parameter @a NumberIntegerType which chooses the type to use. +template < + typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < + is_compatible_array_type::value or + std::is_same::value, + int > = 0 > +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor::construct(j, arr); +} - #### Default type +template < + typename BasicJsonType, typename CompatibleObjectType, + enable_if_t::value, + int> = 0 > +void to_json(BasicJsonType& j, const CompatibleObjectType& arr) +{ + external_constructor::construct(j, arr); +} - With the default values for @a NumberIntegerType (`int64_t`), the default - value for @a number_integer_t is: +template ::value, + int> = 0> +void to_json(BasicJsonType& j, T (&arr)[N]) +{ + external_constructor::construct(j, arr); +} - @code {.cpp} - int64_t - @endcode +/////////////// +// from_json // +/////////////// + +// overloads for basic_json template parameters +template::value and + not std::is_same::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast( + *j.template get_ptr()); + break; + } + default: + { + JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name())); + } + } +} - #### Default behavior +template +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (not j.is_boolean()) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + j.type_name())); + } + b = *j.template get_ptr(); +} - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (not j.is_string()) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + j.type_name())); + } + s = *j.template get_ptr(); +} - #### Limits +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} - When the default type is used, the maximal integer number that can be - stored is `9223372036854775807` (INT64_MAX) and the minimal integer number - that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers - that are out of range will yield over/underflow when used in a - constructor. During deserialization, too large or small integer numbers - will be automatically be stored as @ref number_unsigned_t or @ref - number_float_t. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. +template::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type::type val; + get_arithmetic_value(j, val); + e = static_cast(val); +} - As this range is a subrange of the exactly supported range [INT64_MIN, - INT64_MAX], this class's integer type is interoperable. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } + arr = *j.template get_ptr(); +} - #### Storage +// forward_list doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list& l) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } - Integer number values are stored directly inside a @ref basic_json type. + for (auto it = j.rbegin(), end = j.rend(); it != end; ++it) + { + l.push_front(it->template get()); + } +} - @sa @ref number_float_t -- type for number values (floating-point) +template +void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0>) +{ + using std::begin; + using std::end; - @sa @ref number_unsigned_t -- type for number values (unsigned integer) + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} - @since version 1.0.0 - */ - using number_integer_t = NumberIntegerType; +template +auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1>) +-> decltype( + arr.reserve(std::declval()), + void()) +{ + using std::begin; + using std::end; - /*! - @brief a type for a number (unsigned) + arr.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. +template::value and + std::is_convertible::value and + not std::is_same::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleArrayType& arr) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. + from_json_array_impl(j, arr, priority_tag<1> {}); +} - To store unsigned integer numbers in C++, a type is defined by the - template parameter @a NumberUnsignedType which chooses the type to use. +template::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleObjectType& obj) +{ + if (not j.is_object()) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + j.type_name())); + } - #### Default type + auto inner_object = j.template get_ptr(); + using std::begin; + using std::end; + // we could avoid the assignment, but this might require a for loop, which + // might be less efficient than the container constructor for some + // containers (would it?) + obj = CompatibleObjectType(begin(*inner_object), end(*inner_object)); +} - With the default values for @a NumberUnsignedType (`uint64_t`), the - default value for @a number_unsigned_t is: +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::boolean: + { + val = static_cast(*j.template get_ptr()); + break; + } + default: + { + JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name())); + } + } +} - @code {.cpp} - uint64_t - @endcode +struct to_json_fn +{ + private: + template + auto call(BasicJsonType& j, T&& val, priority_tag<1>) const noexcept(noexcept(to_json(j, std::forward(val)))) + -> decltype(to_json(j, std::forward(val)), void()) + { + return to_json(j, std::forward(val)); + } - #### Default behavior + template + void call(BasicJsonType&, T&&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find to_json() method in T's namespace"); + } - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. + public: + template + void operator()(BasicJsonType& j, T&& val) const + noexcept(noexcept(std::declval().call(j, std::forward(val), priority_tag<1> {}))) + { + return call(j, std::forward(val), priority_tag<1> {}); + } +}; - #### Limits +struct from_json_fn +{ + private: + template + auto call(const BasicJsonType& j, T& val, priority_tag<1>) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); + } - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. + template + void call(const BasicJsonType&, T&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find from_json() method in T's namespace"); + } - When the default type is used, the maximal integer number that can be - stored is `18446744073709551615` (UINT64_MAX) and the minimal integer - number that can be stored is `0`. Integer numbers that are out of range - will yield over/underflow when used in a constructor. During - deserialization, too large or small integer numbers will be automatically - be stored as @ref number_integer_t or @ref number_float_t. + public: + template + void operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(std::declval().call(j, val, priority_tag<1> {}))) + { + return call(j, val, priority_tag<1> {}); + } +}; - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; - As this range is a subrange (when considered in conjunction with the - number_integer_t type) of the exactly supported range [0, UINT64_MAX], - this class's integer type is interoperable. +template +constexpr T static_const::value; +} // namespace detail - #### Storage - Integer number values are stored directly inside a @ref basic_json type. +/// namespace to hold default `to_json` / `from_json` functions +namespace +{ +constexpr const auto& to_json = detail::static_const::value; +constexpr const auto& from_json = detail::static_const::value; +} - @sa @ref number_float_t -- type for number values (floating-point) - @sa @ref number_integer_t -- type for number values (integer) - @since version 2.0.0 - */ - using number_unsigned_t = NumberUnsignedType; +/*! +@brief default JSONSerializer template argument +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer +{ /*! - @brief a type for a number (floating-point) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store floating-point numbers in C++, a type is defined by the template - parameter @a NumberFloatType which chooses the type to use. - - #### Default type - - With the default values for @a NumberFloatType (`double`), the default - value for @a number_float_t is: - - @code {.cpp} - double - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in floating-point literals will be ignored. Internally, - the value will be stored as decimal number. For instance, the C++ - floating-point literal `01.2` will be serialized to `1.2`. During - deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > This specification allows implementations to set limits on the range and - > precision of numbers accepted. Since software that implements IEEE - > 754-2008 binary64 (double precision) numbers is generally available and - > widely used, good interoperability can be achieved by implementations - > that expect no more precision or range than these provide, in the sense - > that implementations will approximate JSON numbers within the expected - > precision. - - This implementation does exactly follow this approach, as it uses double - precision floating-point numbers. Note values smaller than - `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` - will be stored as NaN internally and be serialized to `null`. - - #### Storage - - Floating-point number values are stored directly inside a @ref basic_json - type. - - @sa @ref number_integer_t -- type for number values (integer) + @brief convert a JSON value to any value type - @sa @ref number_unsigned_t -- type for number values (unsigned integer) + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). - @since version 1.0.0 + @param[in] j JSON value to read from + @param[in,out] val value to write to */ - using number_float_t = NumberFloatType; - - /// @} - - - /////////////////////////// - // JSON type enumeration // - /////////////////////////// + template + static void from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + { + ::nlohmann::from_json(std::forward(j), val); + } /*! - @brief the JSON type enumeration + @brief convert any value type to a JSON value - This enumeration collects the different JSON types. It is internally used - to distinguish the stored values, and the functions @ref is_null(), @ref - is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref - is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and - @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and - @ref is_structured() rely on it. + This function is usually called by the constructors of the @ref basic_json + class. - @note There are three enumeration entries (number_integer, - number_unsigned, and number_float), because the library distinguishes - these three types for numbers: @ref number_unsigned_t is used for unsigned - integers, @ref number_integer_t is used for signed integers, and @ref - number_float_t is used for floating-point numbers or to approximate - integers which do not fit in the limits of their respective type. + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template + static void to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; - @sa @ref basic_json(const value_t value_type) -- create a JSON value with - the default value for a given type - @since version 1.0.0 - */ - enum class value_t : uint8_t - { - null, ///< null value - object, ///< object (unordered set of name/value pairs) - array, ///< array (ordered collection of values) - string, ///< string value - boolean, ///< boolean value - number_integer, ///< number value (signed integer) - number_unsigned, ///< number value (unsigned integer) - number_float, ///< number value (floating-point) - discarded ///< discarded by the the parser callback function - }; +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) +@tparam JSONSerializer the serializer to resolve internal calls to `to_json()` +and `from_json()` (@ref adl_serializer by default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null + value. + - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + JSON values have + [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the + class has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](http://en.cppreference.com/w/cpp/concept/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 +@nosubgrouping +*/ +template < + template class ObjectType = std::map, + template class ArrayType = std::vector, + class StringType = std::string, + class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = adl_serializer + > +class basic_json +{ private: + template friend struct detail::external_constructor; + /// workaround type for MSVC + using basic_json_t = basic_json; - /// helper for exception-safe object creation - template - static T* create(Args&& ... args) + public: + using value_t = detail::value_t; + // forward declarations + template class iter_impl; + template class json_reverse_iterator; + class json_pointer; + template + using json_serializer = JSONSerializer; + + + //////////////// + // exceptions // + //////////////// + + /// @name exceptions + /// Classes to implement user-defined exceptions. + /// @{ + + /// @copydoc detail::exception + using exception = detail::exception; + /// @copydoc detail::parse_error + using parse_error = detail::parse_error; + /// @copydoc detail::invalid_iterator + using invalid_iterator = detail::invalid_iterator; + /// @copydoc detail::type_error + using type_error = detail::type_error; + /// @copydoc detail::out_of_range + using out_of_range = detail::out_of_range; + /// @copydoc detail::other_error + using other_error = detail::other_error; + + /// @} + + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + using iterator = iter_impl; + /// a const iterator for a basic_json container + using const_iterator = iter_impl; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() { - AllocatorType alloc; - auto deleter = [&](T * object) - { - alloc.deallocate(object, 1); - }; - std::unique_ptr object(alloc.allocate(1), deleter); - alloc.construct(object.get(), std::forward(args)...); - assert(object.get() != nullptr); - return object.release(); + return allocator_type(); } - //////////////////////// - // JSON value storage // - //////////////////////// - /*! - @brief a JSON value + @brief returns version information on the library - The actual storage for a JSON value of the @ref basic_json class. This - union combines the different storage types for the JSON value types - defined in @ref value_t. + This function returns a JSON object with information about the library, + including the version number and information on the platform and compiler. - JSON type | value_t type | used type - --------- | --------------- | ------------------------ - object | object | pointer to @ref object_t - array | array | pointer to @ref array_t - string | string | pointer to @ref string_t - boolean | boolean | @ref boolean_t - number | number_integer | @ref number_integer_t - number | number_unsigned | @ref number_unsigned_t - number | number_float | @ref number_float_t - null | null | *no value is stored* + @return JSON object holding version information + key | description + ----------- | --------------- + `compiler` | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). + `copyright` | The copyright line for the library as string. + `name` | The name of the library as string. + `platform` | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`. + `url` | The URL of the project as string. + `version` | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string). - @note Variable-length types (objects, arrays, and strings) are stored as - pointers. The size of the union should not exceed 64 bits if the default - value types are used. + @liveexample{The following code shows an example output of the `meta()` + function.,meta} - @since version 1.0.0 + @complexity Constant. + + @since 2.1.0 */ - union json_value + static basic_json meta() { - /// object (stored with pointer to save storage) - object_t* object; - /// array (stored with pointer to save storage) - array_t* array; - /// string (stored with pointer to save storage) - string_t* string; - /// boolean - boolean_t boolean; - /// number (integer) - number_integer_t number_integer; - /// number (unsigned integer) - number_unsigned_t number_unsigned; - /// number (floating-point) - number_float_t number_float; + basic_json result; - /// default constructor (for null values) - json_value() = default; - /// constructor for booleans - json_value(boolean_t v) noexcept : boolean(v) {} - /// constructor for numbers (integer) - json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} - /// constructor for numbers (floating-point) - json_value(number_float_t v) noexcept : number_float(v) {} - /// constructor for empty values of a given type - json_value(value_t t) + result["copyright"] = "(C) 2013-2017 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"] = { - switch (t) - { - case value_t::object: - { - object = create(); - break; - } + {"string", "2.1.1"}, {"major", 2}, {"minor", 1}, {"patch", 1} + }; - case value_t::array: - { - array = create(); - break; - } +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif - case value_t::string: - { - string = create(""); - break; - } +#if defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif - case value_t::boolean: - { - boolean = boolean_t(false); - break; - } +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } - case value_t::number_integer: - { - number_integer = number_integer_t(0); - break; - } - case value_t::number_unsigned: - { - number_unsigned = number_unsigned_t(0); - break; - } + /////////////////////////// + // JSON value data types // + /////////////////////////// - case value_t::number_float: - { - number_float = number_float_t(0.0); - break; - } + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ - case value_t::null: - { - break; - } + /*! + @brief a type for an object - default: - { - if (t == value_t::null) - { - throw std::domain_error("961c151d2e87f2686a955a9be24d316f1362bf21 2.0.10"); // LCOV_EXCL_LINE - } - break; - } - } - } + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. - /// constructor for strings - json_value(const string_t& value) - { - string = create(value); - } + To store objects in C++, a type is defined by the template parameters + described below. - /// constructor for objects - json_value(const object_t& value) - { - object = create(value); - } + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) - /// constructor for arrays - json_value(const array_t& value) - { - array = create(value); - } - }; + #### Default type - /*! - @brief checks the class invariants + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: - This function asserts the class invariants. It needs to be called at the - end of every constructor to make sure that created objects respect the - invariant. Furthermore, it has to be called each time the type of a JSON - value is changed, because the invariant expresses a relationship between - @a m_type and @a m_value. - */ - void assert_invariant() const - { - assert(m_type != value_t::object or m_value.object != nullptr); - assert(m_type != value_t::array or m_value.array != nullptr); - assert(m_type != value_t::string or m_value.string != nullptr); - } + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less, // key_compare + std::allocator> // allocator_type + > + @endcode - public: - ////////////////////////// - // JSON parser callback // - ////////////////////////// + #### Behavior - /*! - @brief JSON callback events + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: - This enumeration lists the parser events that can trigger calling a - callback function of type @ref parser_callback_t during parsing. + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, later stored name/value + pairs overwrite previously stored name/value pairs, leaving the used + names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will + be treated as equal and both stored as `{"key": 1}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. - @image html callback_events.png "Example when certain parse events are triggered" + #### Limits - @since version 1.0.0 - */ - enum class parse_event_t : uint8_t - { - /// the parser read `{` and started to process a JSON object - object_start, - /// the parser read `}` and finished processing a JSON object - object_end, - /// the parser read `[` and started to process a JSON array - array_start, - /// the parser read `]` and finished processing a JSON array - array_end, - /// the parser read a key of a value in an object - key, - /// the parser finished reading a JSON value - value - }; + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType, + AllocatorType>>; /*! - @brief per-element parser callback type + @brief a type for an array - With a parser callback function, the result of parsing a JSON text can be - influenced. When passed to @ref parse(std::istream&, const - parser_callback_t) or @ref parse(const CharT, const parser_callback_t), - it is called on certain events (passed as @ref parse_event_t via parameter - @a event) with a set recursion depth @a depth and context JSON value - @a parsed. The return value of the callback function is a boolean - indicating whether the element that emitted the callback shall be kept or - not. + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. - We distinguish six scenarios (determined by the event type) in which the - callback function can be called. The following table describes the values - of the parameters @a depth, @a event, and @a parsed. + To store objects in C++, a type is defined by the template parameters + explained below. - parameter @a event | description | parameter @a depth | parameter @a parsed - ------------------ | ----------- | ------------------ | ------------------- - parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded - parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key - parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object - parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded - parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array - parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) - @image html callback_events.png "Example when certain parse events are triggered" + #### Default type - Discarding a value (i.e., returning `false`) has different effects - depending on the context in which function was called: + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: - - Discarded values in structured types are skipped. That is, the parser - will behave as if the discarded value was never read. - - In case a value outside a structured type is skipped, it is replaced - with `null`. This case happens if the top-level element is skipped. + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator // allocator_type + > + @endcode - @param[in] depth the depth of the recursion during parsing + #### Limits - @param[in] event an event of type parse_event_t indicating the context in - the callback function has been called + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. - @param[in,out] parsed the current intermediate parse result; note that - writing to this value has no effect for parse_event_t::key events + In this class, the array's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. - @return Whether the JSON value which called the function during parsing - should be kept (`true`) or not (`false`). In the latter case, it is either - skipped completely or replaced by an empty discarded object. + #### Storage - @sa @ref parse(std::istream&, parser_callback_t) or - @ref parse(const CharT, const parser_callback_t) for examples + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value @since version 1.0.0 */ - using parser_callback_t = std::function; + using array_t = ArrayType>; + /*! + @brief a type for a string - ////////////////// - // constructors // - ////////////////// + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. - /// @name constructors and destructors - /// Constructors of class @ref basic_json, copy/move constructor, copy - /// assignment, static functions creating objects, and the destructor. - /// @{ + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. - /*! - @brief create an empty value with a given type + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. - Create an empty JSON value with a given type. The value will be default - initialized with an empty value which depends on the type: + #### Default type - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: - @param[in] value_type the type of the value to create + @code {.cpp} + std::string + @endcode - @complexity Constant. + #### Encoding - @throw std::bad_alloc if allocation for object, array, or string value - fails + Strings are stored in UTF-8 encoding. Therefore, functions like + `std::string::size()` or `std::string::length()` return the number of + bytes in the string rather than the number of characters or glyphs. - @liveexample{The following code shows the constructor for different @ref - value_t values,basic_json__value_t} + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - @sa @ref basic_json(boolean_t value) -- create a boolean value - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const object_t&) -- create a object value - @sa @ref basic_json(const array_t&) -- create a array value - @sa @ref basic_json(const number_float_t) -- create a number - (floating-point) value - @sa @ref basic_json(const number_integer_t) -- create a number (integer) - value - @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) - value + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. @since version 1.0.0 */ - basic_json(const value_t value_type) - : m_type(value_type), m_value(value_type) - { - assert_invariant(); - } + using string_t = StringType; /*! - @brief create a null object + @brief a type for a boolean - Create a `null` JSON value. It either takes a null pointer as parameter - (explicitly creating `null`) or no parameter (implicitly creating `null`). - The passed null pointer itself is not read -- it is only used to choose - the right constructor. + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. - @complexity Constant. + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. + #### Default type - @liveexample{The following code shows the constructor with and without a - null pointer parameter.,basic_json__nullptr_t} + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. @since version 1.0.0 */ - basic_json(std::nullptr_t = nullptr) noexcept - : basic_json(value_t::null) - { - assert_invariant(); - } + using boolean_t = BooleanType; /*! - @brief create an object (explicit) + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. - Create an object JSON value with a given content. + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. - @param[in] val a value for the object + #### Default type - @complexity Linear in the size of the passed @a val. + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: - @throw std::bad_alloc if allocation for object value fails + @code {.cpp} + int64_t + @endcode - @liveexample{The following code shows the constructor with an @ref - object_t parameter.,basic_json__object_t} + #### Default behavior - @sa @ref basic_json(const CompatibleObjectType&) -- create an object value - from a compatible STL container + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. - @since version 1.0.0 - */ - basic_json(const object_t& val) - : m_type(value_t::object), m_value(val) - { - assert_invariant(); - } + #### Limits - /*! - @brief create an object (implicit) + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. - Create an object JSON value with a given content. This constructor allows - any type @a CompatibleObjectType that can be used to construct values of - type @ref object_t. + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. - @tparam CompatibleObjectType An object type whose `key_type` and - `value_type` is compatible to @ref object_t. Examples include `std::map`, - `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with - a `key_type` of `std::string`, and a `value_type` from which a @ref - basic_json value can be constructed. + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. - @param[in] val a value for the object + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. - @complexity Linear in the size of the passed @a val. + #### Storage - @throw std::bad_alloc if allocation for object value fails + Integer number values are stored directly inside a @ref basic_json type. - @liveexample{The following code shows the constructor with several - compatible object type parameters.,basic_json__CompatibleObjectType} + @sa @ref number_float_t -- type for number values (floating-point) - @sa @ref basic_json(const object_t&) -- create an object value + @sa @ref number_unsigned_t -- type for number values (unsigned integer) @since version 1.0.0 */ - template::value and - std::is_constructible::value, int>::type = 0> - basic_json(const CompatibleObjectType& val) - : m_type(value_t::object) - { - using std::begin; - using std::end; - m_value.object = create(begin(val), end(val)); - assert_invariant(); - } + using number_integer_t = NumberIntegerType; /*! - @brief create an array (explicit) + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. - Create an array JSON value with a given content. + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. - @param[in] val a value for the array + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. - @complexity Linear in the size of the passed @a val. + #### Default type - @throw std::bad_alloc if allocation for array value fails + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: - @liveexample{The following code shows the constructor with an @ref array_t - parameter.,basic_json__array_t} + @code {.cpp} + uint64_t + @endcode - @sa @ref basic_json(const CompatibleArrayType&) -- create an array value - from a compatible STL containers + #### Default behavior - @since version 1.0.0 - */ - basic_json(const array_t& val) - : m_type(value_t::array), m_value(val) - { - assert_invariant(); - } + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. - /*! - @brief create an array (implicit) + #### Limits - Create an array JSON value with a given content. This constructor allows - any type @a CompatibleArrayType that can be used to construct values of - type @ref array_t. + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. - @tparam CompatibleArrayType An object type whose `value_type` is - compatible to @ref array_t. Examples include `std::vector`, `std::deque`, - `std::list`, `std::forward_list`, `std::array`, `std::set`, - `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a - `value_type` from which a @ref basic_json value can be constructed. + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. - @param[in] val a value for the array + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. - @complexity Linear in the size of the passed @a val. + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. - @throw std::bad_alloc if allocation for array value fails + #### Storage - @liveexample{The following code shows the constructor with several - compatible array type parameters.,basic_json__CompatibleArrayType} + Integer number values are stored directly inside a @ref basic_json type. - @sa @ref basic_json(const array_t&) -- create an array value + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) - @since version 1.0.0 + @since version 2.0.0 */ - template::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type = 0> - basic_json(const CompatibleArrayType& val) - : m_type(value_t::array) - { - using std::begin; - using std::end; - m_value.array = create(begin(val), end(val)); - assert_invariant(); - } + using number_unsigned_t = NumberUnsignedType; /*! - @brief create a string (explicit) + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. - Create an string JSON value with a given content. + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. - @param[in] val a value for the string + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. - @complexity Linear in the size of the passed @a val. + #### Default type - @throw std::bad_alloc if allocation for string value fails + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: - @liveexample{The following code shows the constructor with an @ref - string_t parameter.,basic_json__string_t} + @code {.cpp} + double + @endcode - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container + #### Default behavior - @since version 1.0.0 - */ - basic_json(const string_t& val) - : m_type(value_t::string), m_value(val) - { - assert_invariant(); - } + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. - /*! - @brief create a string (explicit) + #### Limits - Create a string JSON value with a given content. + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. - @param[in] val a literal value for the string + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. - @complexity Linear in the size of the passed @a val. + #### Storage - @throw std::bad_alloc if allocation for string value fails + Floating-point number values are stored directly inside a @ref basic_json + type. - @liveexample{The following code shows the constructor with string literal - parameter.,basic_json__string_t_value_type} + @sa @ref number_integer_t -- type for number values (integer) - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container + @sa @ref number_unsigned_t -- type for number values (unsigned integer) @since version 1.0.0 */ - basic_json(const typename string_t::value_type* val) - : basic_json(string_t(val)) - { - assert_invariant(); - } + using number_float_t = NumberFloatType; - /*! - @brief create a string (implicit) + /// @} - Create a string JSON value with a given content. + private: - @param[in] val a value for the string + /// helper for exception-safe object creation + template + static T* create(Args&& ... args) + { + AllocatorType alloc; + auto deleter = [&](T * object) + { + alloc.deallocate(object, 1); + }; + std::unique_ptr object(alloc.allocate(1), deleter); + alloc.construct(object.get(), std::forward(args)...); + assert(object != nullptr); + return object.release(); + } - @tparam CompatibleStringType an string type which is compatible to @ref - string_t, for instance `std::string`. + //////////////////////// + // JSON value storage // + //////////////////////// - @complexity Linear in the size of the passed @a val. + /*! + @brief a JSON value - @throw std::bad_alloc if allocation for string value fails + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. - @liveexample{The following code shows the construction of a string value - from a compatible type.,basic_json__CompatibleStringType} + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. @since version 1.0.0 */ - template::value, int>::type = 0> - basic_json(const CompatibleStringType& val) - : basic_json(string_t(val)) + union json_value { - assert_invariant(); - } + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; - /*! - @brief create a boolean (explicit) + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } - Creates a JSON boolean type from a given value. + case value_t::array: + { + array = create(); + break; + } - @param[in] val a boolean value to store + case value_t::string: + { + string = create(""); + break; + } - @complexity Constant. + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } - @liveexample{The example below demonstrates boolean - values.,basic_json__boolean_t} + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } - @since version 1.0.0 - */ - basic_json(boolean_t val) noexcept - : m_type(value_t::boolean), m_value(val) - { - assert_invariant(); - } + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } - /*! - @brief create an integer number (explicit) + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } - Create an integer number JSON value with a given content. + case value_t::null: + { + break; + } - @tparam T A helper type to remove this function via SFINAE in case @ref - number_integer_t is the same as `int`. In this case, this constructor - would have the same signature as @ref basic_json(const int value). Note - the helper type @a T is not visible in this constructor's interface. + default: + { + if (t == value_t::null) + { + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE + } + break; + } + } + } - @param[in] val an integer to create a JSON number from + /// constructor for strings + json_value(const string_t& value) + { + string = create(value); + } - @complexity Constant. + /// constructor for objects + json_value(const object_t& value) + { + object = create(value); + } - @liveexample{The example below shows the construction of an integer - number value.,basic_json__number_integer_t} + /// constructor for arrays + json_value(const array_t& value) + { + array = create(value); + } + }; - @sa @ref basic_json(const int) -- create a number value (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type + /*! + @brief checks the class invariants - @since version 1.0.0 + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. */ - template::value) and - std::is_same::value, int>::type = 0> - basic_json(const number_integer_t val) noexcept - : m_type(value_t::number_integer), m_value(val) + void assert_invariant() const { - assert_invariant(); + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); } - /*! - @brief create an integer number from an enum type (explicit) - - Create an integer number JSON value with a given content. - - @param[in] val an integer to create a JSON number from - - @note This constructor allows to pass enums directly to a constructor. As - C++ has no way of specifying the type of an anonymous enum explicitly, we - can only rely on the fact that such values implicitly convert to int. As - int may already be the same type of number_integer_t, we may need to - switch off the constructor @ref basic_json(const number_integer_t). + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// - @complexity Constant. + /*! + @brief JSON callback events - @liveexample{The example below shows the construction of an integer - number value from an anonymous enum.,basic_json__const_int} + This enumeration lists the parser events that can trigger calling a + callback function of type @ref parser_callback_t during parsing. - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type + @image html callback_events.png "Example when certain parse events are triggered" @since version 1.0.0 */ - basic_json(const int val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) + enum class parse_event_t : uint8_t { - assert_invariant(); - } + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; /*! - @brief create an integer number (implicit) + @brief per-element parser callback type - Create an integer number JSON value with a given content. This constructor - allows any type @a CompatibleNumberIntegerType that can be used to - construct values of type @ref number_integer_t. + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse(std::istream&, const + parser_callback_t) or @ref parse(const CharT, const parser_callback_t), + it is called on certain events (passed as @ref parse_event_t via parameter + @a event) with a set recursion depth @a depth and context JSON value + @a parsed. The return value of the callback function is a boolean + indicating whether the element that emitted the callback shall be kept or + not. - @tparam CompatibleNumberIntegerType An integer type which is compatible to - @ref number_integer_t. Examples include the types `int`, `int32_t`, - `long`, and `short`. + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. - @param[in] val an integer to create a JSON number from + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value - @complexity Constant. + @image html callback_events.png "Example when certain parse events are triggered" - @liveexample{The example below shows the construction of several integer - number values from compatible - types.,basic_json__CompatibleIntegerNumberType} + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const int) -- create a number value (integer) + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. - @since version 1.0.0 - */ - template::value and - std::numeric_limits::is_integer and - std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type = 0> - basic_json(const CompatibleNumberIntegerType val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } + @param[in] depth the depth of the recursion during parsing - /*! - @brief create an unsigned integer number (explicit) + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called - Create an unsigned integer number JSON value with a given content. + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events - @tparam T helper type to compare number_unsigned_t and unsigned int (not - visible in) the interface. + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. - @param[in] val an integer to create a JSON number from + @sa @ref parse(std::istream&, parser_callback_t) or + @ref parse(const CharT, const parser_callback_t) for examples - @complexity Constant. + @since version 1.0.0 + */ + using parser_callback_t = std::function; - @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number - value (unsigned integer) from a compatible number type - @since version 2.0.0 - */ - template::value) and - std::is_same::value, int>::type = 0> - basic_json(const number_unsigned_t val) noexcept - : m_type(value_t::number_unsigned), m_value(val) - { - assert_invariant(); - } + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ /*! - @brief create an unsigned number (implicit) + @brief create an empty value with a given type - Create an unsigned number JSON value with a given content. This - constructor allows any type @a CompatibleNumberUnsignedType that can be - used to construct values of type @ref number_unsigned_t. + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: - @tparam CompatibleNumberUnsignedType An integer type which is compatible - to @ref number_unsigned_t. Examples may include the types `unsigned int`, - `uint32_t`, or `unsigned short`. + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` - @param[in] val an unsigned integer to create a JSON number from + @param[in] value_type the type of the value to create @complexity Constant. - @sa @ref basic_json(const number_unsigned_t) -- create a number value - (unsigned) + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} - @since version 2.0.0 + @since version 1.0.0 */ - template::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type = 0> - basic_json(const CompatibleNumberUnsignedType val) noexcept - : m_type(value_t::number_unsigned), - m_value(static_cast(val)) + basic_json(const value_t value_type) + : m_type(value_type), m_value(value_type) { assert_invariant(); } /*! - @brief create a floating-point number (explicit) - - Create a floating-point number JSON value with a given content. - - @param[in] val a floating-point value to create a JSON number from + @brief create a null object - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is created - instead. + Create a `null` JSON value. It either takes a null pointer as parameter + (explicitly creating `null`) or no parameter (implicitly creating `null`). + The passed null pointer itself is not read -- it is only used to choose + the right constructor. @complexity Constant. - @liveexample{The following example creates several floating-point - values.,basic_json__number_float_t} + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. - @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number - value (floating-point) from a compatible number type + @liveexample{The following code shows the constructor with and without a + null pointer parameter.,basic_json__nullptr_t} @since version 1.0.0 */ - basic_json(const number_float_t val) noexcept - : m_type(value_t::number_float), m_value(val) + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) { - // replace infinity and NAN by null - if (not std::isfinite(val)) - { - m_type = value_t::null; - m_value = json_value(); - } - assert_invariant(); } /*! - @brief create an floating-point number (implicit) + @brief create a JSON value - Create an floating-point number JSON value with a given content. This - constructor allows any type @a CompatibleNumberFloatType that can be used - to construct values of type @ref number_float_t. + This is a "catch all" constructor for all compatible JSON types; that is, + types for which a `to_json()` method exsits. The constructor forwards the + parameter @a val to that method (to `json_serializer::to_json` method + with `U = uncvref_t`, to be exact). - @tparam CompatibleNumberFloatType A floating-point type which is - compatible to @ref number_float_t. Examples may include the types `float` - or `double`. + Template type @a CompatibleType includes, but is not limited to, the + following types: + - **arrays**: @ref array_t and all kinds of compatible containers such as + `std::vector`, `std::deque`, `std::list`, `std::forward_list`, + `std::array`, `std::set`, `std::unordered_set`, `std::multiset`, and + `unordered_multiset` with a `value_type` from which a @ref basic_json + value can be constructed. + - **objects**: @ref object_t and all kinds of compatible associative + containers such as `std::map`, `std::unordered_map`, `std::multimap`, + and `std::unordered_multimap` with a `key_type` compatible to + @ref string_t and a `value_type` from which a @ref basic_json value can + be constructed. + - **strings**: @ref string_t, string literals, and all compatible string + containers can be used. + - **numbers**: @ref number_integer_t, @ref number_unsigned_t, + @ref number_float_t, and all convertible number types such as `int`, + `size_t`, `int64_t`, `float` or `double` can be used. + - **boolean**: @ref boolean_t / `bool` can be used. - @param[in] val a floating-point to create a JSON number from + See the examples below. - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is - created instead. + @tparam CompatibleType a type such that: + - @a CompatibleType is not derived from `std::istream`, + - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move + constructors), + - @a CompatibleType is not a @ref basic_json nested type (e.g., + @ref json_pointer, @ref iterator, etc ...) + - @ref @ref json_serializer has a + `to_json(basic_json_t&, CompatibleType&&)` method - @complexity Constant. + @tparam U = `uncvref_t` - @liveexample{The example below shows the construction of several - floating-point number values from compatible - types.,basic_json__CompatibleNumberFloatType} + @param[in] val the value to be forwarded - @sa @ref basic_json(const number_float_t) -- create a number value - (floating-point) + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. - @since version 1.0.0 + @throw what `json_serializer::to_json()` throws + + @liveexample{The following code shows the constructor with several + compatible types.,basic_json__CompatibleType} + + @since version 2.1.0 */ - template::value and - std::is_floating_point::value>::type> - basic_json(const CompatibleNumberFloatType val) noexcept - : basic_json(number_float_t(val)) + template, + detail::enable_if_t::value and + not std::is_same::value and + not detail::is_basic_json_nested_type< + basic_json_t, U>::value and + detail::has_to_json::value, + int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer::to_json( + std::declval(), std::forward(val)))) { + JSONSerializer::to_json(*this, std::forward(val)); assert_invariant(); } @@ -1613,10 +2330,12 @@ class basic_json value_t::array and @ref value_t::object are valid); when @a type_deduction is set to `true`, this parameter has no effect - @throw std::domain_error if @a type_deduction is `false`, @a manual_type - is `value_t::object`, but @a init contains an element which is not a pair - whose first element is a string; example: `"cannot create object from - initializer list"` + @throw type_error.301 if @a type_deduction is `false`, @a manual_type is + `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string. In this case, the constructor could not + create an object. If @a type_deduction would have be `true`, an array + would have been created. See @ref object(std::initializer_list) + for an example. @complexity Linear in the size of the initializer list @a init. @@ -1654,7 +2373,7 @@ class basic_json // if object is wanted but impossible, throw an exception if (manual_type == value_t::object and not is_an_object) { - throw std::domain_error("cannot create object from initializer list"); + JSON_THROW(type_error::create(301, "cannot create object from initializer list")); } } @@ -1730,16 +2449,17 @@ class basic_json related function @ref array(std::initializer_list), there are no cases which can only be expressed by this function. That is, any initializer list @a init can also be passed to the initializer list - constructor @ref basic_json(std::initializer_list, bool, - value_t). + constructor @ref basic_json(std::initializer_list, bool, value_t). @param[in] init initializer list to create an object from (optional) @return JSON object value - @throw std::domain_error if @a init is not a pair whose first elements are - strings; thrown by - @ref basic_json(std::initializer_list, bool, value_t) + @throw type_error.301 if @a init is not a list of pairs whose first + elements are strings. In this case, no object can be created. When such a + value is passed to @ref basic_json(std::initializer_list, bool, value_t), + an array would have been created from the passed initializer list @a init. + See example below. @complexity Linear in the size of @a init. @@ -1791,10 +2511,10 @@ class basic_json The semantics depends on the different types a JSON value can have: - In case of primitive types (number, boolean, or string), @a first must be `begin()` and @a last must be `end()`. In this case, the value is - copied. Otherwise, std::out_of_range is thrown. + copied. Otherwise, invalid_iterator.204 is thrown. - In case of structured types (array, object), the constructor behaves as similar versions for `std::vector`. - - In case of a null type, std::domain_error is thrown. + - In case of a null type, invalid_iterator.206 is thrown. @tparam InputIT an input iterator type (@ref iterator or @ref const_iterator) @@ -1805,14 +2525,19 @@ class basic_json @pre Iterators @a first and @a last must be initialized. **This precondition is enforced with an assertion.** - @throw std::domain_error if iterators are not compatible; that is, do not - belong to the same JSON value; example: `"iterators are not compatible"` - @throw std::out_of_range if iterators are for a primitive type (number, - boolean, or string) where an out of range error can be detected easily; - example: `"iterators out of range"` - @throw std::bad_alloc if allocation for object, array, or string fails - @throw std::domain_error if called with a null value; example: `"cannot - use construct with iterators from null"` + @pre Range `[first, last)` is valid. Usually, this precondition cannot be + checked efficiently. Only certain edge cases are detected; see the + description of the exceptions below. + + @throw invalid_iterator.201 if iterators @a first and @a last are not + compatible (i.e., do not belong to the same JSON value). In this case, + the range `[first, last)` is undefined. + @throw invalid_iterator.204 if iterators @a first and @a last belong to a + primitive type (number, boolean, or string), but @a first does not point + to the first element any more. In this case, the range `[first, last)` is + undefined. See example code below. + @throw invalid_iterator.206 if iterators @a first and @a last belong to a + null value. In this case, the range `[first, last)` is undefined. @complexity Linear in distance between @a first and @a last. @@ -1832,7 +2557,7 @@ class basic_json // make sure iterator fits the current value if (first.m_object != last.m_object) { - throw std::domain_error("iterators are not compatible"); + JSON_THROW(invalid_iterator::create(201, "iterators are not compatible")); } // copy type from first iterator @@ -1849,7 +2574,7 @@ class basic_json { if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) { - throw std::out_of_range("iterators out of range"); + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); } break; } @@ -1894,59 +2619,28 @@ class basic_json case value_t::object: { - m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); + m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); break; } case value_t::array: { - m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); + m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); break; } default: { - throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); + JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " + + first.m_object->type_name())); } } assert_invariant(); } - /*! - @brief construct a JSON value given an input stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @deprecated This constructor is deprecated and will be removed in version - 3.0.0 to unify the interface of the library. Deserialization will be - done by stream operators or by calling one of the `parse` functions, - e.g. @ref parse(std::istream&, const parser_callback_t). That is, calls - like `json j(i);` for an input stream @a i need to be replaced by - `json j = json::parse(i);`. See the example below. - - @liveexample{The example below demonstrates constructing a JSON value from - a `std::stringstream` with and without callback - function.,basic_json__istream} - - @since version 2.0.0, deprecated in version 2.0.3, to be removed in - version 3.0.0 - */ - JSON_DEPRECATED - explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) - { - *this = parser(i, cb).parse(); - assert_invariant(); - } /////////////////////////////////////// // other constructors and destructor // @@ -1967,8 +2661,6 @@ class basic_json - The complexity is linear. - As postcondition, it holds: `other == basic_json(other)`. - @throw std::bad_alloc if allocation for object, array, or string fails. - @liveexample{The following code shows an example for the copy constructor.,basic_json__basic_json} @@ -2196,22 +2888,15 @@ class basic_json string_t dump(const int indent = -1) const { std::stringstream ss; - // fix locale problems - ss.imbue(std::locale::classic()); - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - ss.precision(std::numeric_limits::digits10); + serializer s(ss); if (indent >= 0) { - dump(ss, true, static_cast(indent)); + s.dump(*this, true, static_cast(indent)); } else { - dump(ss, false, 0); + s.dump(*this, false, 0); } return ss.str(); @@ -2577,246 +3262,99 @@ class basic_json private: ////////////////// // value access // - ////////////////// - - /// get an object (explicit) - template::value and - std::is_convertible::value, int>::type = 0> - T get_impl(T*) const - { - if (is_object()) - { - return T(m_value.object->begin(), m_value.object->end()); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an object (explicit) - object_t get_impl(object_t*) const - { - if (is_object()) - { - return *(m_value.object); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value, int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - T to_vector; - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not std::is_same::value, int>::type = 0> - std::vector get_impl(std::vector*) const - { - if (is_array()) - { - std::vector to_vector; - to_vector.reserve(m_value.array->size()); - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not has_mapped_type::value, int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - return T(m_value.array->begin(), m_value.array->end()); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - array_t get_impl(array_t*) const - { - if (is_array()) - { - return *(m_value.array); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get a string (explicit) - template::value, int>::type = 0> - T get_impl(T*) const - { - if (is_string()) - { - return *m_value.string; - } - else - { - throw std::domain_error("type must be string, but is " + type_name()); - } - } - - /// get a number (explicit) - template::value, int>::type = 0> - T get_impl(T*) const - { - switch (m_type) - { - case value_t::number_integer: - { - return static_cast(m_value.number_integer); - } - - case value_t::number_unsigned: - { - return static_cast(m_value.number_unsigned); - } - - case value_t::number_float: - { - return static_cast(m_value.number_float); - } - - default: - { - throw std::domain_error("type must be number, but is " + type_name()); - } - } - } + ////////////////// /// get a boolean (explicit) - constexpr boolean_t get_impl(boolean_t*) const + boolean_t get_impl(boolean_t* /*unused*/) const { - return is_boolean() - ? m_value.boolean - : throw std::domain_error("type must be boolean, but is " + type_name()); + if (is_boolean()) + { + return m_value.boolean; + } + + JSON_THROW(type_error::create(302, "type must be boolean, but is " + type_name())); } /// get a pointer to the value (object) - object_t* get_impl_ptr(object_t*) noexcept + object_t* get_impl_ptr(object_t* /*unused*/) noexcept { return is_object() ? m_value.object : nullptr; } /// get a pointer to the value (object) - constexpr const object_t* get_impl_ptr(const object_t*) const noexcept + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept { return is_object() ? m_value.object : nullptr; } /// get a pointer to the value (array) - array_t* get_impl_ptr(array_t*) noexcept + array_t* get_impl_ptr(array_t* /*unused*/) noexcept { return is_array() ? m_value.array : nullptr; } /// get a pointer to the value (array) - constexpr const array_t* get_impl_ptr(const array_t*) const noexcept + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept { return is_array() ? m_value.array : nullptr; } /// get a pointer to the value (string) - string_t* get_impl_ptr(string_t*) noexcept + string_t* get_impl_ptr(string_t* /*unused*/) noexcept { return is_string() ? m_value.string : nullptr; } /// get a pointer to the value (string) - constexpr const string_t* get_impl_ptr(const string_t*) const noexcept + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept { return is_string() ? m_value.string : nullptr; } /// get a pointer to the value (boolean) - boolean_t* get_impl_ptr(boolean_t*) noexcept + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept { return is_boolean() ? &m_value.boolean : nullptr; } /// get a pointer to the value (boolean) - constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept { return is_boolean() ? &m_value.boolean : nullptr; } /// get a pointer to the value (integer number) - number_integer_t* get_impl_ptr(number_integer_t*) noexcept + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept { return is_number_integer() ? &m_value.number_integer : nullptr; } /// get a pointer to the value (integer number) - constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept { return is_number_integer() ? &m_value.number_integer : nullptr; } /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept { return is_number_unsigned() ? &m_value.number_unsigned : nullptr; } /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept { return is_number_unsigned() ? &m_value.number_unsigned : nullptr; } /// get a pointer to the value (floating-point number) - number_float_t* get_impl_ptr(number_float_t*) noexcept + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept { return is_number_float() ? &m_value.number_float : nullptr; } /// get a pointer to the value (floating-point number) - constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept { return is_number_float() ? &m_value.number_float : nullptr; } @@ -2829,7 +3367,7 @@ class basic_json @tparam ThisType will be deduced as `basic_json` or `const basic_json` - @throw std::domain_error if ReferenceType does not match underlying value + @throw type_error.303 if ReferenceType does not match underlying value type of the current JSON */ template @@ -2845,34 +3383,68 @@ class basic_json { return *ptr; } - else - { - throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + - obj.type_name()); - } + + JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + obj.type_name())); } public: - /// @name value access /// Direct access to the stored value of a JSON value. /// @{ + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template < + typename BasicJsonType, + detail::enable_if_t::type, + basic_json_t>::value, + int> = 0 > + basic_json get() const + { + return *this; + } + /*! @brief get a value (explicit) - Explicit type conversion between the JSON value and a compatible value. + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + @endcode - @return copy of the JSON value, converted to type @a ValueType + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer has a `from_json()` method of the form + `void from_json(const @ref basic_json&, ValueType&)`, and + - @ref json_serializer does not have a `from_json()` method of + the form `ValueType from_json(const @ref basic_json&)` - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON; example: `"type must be object, but is null"` + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type - @complexity Linear in the size of the JSON value. + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws @liveexample{The example below shows several conversions from JSON values to other types. There a few things to note: (1) Floating-point numbers can @@ -2881,21 +3453,75 @@ class basic_json associative containers such as `std::unordered_map`.,get__ValueType_const} - @internal - The idea of using a casted null pointer to choose the correct - implementation is from . - @endinternal + @since version 2.1.0 + */ + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t < + not std::is_same::value and + detail::has_from_json::value and + not detail::has_non_default_from_json::value, + int > = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), std::declval()))) + { + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get(), which is why we + // still need the uncvref + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + static_assert(std::is_default_constructible::value, + "types must be DefaultConstructible when used with get()"); - @sa @ref operator ValueType() const for implicit conversion - @sa @ref get() for pointer-member access + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + } - @since version 1.0.0 + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer has a `from_json()` method of the form + `ValueType from_json(const @ref basic_json&)` + + @note If @ref json_serializer has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @since version 2.1.0 */ - template::value, int>::type = 0> - ValueType get() const + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t::value and + detail::has_non_default_from_json::value, int> = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval()))) { - return get_impl(static_cast(nullptr)); + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return JSONSerializer::from_json(*this); } /*! @@ -3025,7 +3651,7 @@ class basic_json /*! @brief get a reference value (implicit) - Implict reference access to the internally stored JSON value. No copies + Implicit reference access to the internally stored JSON value. No copies are made. @warning Writing data to the referee of the result yields an undefined @@ -3037,10 +3663,10 @@ class basic_json @return reference to the internally stored JSON value if the requested reference type @a ReferenceType fits to the JSON value; throws - std::domain_error otherwise + type_error.303 otherwise - @throw std::domain_error in case passed type @a ReferenceType is - incompatible with the stored JSON value + @throw type_error.303 in case passed type @a ReferenceType is incompatible + with the stored JSON value; see example below @complexity Constant. @@ -3083,8 +3709,9 @@ class basic_json @return copy of the JSON value, converted to type @a ValueType - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON, thrown by @ref get() const + @throw type_error.302 in case passed type @a ValueType is incompatible + to the JSON value type (e.g., the JSON value is of type boolean, but a + string is requested); see example below @complexity Linear in the size of the JSON value. @@ -3100,8 +3727,11 @@ class basic_json template < typename ValueType, typename std::enable_if < not std::is_pointer::value and not std::is_same::value -#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 +#ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same>::value +#endif +#if defined(_MSC_VER) && _MSC_VER >1900 && defined(_HAS_CXX17) && _HAS_CXX17 == 1 // fix for issue #464 + and not std::is_same::value #endif , int >::type = 0 > operator ValueType() const @@ -3131,36 +3761,40 @@ class basic_json @return reference to the element at index @a idx - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. - @complexity Constant. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how array elements can be read and - written using `at()`.,at__size_type} + @complexity Constant. @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__size_type} */ reference at(size_type idx) { // at only works for arrays if (is_array()) { - try + JSON_TRY { return m_value.array->at(idx); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3174,36 +3808,40 @@ class basic_json @return const reference to the element at index @a idx - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. - @complexity Constant. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how array elements can be read using - `at()`.,at__size_type_const} + @complexity Constant. @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__size_type_const} */ const_reference at(size_type idx) const { // at only works for arrays if (is_array()) { - try + JSON_TRY { return m_value.array->at(idx); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3217,40 +3855,44 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. - @complexity Logarithmic in the size of the container. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how object elements can be read and - written using `at()`.,at__object_t_key_type} + @complexity Logarithmic in the size of the container. @sa @ref operator[](const typename object_t::key_type&) for unchecked access by reference @sa @ref value() for access by value with a default value @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__object_t_key_type} */ reference at(const typename object_t::key_type& key) { // at only works for objects if (is_object()) { - try + JSON_TRY { return m_value.object->at(key); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3264,40 +3906,44 @@ class basic_json @return const reference to the element at key @a key - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. - @complexity Logarithmic in the size of the container. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how object elements can be read using - `at()`.,at__object_t_key_type_const} + @complexity Logarithmic in the size of the container. @sa @ref operator[](const typename object_t::key_type&) for unchecked access by reference @sa @ref value() for access by value with a default value @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__object_t_key_type_const} */ const_reference at(const typename object_t::key_type& key) const { // at only works for objects if (is_object()) { - try + JSON_TRY { return m_value.object->at(key); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3314,8 +3960,8 @@ class basic_json @return reference to the element at index @a idx - @throw std::domain_error if JSON is not an array or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an array or null; in that + cases, using the [] operator with an index makes no sense. @complexity Constant if @a idx is in the range of the array. Otherwise linear in `idx - size()`. @@ -3349,10 +3995,8 @@ class basic_json return m_value.array->operator[](idx); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3364,8 +4008,8 @@ class basic_json @return const reference to the element at index @a idx - @throw std::domain_error if JSON is not an array; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an array; in that cases, + using the [] operator with an index makes no sense. @complexity Constant. @@ -3381,10 +4025,8 @@ class basic_json { return m_value.array->operator[](idx); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3400,8 +4042,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3429,10 +4071,8 @@ class basic_json { return m_value.object->operator[](key); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3451,8 +4091,8 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3473,10 +4113,8 @@ class basic_json assert(m_value.object->find(key) != m_value.object->end()); return m_value.object->find(key)->second; } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3492,8 +4130,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3527,8 +4165,8 @@ class basic_json @return const reference to the element at key @a key - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3560,8 +4198,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3590,10 +4228,8 @@ class basic_json { return m_value.object->operator[](key); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3612,8 +4248,8 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3635,10 +4271,8 @@ class basic_json assert(m_value.object->find(key) != m_value.object->end()); return m_value.object->find(key)->second; } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3651,7 +4285,7 @@ class basic_json @code {.cpp} try { return at(key); - } catch(std::out_of_range) { + } catch(out_of_range) { return default_value; } @endcode @@ -3674,8 +4308,8 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` + @throw type_error.306 if the JSON value is not an objec; in that cases, + using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3702,14 +4336,12 @@ class basic_json { return *it; } - else - { - return default_value; - } + + return default_value; } else { - throw std::domain_error("cannot use value() with " + type_name()); + JSON_THROW(type_error::create(306, "cannot use value() with " + type_name())); } } @@ -3732,7 +4364,7 @@ class basic_json @code {.cpp} try { return at(ptr); - } catch(std::out_of_range) { + } catch(out_of_range) { return default_value; } @endcode @@ -3751,8 +4383,8 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` + @throw type_error.306 if the JSON value is not an objec; in that cases, + using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3771,19 +4403,17 @@ class basic_json if (is_object()) { // if pointer resolves a value, return it or use default value - try + JSON_TRY { return ptr.get_checked(this); } - catch (std::out_of_range&) + JSON_CATCH (out_of_range&) { return default_value; } } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } + + JSON_THROW(type_error::create(306, "cannot use value() with " + type_name())); } /*! @@ -3812,7 +4442,7 @@ class basic_json assertions**). @post The JSON value remains unchanged. - @throw std::out_of_range when called on `null` value + @throw invalid_iterator.214 when called on `null` value @liveexample{The following code shows an example for `front()`.,front} @@ -3855,7 +4485,8 @@ class basic_json assertions**). @post The JSON value remains unchanged. - @throw std::out_of_range when called on `null` value. + @throw invalid_iterator.214 when called on a `null` value. See example + below. @liveexample{The following code shows an example for `back()`.,back} @@ -3899,17 +4530,18 @@ class basic_json @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on an iterator which does not belong to - the current JSON value; example: `"iterator does not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.202 if called on an iterator which does not belong + to the current JSON value; example: `"iterator does not fit current + value"` + @throw invalid_iterator.205 if called on a primitive type with invalid iterator (i.e., any iterator which is not `begin()`); example: `"iterator out of range"` @complexity The complexity depends on the type: - objects: amortized constant - - arrays: linear in distance between pos and the end of the container + - arrays: linear in distance between @a pos and the end of the container - strings: linear in the length of the string - other types: constant @@ -3934,7 +4566,7 @@ class basic_json // make sure iterator fits the current value if (this != pos.m_object) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } IteratorType result = end(); @@ -3949,7 +4581,7 @@ class basic_json { if (not pos.m_it.primitive_iterator.is_begin()) { - throw std::out_of_range("iterator out of range"); + JSON_THROW(invalid_iterator::create(205, "iterator out of range")); } if (is_string()) @@ -3979,7 +4611,7 @@ class basic_json default: { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4006,11 +4638,11 @@ class basic_json @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on iterators which does not belong to - the current JSON value; example: `"iterators do not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.203 if called on iterators which does not belong + to the current JSON value; example: `"iterators do not fit current value"` + @throw invalid_iterator.204 if called on a primitive type with invalid iterators (i.e., if `first != begin()` and `last != end()`); example: `"iterators out of range"` @@ -4041,7 +4673,7 @@ class basic_json // make sure iterator fits the current value if (this != first.m_object or this != last.m_object) { - throw std::domain_error("iterators do not fit current value"); + JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value")); } IteratorType result = end(); @@ -4056,7 +4688,7 @@ class basic_json { if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) { - throw std::out_of_range("iterators out of range"); + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); } if (is_string()) @@ -4088,7 +4720,7 @@ class basic_json default: { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4109,7 +4741,7 @@ class basic_json @post References and iterators to the erased elements are invalidated. Other references and iterators are not affected. - @throw std::domain_error when called on a type other than JSON object; + @throw type_error.307 when called on a type other than JSON object; example: `"cannot use erase() with null"` @complexity `log(size()) + count(key)` @@ -4131,10 +4763,8 @@ class basic_json { return m_value.object->erase(key); } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } + + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } /*! @@ -4144,9 +4774,9 @@ class basic_json @param[in] idx index of the element to remove - @throw std::domain_error when called on a type other than JSON array; + @throw type_error.307 when called on a type other than JSON object; example: `"cannot use erase() with null"` - @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + @throw out_of_range.401 when `idx >= size()`; example: `"array index 17 is out of range"` @complexity Linear in distance between @a idx and the end of the container. @@ -4168,14 +4798,14 @@ class basic_json { if (idx >= size()) { - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } m_value.array->erase(m_value.array->begin() + static_cast(idx)); } else { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4877,7 +5507,7 @@ class basic_json @param[in] val the value to add to the JSON array - @throw std::domain_error when called on a type other than JSON array or + @throw type_error.308 when called on a type other than JSON array or null; example: `"cannot use push_back() with number"` @complexity Amortized constant. @@ -4893,7 +5523,7 @@ class basic_json // push_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an array @@ -4929,7 +5559,7 @@ class basic_json // push_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an array @@ -4963,7 +5593,7 @@ class basic_json @param[in] val the value to add to the JSON object - @throw std::domain_error when called on a type other than JSON object or + @throw type_error.308 when called on a type other than JSON object or null; example: `"cannot use push_back() with number"` @complexity Logarithmic in the size of the container, O(log(`size()`)). @@ -4979,7 +5609,7 @@ class basic_json // push_back only works for null objects or objects if (not(is_null() or is_object())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an object @@ -5017,7 +5647,7 @@ class basic_json @ref push_back(const typename object_t::value_type&). Otherwise, @a init is converted to a JSON value and added using @ref push_back(basic_json&&). - @param init an initializer list + @param[in] init an initializer list @complexity Linear in the size of the initializer list @a init. @@ -5062,7 +5692,7 @@ class basic_json @param[in] args arguments to forward to a constructor of @ref basic_json @tparam Args compatible types to create a @ref basic_json object - @throw std::domain_error when called on a type other than JSON array or + @throw type_error.311 when called on a type other than JSON array or null; example: `"cannot use emplace_back() with number"` @complexity Amortized constant. @@ -5079,7 +5709,7 @@ class basic_json // emplace_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use emplace_back() with " + type_name()); + JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + type_name())); } // transform null object into an array @@ -5097,8 +5727,8 @@ class basic_json /*! @brief add an object to an object if key does not exist - Inserts a new element into a JSON object constructed in-place with the given - @a args if there is no element with the key in the container. If the + Inserts a new element into a JSON object constructed in-place with the + given @a args if there is no element with the key in the container. If the function is called on a JSON null value, an empty object is created before appending the value created from @a args. @@ -5109,7 +5739,7 @@ class basic_json already-existing element if no insertion happened, and a bool denoting whether the insertion took place. - @throw std::domain_error when called on a type other than JSON object or + @throw type_error.311 when called on a type other than JSON object or null; example: `"cannot use emplace() with number"` @complexity Logarithmic in the size of the container, O(log(`size()`)). @@ -5127,7 +5757,7 @@ class basic_json // emplace only works for null objects or arrays if (not(is_null() or is_object())) { - throw std::domain_error("cannot use emplace() with " + type_name()); + JSON_THROW(type_error::create(311, "cannot use emplace() with " + type_name())); } // transform null object into an object @@ -5158,13 +5788,13 @@ class basic_json @param[in] val element to insert @return iterator pointing to the inserted @a val. - @throw std::domain_error if called on JSON values other than arrays; + @throw type_error.309 if called on JSON values other than arrays; example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` - @complexity Constant plus linear in the distance between pos and end of the - container. + @complexity Constant plus linear in the distance between @a pos and end of + the container. @liveexample{The example shows how `insert()` is used.,insert} @@ -5178,7 +5808,7 @@ class basic_json // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5186,10 +5816,8 @@ class basic_json result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); return result; } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } /*! @@ -5213,10 +5841,10 @@ class basic_json @return iterator pointing to the first element inserted, or @a pos if `cnt==0` - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` @complexity Linear in @a cnt plus linear in the distance between @a pos and end of the container. @@ -5233,7 +5861,7 @@ class basic_json // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5241,10 +5869,8 @@ class basic_json result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); return result; } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } /*! @@ -5257,13 +5883,13 @@ class basic_json @param[in] first begin of the range of elements to insert @param[in] last end of the range of elements to insert - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - @throw std::domain_error if @a first and @a last do not belong to the same - JSON value; example: `"iterators do not fit"` - @throw std::domain_error if @a first or @a last are iterators into + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + @throw invalid_iterator.211 if @a first or @a last are iterators into container for which insert is called; example: `"passed iterators may not belong to container"` @@ -5282,24 +5908,24 @@ class basic_json // insert only works for arrays if (not is_array()) { - throw std::domain_error("cannot use insert() with " + type_name()); + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // check if range iterators belong to the same JSON object if (first.m_object != last.m_object) { - throw std::domain_error("iterators do not fit"); + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); } if (first.m_object == this or last.m_object == this) { - throw std::domain_error("passed iterators may not belong to container"); + JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container")); } // insert to array and return iterator @@ -5320,10 +5946,10 @@ class basic_json the end() iterator @param[in] ilist initializer list to insert the values from - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` @return iterator pointing to the first element inserted, or @a pos if `ilist` is empty @@ -5340,13 +5966,13 @@ class basic_json // insert only works for arrays if (not is_array()) { - throw std::domain_error("cannot use insert() with " + type_name()); + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5355,6 +5981,52 @@ class basic_json return result; } + /*! + @brief inserts elements + + Inserts elements from range `[first, last)`. + + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw type_error.309 if called on JSON values other than objects; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if iterator @a first or @a last does does not + point to an object; example: `"iterators first and last must point to + objects"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + + @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number + of elements to insert. + + @liveexample{The example shows how `insert()` is used.,insert__range_object} + + @since version 3.0.0 + */ + void insert(const_iterator first, const_iterator last) + { + // insert only works for objects + if (not is_object()) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); + } + + // passed iterators must belong to objects + if (not first.m_object->is_object() or not first.m_object->is_object()) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); + } + + m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + } + /*! @brief exchanges the values @@ -5394,7 +6066,7 @@ class basic_json @param[in,out] other array to exchange the contents with - @throw std::domain_error when JSON value is not an array; example: `"cannot + @throw type_error.310 when JSON value is not an array; example: `"cannot use swap() with string"` @complexity Constant. @@ -5413,7 +6085,7 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } @@ -5427,7 +6099,7 @@ class basic_json @param[in,out] other object to exchange the contents with - @throw std::domain_error when JSON value is not an object; example: + @throw type_error.310 when JSON value is not an object; example: `"cannot use swap() with string"` @complexity Constant. @@ -5446,7 +6118,7 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } @@ -5460,7 +6132,7 @@ class basic_json @param[in,out] other string to exchange the contents with - @throw std::domain_error when JSON value is not a string; example: `"cannot + @throw type_error.310 when JSON value is not a string; example: `"cannot use swap() with boolean"` @complexity Constant. @@ -5479,13 +6151,13 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } /// @} - + public: ////////////////////////////////////////// // lexicographical comparison operators // ////////////////////////////////////////// @@ -5493,64 +6165,198 @@ class basic_json /// @name lexicographical comparison operators /// @{ - private: /*! - @brief comparison operator for JSON types + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same according to their respective + `operator==`. + - Integer and floating-point numbers are automatically converted before + comparison. Floating-point numbers are compared indirectly: two + floating-point numbers `f1` and `f2` are considered equal if neither + `f1 > f2` nor `f2 > f1` holds. Note than two NaN values are always + treated as unequal. + - Two JSON null values are equal. + + @note NaN values never compare equal to themselves or to other NaN values. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array == *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object == *rhs.m_value.object; + } + case value_t::null: + { + return true; + } + case value_t::string: + { + return *lhs.m_value.string == *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean == rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer == rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float == rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs == basic_json(rhs)); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) == rhs); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal - Returns an ordering that is similar to Python: - - order: null < boolean < number < object < array < string - - furthermore, each type is not smaller than itself + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} @since version 1.0.0 */ - friend bool operator<(const value_t lhs, const value_t rhs) noexcept + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept { - static constexpr std::array order = {{ - 0, // null - 3, // object - 4, // array - 5, // string - 1, // boolean - 2, // integer - 2, // unsigned - 2, // float - } - }; + return not (lhs == rhs); + } - // discarded values are not comparable - if (lhs == value_t::discarded or rhs == value_t::discarded) - { - return false; - } + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs != basic_json(rhs)); + } - return order[static_cast(lhs)] < order[static_cast(rhs)]; + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) != rhs); } - public: /*! - @brief comparison: equal + @brief comparison: less than - Compares two JSON values for equality according to the following rules: - - Two JSON values are equal if (1) they are from the same type and (2) - their stored values are the same. + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. - Integer and floating-point numbers are automatically converted before - comparison. Floating-point numbers are compared indirectly: two - floating-point numbers `f1` and `f2` are considered equal if neither - `f1 > f2` nor `f2 > f1` holds. - - Two JSON null values are equal. + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). @param[in] lhs first JSON value to consider @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are equal + @return whether @a lhs is less than @a rhs @complexity Linear. @liveexample{The example demonstrates comparing several JSON - types.,operator__equal} + types.,operator__less} @since version 1.0.0 */ - friend bool operator==(const_reference lhs, const_reference rhs) noexcept + friend bool operator<(const_reference lhs, const_reference rhs) noexcept { const auto lhs_type = lhs.type(); const auto rhs_type = rhs.type(); @@ -5561,35 +6367,35 @@ class basic_json { case value_t::array: { - return *lhs.m_value.array == *rhs.m_value.array; + return *lhs.m_value.array < *rhs.m_value.array; } case value_t::object: { - return *lhs.m_value.object == *rhs.m_value.object; + return *lhs.m_value.object < *rhs.m_value.object; } case value_t::null: { - return true; + return false; } case value_t::string: { - return *lhs.m_value.string == *rhs.m_value.string; + return *lhs.m_value.string < *rhs.m_value.string; } case value_t::boolean: { - return lhs.m_value.boolean == rhs.m_value.boolean; + return lhs.m_value.boolean < rhs.m_value.boolean; } case value_t::number_integer: { - return lhs.m_value.number_integer == rhs.m_value.number_integer; + return lhs.m_value.number_integer < rhs.m_value.number_integer; } case value_t::number_unsigned: { - return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; } case value_t::number_float: { - return lhs.m_value.number_float == rhs.m_value.number_float; + return lhs.m_value.number_float < rhs.m_value.number_float; } default: { @@ -5599,295 +6405,786 @@ class basic_json } else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) { - return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); } else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) { - return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; } - return false; + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); } /*! - @brief comparison: equal + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs < basic_json(rhs)); + } + + /*! + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) < rhs); + } - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `v.is_null()`. + /*! + @brief comparison: less than or equal - @param[in] v JSON value to consider - @return whether @a v is null + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. - @complexity Constant. + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. - @liveexample{The example compares several JSON types to the null pointer. - ,operator__equal__nullptr_t} + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} @since version 1.0.0 */ - friend bool operator==(const_reference v, std::nullptr_t) noexcept + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept { - return v.is_null(); + return not (rhs < lhs); } /*! - @brief comparison: equal - @copydoc operator==(const_reference, std::nullptr_t) + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) */ - friend bool operator==(std::nullptr_t, const_reference v) noexcept + template::value, int>::type = 0> + friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept { - return v.is_null(); + return (lhs <= basic_json(rhs)); } /*! - @brief comparison: not equal + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) <= rhs); + } - Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. @param[in] lhs first JSON value to consider @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are not equal + @return whether @a lhs is greater than to @a rhs @complexity Linear. @liveexample{The example demonstrates comparing several JSON - types.,operator__notequal} + types.,operator__lessequal} @since version 1.0.0 */ - friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + friend bool operator>(const_reference lhs, const_reference rhs) noexcept { - return not (lhs == rhs); + return not (lhs <= rhs); } /*! - @brief comparison: not equal + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs > basic_json(rhs)); + } + + /*! + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) > rhs); + } - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `not v.is_null()`. + /*! + @brief comparison: greater than or equal - @param[in] v JSON value to consider - @return whether @a v is not null + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. - @complexity Constant. + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. - @liveexample{The example compares several JSON types to the null pointer. - ,operator__notequal__nullptr_t} + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} @since version 1.0.0 */ - friend bool operator!=(const_reference v, std::nullptr_t) noexcept + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept { - return not v.is_null(); + return not (lhs < rhs); } /*! - @brief comparison: not equal - @copydoc operator!=(const_reference, std::nullptr_t) + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) */ - friend bool operator!=(std::nullptr_t, const_reference v) noexcept + template::value, int>::type = 0> + friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept { - return not v.is_null(); + return (lhs >= basic_json(rhs)); } /*! - @brief comparison: less than + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) >= rhs); + } - Compares whether one JSON value @a lhs is less than another JSON value @a - rhs according to the following rules: - - If @a lhs and @a rhs have the same type, the values are compared using - the default `<` operator. - - Integer and floating-point numbers are automatically converted before - comparison - - In case @a lhs and @a rhs have different types, the values are ignored - and the order of the types is considered, see - @ref operator<(const value_t, const value_t). + /// @} - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than @a rhs - @complexity Linear. + /////////////////// + // serialization // + /////////////////// - @liveexample{The example demonstrates comparing several JSON - types.,operator__less} + /// @name serialization + /// @{ - @since version 1.0.0 + private: + /*! + @brief wrapper around the serialization functions */ - friend bool operator<(const_reference lhs, const_reference rhs) noexcept + class serializer { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); + private: + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; - if (lhs_type == rhs_type) + public: + /*! + @param[in] s output stream to serialize to + */ + serializer(std::ostream& s) + : o(s), loc(std::localeconv()), + thousands_sep(!loc->thousands_sep ? '\0' : loc->thousands_sep[0]), + decimal_point(!loc->decimal_point ? '\0' : loc->decimal_point[0]) + {} + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and + organizes the serialization internally. The indentation level is + propagated as additional parameter. In case of arrays and objects, the + function is called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const basic_json& val, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) { - switch (lhs_type) + switch (val.m_type) { - case value_t::array: - { - return *lhs.m_value.array < *rhs.m_value.array; - } case value_t::object: { - return *lhs.m_value.object < *rhs.m_value.object; + if (val.m_value.object->empty()) + { + o.write("{}", 2); + return; + } + + if (pretty_print) + { + o.write("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (indent_string.size() < new_indent) + { + indent_string.resize(new_indent, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o.write(indent_string.c_str(), static_cast(new_indent)); + o.put('\"'); + dump_escaped(i->first); + o.write("\": ", 3); + dump(i->second, true, indent_step, new_indent); + o.write(",\n", 2); + } + + // last element + assert(i != val.m_value.object->cend()); + o.write(indent_string.c_str(), static_cast(new_indent)); + o.put('\"'); + dump_escaped(i->first); + o.write("\": ", 3); + dump(i->second, true, indent_step, new_indent); + + o.put('\n'); + o.write(indent_string.c_str(), static_cast(current_indent)); + o.put('}'); + } + else + { + o.put('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o.put('\"'); + dump_escaped(i->first); + o.write("\":", 2); + dump(i->second, false, indent_step, current_indent); + o.put(','); + } + + // last element + assert(i != val.m_value.object->cend()); + o.put('\"'); + dump_escaped(i->first); + o.write("\":", 2); + dump(i->second, false, indent_step, current_indent); + + o.put('}'); + } + + return; } - case value_t::null: + + case value_t::array: { - return false; + if (val.m_value.array->empty()) + { + o.write("[]", 2); + return; + } + + if (pretty_print) + { + o.write("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (indent_string.size() < new_indent) + { + indent_string.resize(new_indent, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i) + { + o.write(indent_string.c_str(), static_cast(new_indent)); + dump(*i, true, indent_step, new_indent); + o.write(",\n", 2); + } + + // last element + assert(not val.m_value.array->empty()); + o.write(indent_string.c_str(), static_cast(new_indent)); + dump(val.m_value.array->back(), true, indent_step, new_indent); + + o.put('\n'); + o.write(indent_string.c_str(), static_cast(current_indent)); + o.put(']'); + } + else + { + o.put('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, indent_step, current_indent); + o.put(','); + } + + // last element + assert(not val.m_value.array->empty()); + dump(val.m_value.array->back(), false, indent_step, current_indent); + + o.put(']'); + } + + return; } + case value_t::string: { - return *lhs.m_value.string < *rhs.m_value.string; + o.put('\"'); + dump_escaped(*val.m_value.string); + o.put('\"'); + return; } + case value_t::boolean: { - return lhs.m_value.boolean < rhs.m_value.boolean; + if (val.m_value.boolean) + { + o.write("true", 4); + } + else + { + o.write("false", 5); + } + return; } + case value_t::number_integer: { - return lhs.m_value.number_integer < rhs.m_value.number_integer; + dump_integer(val.m_value.number_integer); + return; } + case value_t::number_unsigned: { - return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + dump_integer(val.m_value.number_unsigned); + return; } + case value_t::number_float: { - return lhs.m_value.number_float < rhs.m_value.number_float; + dump_float(val.m_value.number_float); + return; } - default: + + case value_t::discarded: { - return false; + o.write("", 11); + return; + } + + case value_t::null: + { + o.write("null", 4); + return; } } } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + + private: + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x0b: + case 0x0e: + case 0x0f: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1a: + case 0x1b: + case 0x1c: + case 0x1d: + case 0x1e: + case 0x1f: + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + + default: + { + return res; + } + } + }); } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence + of an escape character (backslash) and another character and other + control characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s) const { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + const auto space = extra_space(s); + if (space == 0) + { + o.write(s.c_str(), static_cast(s.size())); + return; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x0b: + case 0x0e: + case 0x0f: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1a: + case 0x1b: + case 0x1c: + case 0x1d: + case 0x1e: + case 0x1f: + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + break; + } + + default: + { + // all other characters are added as-is + result[pos++] = c; + break; + } + } + } + + assert(pos == s.size() + space); + o.write(result.c_str(), static_cast(result.size())); } - // We only reach this line if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - return operator<(lhs_type, rhs_type); - } + /*! + @brief dump an integer - /*! - @brief comparison: less than or equal + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. - Compares whether one JSON value @a lhs is less than or equal to another - JSON value by calculating `not (rhs < lhs)`. + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template::value or + std::is_same::value, int> = 0> + void dump_integer(NumberType x) + { + // special case for "0" + if (x == 0) + { + o.put('0'); + return; + } - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than or equal to @a rhs + const bool is_negative = x < 0; + size_t i = 0; - @complexity Linear. + // spare 1 byte for '\0' + while (x != 0 and i < number_buffer.size() - 1) + { + const auto digit = std::labs(static_cast(x % 10)); + number_buffer[i++] = static_cast('0' + digit); + x /= 10; + } - @liveexample{The example demonstrates comparing several JSON - types.,operator__greater} + // make sure the number has been processed completely + assert(x == 0); - @since version 1.0.0 - */ - friend bool operator<=(const_reference lhs, const_reference rhs) noexcept - { - return not (rhs < lhs); - } + if (is_negative) + { + // make sure there is capacity for the '-' + assert(i < number_buffer.size() - 2); + number_buffer[i++] = '-'; + } - /*! - @brief comparison: greater than + std::reverse(number_buffer.begin(), number_buffer.begin() + i); + o.write(number_buffer.data(), static_cast(i)); + } - Compares whether one JSON value @a lhs is greater than another - JSON value by calculating `not (lhs <= rhs)`. + /*! + @brief dump a floating-point number - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than to @a rhs + Dump a given floating-point number to output stream @a o. Works + internally with @a number_buffer. - @complexity Linear. + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (not std::isfinite(x) or std::isnan(x)) + { + o.write("null", 4); + return; + } - @liveexample{The example demonstrates comparing several JSON - types.,operator__lessequal} + // special case for 0.0 and -0.0 + if (x == 0) + { + if (std::signbit(x)) + { + o.write("-0.0", 4); + } + else + { + o.write("0.0", 3); + } + return; + } - @since version 1.0.0 - */ - friend bool operator>(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs <= rhs); - } + // get number of digits for a text -> float -> text round-trip + static constexpr auto d = std::numeric_limits::digits10; - /*! - @brief comparison: greater than or equal + // the actual conversion + std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), + "%.*g", d, x); - Compares whether one JSON value @a lhs is greater than or equal to another - JSON value by calculating `not (lhs < rhs)`. + // negative value indicates an error + assert(len > 0); + // check if buffer was large enough + assert(static_cast(len) < number_buffer.size()); - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than or equal to @a rhs + // erase thousands separator + if (thousands_sep != '\0') + { + const auto end = std::remove(number_buffer.begin(), + number_buffer.begin() + len, + thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + assert((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } - @complexity Linear. + // convert decimal point to '.' + if (decimal_point != '\0' and decimal_point != '.') + { + for (auto& c : number_buffer) + { + if (c == decimal_point) + { + c = '.'; + break; + } + } + } - @liveexample{The example demonstrates comparing several JSON - types.,operator__greaterequal} + o.write(number_buffer.data(), static_cast(len)); - @since version 1.0.0 - */ - friend bool operator>=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs < rhs); - } + // determine if need to append ".0" + const bool value_is_int_like = std::none_of(number_buffer.begin(), + number_buffer.begin() + len + 1, + [](char c) + { + return c == '.' or c == 'e'; + }); - /// @} + if (value_is_int_like) + { + o.write(".0", 2); + } + } + private: + /// the output of the serializer + std::ostream& o; - /////////////////// - // serialization // - /////////////////// + /// a (hopefully) large enough character buffer + std::array number_buffer{{}}; - /// @name serialization - /// @{ + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// the indentation string + string_t indent_string = string_t(512, ' '); + }; + public: /*! @brief serialize to stream @@ -5898,10 +7195,6 @@ class basic_json `std::setw(4)` on @a o sets the indentation level to `4` and the serialization result is the same as calling `dump(4)`. - @note During serializaion, the locale and the precision of the output - stream @a o are changed. The original values are restored when the - function returns. - @param[in,out] o stream to serialize to @param[in] j JSON value to serialize @@ -5923,29 +7216,20 @@ class basic_json // reset width to 0 for subsequent calls to this stream o.width(0); - // fix locale problems - const auto old_locale = o.imbue(std::locale::classic()); - // set precision - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - const auto old_precision = o.precision(std::numeric_limits::digits10); - // do the actual serialization - j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale and precision - o.imbue(old_locale); - o.precision(old_precision); + serializer s(o); + s.dump(j, pretty_print, static_cast(indentation)); return o; } /*! @brief serialize to stream - @copydoc operator<<(std::ostream&, const basic_json&) + @deprecated This stream operator is deprecated and will be removed in a + future version of the library. Please use + @ref std::ostream& operator<<(std::ostream&, const basic_json&) + instead; that is, replace calls like `j >> o;` with `o << j;`. */ + JSON_DEPRECATED friend std::ostream& operator>>(const basic_json& j, std::ostream& o) { return o << j; @@ -5977,6 +7261,11 @@ class basic_json @return result of the deserialization + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6007,6 +7296,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6043,6 +7336,11 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @throw parse_error.111 if input stream is in a bad state + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6102,6 +7400,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6122,7 +7424,7 @@ class basic_json { // assertion to check that the iterator range is indeed contiguous, // see http://stackoverflow.com/a/35008842/266378 for more discussion - assert(std::accumulate(first, last, std::make_pair(true, 0), + assert(std::accumulate(first, last, std::pair(true, 0), [&first](std::pair res, decltype(*first) val) { res.first &= (val == *(std::next(std::addressof(*first), res.second++))); @@ -6172,6 +7474,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6192,8 +7498,22 @@ class basic_json static basic_json parse(const ContiguousContainer& c, const parser_callback_t cb = nullptr) { - // delegate the call to the iterator-range parse overload - return parse(std::begin(c), std::end(c), cb); + // delegate the call to the iterator-range parse overload + return parse(std::begin(c), std::end(c), cb); + } + + /*! + @brief deserialize from stream + @deprecated This stream operator is deprecated and will be removed in a + future version of the library. Please use + @ref std::istream& operator>>(std::istream&, basic_json&) + instead; that is, replace calls like `j << i;` with `i >> j;`. + */ + JSON_DEPRECATED + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; } /*! @@ -6204,7 +7524,10 @@ class basic_json @param[in,out] i input stream to read a serialized JSON value from @param[in,out] j JSON value to write the deserialized input to - @throw std::invalid_argument in case of parse errors + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @throw parse_error.111 if input stream is in a bad state @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. @@ -6219,16 +7542,6 @@ class basic_json @since version 1.0.0 */ - friend std::istream& operator<<(basic_json& j, std::istream& i) - { - j = parser(i).parse(); - return i; - } - - /*! - @brief deserialize from stream - @copydoc operator<<(basic_json&, std::istream&) - */ friend std::istream& operator>>(std::istream& i, basic_json& j) { j = parser(i).parse(); @@ -6245,6 +7558,11 @@ class basic_json /// @{ private: + /*! + @note Some code in the switch cases has been copied, because otherwise + copilers would complain about implicit fallthrough and there is no + portable attribute to mute such warnings. + */ template static void add_to_vector(std::vector& vec, size_t bytes, const T number) { @@ -6254,24 +7572,31 @@ class basic_json { case 8: { - vec.push_back(static_cast((number >> 070) & 0xff)); - vec.push_back(static_cast((number >> 060) & 0xff)); - vec.push_back(static_cast((number >> 050) & 0xff)); - vec.push_back(static_cast((number >> 040) & 0xff)); - // intentional fall-through + vec.push_back(static_cast((static_cast(number) >> 070) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 060) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 050) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 040) & 0xff)); + vec.push_back(static_cast((number >> 030) & 0xff)); + vec.push_back(static_cast((number >> 020) & 0xff)); + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; } case 4: { vec.push_back(static_cast((number >> 030) & 0xff)); vec.push_back(static_cast((number >> 020) & 0xff)); - // intentional fall-through + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; } case 2: { vec.push_back(static_cast((number >> 010) & 0xff)); - // intentional fall-through + vec.push_back(static_cast(number & 0xff)); + break; } case 1: @@ -6296,7 +7621,7 @@ class basic_json @tparam T the integral return type - @throw std::out_of_range if there are less than sizeof(T)+1 bytes in the + @throw parse_error.110 if there are less than sizeof(T)+1 bytes in the vector @a vec to read In the for loop, the bytes from the vector are copied in reverse order into @@ -6321,13 +7646,11 @@ class basic_json template static T get_from_vector(const std::vector& vec, const size_t current_index) { - if (current_index + sizeof(T) + 1 > vec.size()) - { - throw std::out_of_range("cannot read " + std::to_string(sizeof(T)) + " bytes from vector"); - } + // check if we can read sizeof(T) bytes starting the next index + check_length(vec.size(), sizeof(T), current_index + 1); T result; - uint8_t* ptr = reinterpret_cast(&result); + auto* ptr = reinterpret_cast(&result); for (size_t i = 0; i < sizeof(T); ++i) { *ptr++ = vec[current_index + sizeof(T) - i]; @@ -6368,32 +7691,33 @@ class basic_json if (j.m_value.number_integer >= 0) { // MessagePack does not differentiate between positive - // signed integers and unsigned integers. Therefore, we used - // the code from the value_t::number_unsigned case here. + // signed integers and unsigned integers. Therefore, we + // used the code from the value_t::number_unsigned case + // here. if (j.m_value.number_unsigned < 128) { // positive fixnum add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT8_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 v.push_back(0xcc); add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT16_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 v.push_back(0xcd); add_to_vector(v, 2, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT32_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 v.push_back(0xce); add_to_vector(v, 4, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT64_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 v.push_back(0xcf); @@ -6407,25 +7731,25 @@ class basic_json // negative fixnum add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT8_MIN and j.m_value.number_integer <= INT8_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 v.push_back(0xd0); add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT16_MIN and j.m_value.number_integer <= INT16_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 v.push_back(0xd1); add_to_vector(v, 2, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT32_MIN and j.m_value.number_integer <= INT32_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 v.push_back(0xd2); add_to_vector(v, 4, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT64_MIN and j.m_value.number_integer <= INT64_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 v.push_back(0xd3); @@ -6442,25 +7766,25 @@ class basic_json // positive fixnum add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT8_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 v.push_back(0xcc); add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT16_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 v.push_back(0xcd); add_to_vector(v, 2, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT32_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 v.push_back(0xce); add_to_vector(v, 4, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT64_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 v.push_back(0xcf); @@ -6473,7 +7797,7 @@ class basic_json { // float 64 v.push_back(0xcb); - const uint8_t* helper = reinterpret_cast(&(j.m_value.number_float)); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); for (size_t i = 0; i < 8; ++i) { v.push_back(helper[7 - i]); @@ -6617,19 +7941,19 @@ class basic_json { add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT8_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x18); // one-byte uint8_t add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT16_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x19); // two-byte uint16_t add_to_vector(v, 2, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT32_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x1a); // four-byte uint32_t @@ -6644,26 +7968,26 @@ class basic_json } else { - // The conversions below encode the sign in the first byte, - // and the value is converted to a positive number. + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. const auto positive_number = -1 - j.m_value.number_integer; if (j.m_value.number_integer >= -24) { v.push_back(static_cast(0x20 + positive_number)); } - else if (positive_number <= UINT8_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 8 v.push_back(0x38); add_to_vector(v, 1, positive_number); } - else if (positive_number <= UINT16_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 16 v.push_back(0x39); add_to_vector(v, 2, positive_number); } - else if (positive_number <= UINT32_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 32 v.push_back(0x3a); @@ -6716,7 +8040,7 @@ class basic_json { // Double-Precision Float v.push_back(0xfb); - const uint8_t* helper = reinterpret_cast(&(j.m_value.number_float)); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); for (size_t i = 0; i < 8; ++i) { v.push_back(helper[7 - i]); @@ -6729,7 +8053,7 @@ class basic_json const auto N = j.m_value.string->size(); if (N <= 0x17) { - v.push_back(0x60 + N); // 1 byte for string + size + v.push_back(static_cast(0x60 + N)); // 1 byte for string + size } else if (N <= 0xff) { @@ -6765,7 +8089,7 @@ class basic_json const auto N = j.m_value.array->size(); if (N <= 0x17) { - v.push_back(0x80 + N); // 1 byte for array + size + v.push_back(static_cast(0x80 + N)); // 1 byte for array + size } else if (N <= 0xff) { @@ -6803,7 +8127,7 @@ class basic_json const auto N = j.m_value.object->size(); if (N <= 0x17) { - v.push_back(0xa0 + N); // 1 byte for object + size + v.push_back(static_cast(0xa0 + N)); // 1 byte for object + size } else if (N <= 0xff) { @@ -6850,12 +8174,12 @@ class basic_json To secure the access to the byte vector during CBOR/MessagePack deserialization, bytes are copied from the vector into buffers. This - function checks if the number of bytes to copy (@a len) does not exceed the - size @s size of the vector. Additionally, an @a offset is given from where - to start reading the bytes. + function checks if the number of bytes to copy (@a len) does not exceed + the size @s size of the vector. Additionally, an @a offset is given from + where to start reading the bytes. - This function checks whether reading the bytes is safe; that is, offset is a - valid index in the vector, offset+len + This function checks whether reading the bytes is safe; that is, offset is + a valid index in the vector, offset+len @param[in] size size of the byte vector @param[in] len number of bytes to read @@ -6872,20 +8196,81 @@ class basic_json // simple case: requested length is greater than the vector's length if (len > size or offset > size) { - throw std::out_of_range("len out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); } // second case: adding offset would result in overflow - if ((size > (std::numeric_limits::max() - offset))) + if ((size > ((std::numeric_limits::max)() - offset))) { - throw std::out_of_range("len+offset out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); } // last case: reading past the end of the vector if (len + offset > size) { - throw std::out_of_range("len+offset out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); + } + } + + /*! + @brief check if the next byte belongs to a string + + While parsing a map, the keys must be strings. This function checks if the + current byte is one of the start bytes for a string in MessagePack: + + - 0xa0 - 0xbf: fixstr + - 0xd9: str 8 + - 0xda: str 16 + - 0xdb: str 32 + + @param[in] v MessagePack serialization + @param[in] idx byte index in @a v to check for a string + + @throw parse_error.113 if `v[idx]` does not belong to a string + */ + static void msgpack_expect_string(const std::vector& v, size_t idx) + { + check_length(v.size(), 1, idx); + + const auto byte = v[idx]; + if ((byte >= 0xa0 and byte <= 0xbf) or (byte >= 0xd9 and byte <= 0xdb)) + { + return; + } + + std::stringstream ss; + ss << std::hex << static_cast(v[idx]); + JSON_THROW(parse_error::create(113, idx + 1, "expected a MessagePack string; last byte: 0x" + ss.str())); + } + + /*! + @brief check if the next byte belongs to a string + + While parsing a map, the keys must be strings. This function checks if the + current byte is one of the start bytes for a string in CBOR: + + - 0x60 - 0x77: fixed length + - 0x78 - 0x7b: variable length + - 0x7f: indefinity length + + @param[in] v CBOR serialization + @param[in] idx byte index in @a v to check for a string + + @throw parse_error.113 if `v[idx]` does not belong to a string + */ + static void cbor_expect_string(const std::vector& v, size_t idx) + { + check_length(v.size(), 1, idx); + + const auto byte = v[idx]; + if ((byte >= 0x60 and byte <= 0x7b) or byte == 0x7f) + { + return; } + + std::stringstream ss; + ss << std::hex << static_cast(v[idx]); + JSON_THROW(parse_error::create(113, idx + 1, "expected a CBOR string; last byte: 0x" + ss.str())); } /*! @@ -6896,32 +8281,34 @@ class basic_json @return deserialized JSON value - @throw std::invalid_argument if unsupported features from MessagePack were + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from MessagePack were used in the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.113 if a string was expected as map key, but not found @sa https://github.com/msgpack/msgpack/blob/master/spec.md */ static basic_json from_msgpack_internal(const std::vector& v, size_t& idx) { - // make sure reading 1 byte is safe - check_length(v.size(), 1, idx); - // store and increment index const size_t current_idx = idx++; + // make sure reading 1 byte is safe + check_length(v.size(), 1, current_idx); + if (v[current_idx] <= 0xbf) { if (v[current_idx] <= 0x7f) // positive fixint { return v[current_idx]; } - else if (v[current_idx] <= 0x8f) // fixmap + if (v[current_idx] <= 0x8f) // fixmap { basic_json result = value_t::object; const size_t len = v[current_idx] & 0x0f; for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -6972,8 +8359,8 @@ class basic_json case 0xca: // float 32 { // copy bytes in reverse order into the double variable - check_length(v.size(), sizeof(float), 1); float res; + check_length(v.size(), sizeof(float), current_idx + 1); for (size_t byte = 0; byte < sizeof(float); ++byte) { reinterpret_cast(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte]; @@ -6985,8 +8372,8 @@ class basic_json case 0xcb: // float 64 { // copy bytes in reverse order into the double variable - check_length(v.size(), sizeof(double), 1); double res; + check_length(v.size(), sizeof(double), current_idx + 1); for (size_t byte = 0; byte < sizeof(double); ++byte) { reinterpret_cast(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte]; @@ -7101,6 +8488,7 @@ class basic_json idx += 2; // skip 2 size bytes for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -7114,6 +8502,7 @@ class basic_json idx += 4; // skip 4 size bytes for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -7122,7 +8511,9 @@ class basic_json default: { - throw std::invalid_argument("error parsing a msgpack @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx]))); + std::stringstream ss; + ss << std::hex << static_cast(v[current_idx]); + JSON_THROW(parse_error::create(112, current_idx + 1, "error reading MessagePack; last byte: 0x" + ss.str())); } } } @@ -7136,9 +8527,10 @@ class basic_json @return deserialized JSON value - @throw std::invalid_argument if unsupported features from CBOR were used in - the given vector @a v or if the input is not valid CBOR - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from CBOR were + used in the given vector @a v or if the input is not valid CBOR + @throw parse_error.113 if a string was expected as map key, but not found @sa https://tools.ietf.org/html/rfc7049 */ @@ -7147,7 +8539,10 @@ class basic_json // store and increment index const size_t current_idx = idx++; - switch (v.at(current_idx)) + // make sure reading 1 byte is safe + check_length(v.size(), 1, current_idx); + + switch (v[current_idx]) { // Integer 0x00..0x17 (0..23) case 0x00: @@ -7328,7 +8723,7 @@ class basic_json case 0x7f: // UTF-8 string (indefinite length) { std::string result; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { string_t s = from_cbor_internal(v, idx); result += s; @@ -7424,7 +8819,7 @@ class basic_json case 0x9f: // array (indefinite length) { basic_json result = value_t::array; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { result.push_back(from_cbor_internal(v, idx)); } @@ -7463,6 +8858,7 @@ class basic_json const auto len = static_cast(v[current_idx] - 0xa0); for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7476,6 +8872,7 @@ class basic_json idx += 1; // skip 1 size byte for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7489,6 +8886,7 @@ class basic_json idx += 2; // skip 2 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7502,6 +8900,7 @@ class basic_json idx += 4; // skip 4 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7515,6 +8914,7 @@ class basic_json idx += 8; // skip 8 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7524,8 +8924,9 @@ class basic_json case 0xbf: // map (indefinite length) { basic_json result = value_t::object; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7551,7 +8952,6 @@ class basic_json case 0xf9: // Half-Precision Float (two-byte IEEE 754) { - check_length(v.size(), 2, 1); idx += 2; // skip two content bytes // code from RFC 7049, Appendix D, Figure 3: @@ -7561,6 +8961,7 @@ class basic_json // include at least decoding support for them even without such // support. An example of a small decoder for half-precision // floating-point numbers in the C language is shown in Fig. 3. + check_length(v.size(), 2, current_idx + 1); const int half = (v[current_idx + 1] << 8) + v[current_idx + 2]; const int exp = (half >> 10) & 0x1f; const int mant = half & 0x3ff; @@ -7575,16 +8976,18 @@ class basic_json } else { - val = mant == 0 ? INFINITY : NAN; + val = mant == 0 + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); } - return half & 0x8000 ? -val : val; + return (half & 0x8000) != 0 ? -val : val; } case 0xfa: // Single-Precision Float (four-byte IEEE 754) { // copy bytes in reverse order into the float variable - check_length(v.size(), sizeof(float), 1); float res; + check_length(v.size(), sizeof(float), current_idx + 1); for (size_t byte = 0; byte < sizeof(float); ++byte) { reinterpret_cast(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte]; @@ -7595,9 +8998,9 @@ class basic_json case 0xfb: // Double-Precision Float (eight-byte IEEE 754) { - check_length(v.size(), sizeof(double), 1); // copy bytes in reverse order into the double variable double res; + check_length(v.size(), sizeof(double), current_idx + 1); for (size_t byte = 0; byte < sizeof(double); ++byte) { reinterpret_cast(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte]; @@ -7608,7 +9011,9 @@ class basic_json default: // anything else (0xFF is handled inside the other types) { - throw std::invalid_argument("error parsing a CBOR @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx]))); + std::stringstream ss; + ss << std::hex << static_cast(v[current_idx]); + JSON_THROW(parse_error::create(112, current_idx + 1, "error reading CBOR; last byte: 0x" + ss.str())); } } } @@ -7621,6 +9026,58 @@ class basic_json serialization format. MessagePack is a binary serialization format which aims to be more compact than JSON itself, yet more efficient to parse. + The library uses the following mapping from JSON values types to + MessagePack types according to the MessagePack specification: + + JSON value type | value/range | MessagePack type | first byte + --------------- | --------------------------------- | ---------------- | ---------- + null | `null` | nil | 0xc0 + boolean | `true` | true | 0xc3 + boolean | `false` | false | 0xc2 + number_integer | -9223372036854775808..-2147483649 | int64 | 0xd3 + number_integer | -2147483648..-32769 | int32 | 0xd2 + number_integer | -32768..-129 | int16 | 0xd1 + number_integer | -128..-33 | int8 | 0xd0 + number_integer | -32..-1 | negative fixint | 0xe0..0xff + number_integer | 0..127 | positive fixint | 0x00..0x7f + number_integer | 128..255 | uint 8 | 0xcc + number_integer | 256..65535 | uint 16 | 0xcd + number_integer | 65536..4294967295 | uint 32 | 0xce + number_integer | 4294967296..18446744073709551615 | uint 64 | 0xcf + number_unsigned | 0..127 | positive fixint | 0x00..0x7f + number_unsigned | 128..255 | uint 8 | 0xcc + number_unsigned | 256..65535 | uint 16 | 0xcd + number_unsigned | 65536..4294967295 | uint 32 | 0xce + number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xcf + number_float | *any value* | float 64 | 0xcb + string | *length*: 0..31 | fixstr | 0xa0..0xbf + string | *length*: 32..255 | str 8 | 0xd9 + string | *length*: 256..65535 | str 16 | 0xda + string | *length*: 65536..4294967295 | str 32 | 0xdb + array | *size*: 0..15 | fixarray | 0x90..0x9f + array | *size*: 16..65535 | array 16 | 0xdc + array | *size*: 65536..4294967295 | array 32 | 0xdd + object | *size*: 0..15 | fix map | 0x80..0x8f + object | *size*: 16..65535 | map 16 | 0xde + object | *size*: 65536..4294967295 | map 32 | 0xdf + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a MessagePack value. + + @note The following values can **not** be converted to a MessagePack value: + - strings with more than 4294967295 bytes + - arrays with more than 4294967295 elements + - objects with more than 4294967295 elements + + @note The following MessagePack types are not used in the conversion: + - bin 8 - bin 32 (0xc4..0xc6) + - ext 8 - ext 32 (0xc7..0xc9) + - float 32 (0xca) + - fixext 1 - fixext 16 (0xd4..0xd8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + @param[in] j JSON value to serialize @return MessagePack serialization as byte vector @@ -7630,9 +9087,11 @@ class basic_json vector in MessagePack format.,to_msgpack} @sa http://msgpack.org - @sa @ref from_msgpack(const std::vector&) for the analogous - deserialization + @sa @ref from_msgpack(const std::vector&, const size_t) for the + analogous deserialization @sa @ref to_cbor(const basic_json& for the related CBOR format + + @since version 2.0.9 */ static std::vector to_msgpack(const basic_json& j) { @@ -7647,12 +9106,54 @@ class basic_json Deserializes a given byte vector @a v to a JSON value using the MessagePack serialization format. + The library maps MessagePack types to JSON value types as follows: + + MessagePack type | JSON value type | first byte + ---------------- | --------------- | ---------- + positive fixint | number_unsigned | 0x00..0x7f + fixmap | object | 0x80..0x8f + fixarray | array | 0x90..0x9f + fixstr | string | 0xa0..0xbf + nil | `null` | 0xc0 + false | `false` | 0xc2 + true | `true` | 0xc3 + float 32 | number_float | 0xca + float 64 | number_float | 0xcb + uint 8 | number_unsigned | 0xcc + uint 16 | number_unsigned | 0xcd + uint 32 | number_unsigned | 0xce + uint 64 | number_unsigned | 0xcf + int 8 | number_integer | 0xd0 + int 16 | number_integer | 0xd1 + int 32 | number_integer | 0xd2 + int 64 | number_integer | 0xd3 + str 8 | string | 0xd9 + str 16 | string | 0xda + str 32 | string | 0xdb + array 16 | array | 0xdc + array 32 | array | 0xdd + map 16 | object | 0xde + map 32 | object | 0xdf + negative fixint | number_integer | 0xe0-0xff + + @warning The mapping is **incomplete** in the sense that not all + MessagePack types can be converted to a JSON value. The following + MessagePack types are not supported and will yield parse errors: + - bin 8 - bin 32 (0xc4..0xc6) + - ext 8 - ext 32 (0xc7..0xc9) + - fixext 1 - fixext 16 (0xd4..0xd8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + @param[in] v a byte vector in MessagePack format + @param[in] start_index the index to start reading from @a v (0 by default) @return deserialized JSON value - @throw std::invalid_argument if unsupported features from MessagePack were + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from MessagePack were used in the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.113 if a string was expected as map key, but not found @complexity Linear in the size of the byte vector @a v. @@ -7661,11 +9162,15 @@ class basic_json @sa http://msgpack.org @sa @ref to_msgpack(const basic_json&) for the analogous serialization - @sa @ref from_cbor(const std::vector&) for the related CBOR format + @sa @ref from_cbor(const std::vector&, const size_t) for the + related CBOR format + + @since version 2.0.9, parameter @a start_index since 2.1.1 */ - static basic_json from_msgpack(const std::vector& v) + static basic_json from_msgpack(const std::vector& v, + const size_t start_index = 0) { - size_t i = 0; + size_t i = start_index; return from_msgpack_internal(v, i); } @@ -7677,6 +9182,65 @@ class basic_json serialization format which aims to be more compact than JSON itself, yet more efficient to parse. + The library uses the following mapping from JSON values types to + CBOR types according to the CBOR specification (RFC 7049): + + JSON value type | value/range | CBOR type | first byte + --------------- | ------------------------------------------ | ---------------------------------- | --------------- + null | `null` | Null | 0xf6 + boolean | `true` | True | 0xf5 + boolean | `false` | False | 0xf4 + number_integer | -9223372036854775808..-2147483649 | Negative integer (8 bytes follow) | 0x3b + number_integer | -2147483648..-32769 | Negative integer (4 bytes follow) | 0x3a + number_integer | -32768..-129 | Negative integer (2 bytes follow) | 0x39 + number_integer | -128..-25 | Negative integer (1 byte follow) | 0x38 + number_integer | -24..-1 | Negative integer | 0x20..0x37 + number_integer | 0..23 | Integer | 0x00..0x17 + number_integer | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_integer | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_integer | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a + number_integer | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b + number_unsigned | 0..23 | Integer | 0x00..0x17 + number_unsigned | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_unsigned | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_unsigned | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a + number_unsigned | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b + number_float | *any value* | Double-Precision Float | 0xfb + string | *length*: 0..23 | UTF-8 string | 0x60..0x77 + string | *length*: 23..255 | UTF-8 string (1 byte follow) | 0x78 + string | *length*: 256..65535 | UTF-8 string (2 bytes follow) | 0x79 + string | *length*: 65536..4294967295 | UTF-8 string (4 bytes follow) | 0x7a + string | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow) | 0x7b + array | *size*: 0..23 | array | 0x80..0x97 + array | *size*: 23..255 | array (1 byte follow) | 0x98 + array | *size*: 256..65535 | array (2 bytes follow) | 0x99 + array | *size*: 65536..4294967295 | array (4 bytes follow) | 0x9a + array | *size*: 4294967296..18446744073709551615 | array (8 bytes follow) | 0x9b + object | *size*: 0..23 | map | 0xa0..0xb7 + object | *size*: 23..255 | map (1 byte follow) | 0xb8 + object | *size*: 256..65535 | map (2 bytes follow) | 0xb9 + object | *size*: 65536..4294967295 | map (4 bytes follow) | 0xba + object | *size*: 4294967296..18446744073709551615 | map (8 bytes follow) | 0xbb + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a CBOR value. + + @note The following CBOR types are not used in the conversion: + - byte strings (0x40..0x5f) + - UTF-8 strings terminated by "break" (0x7f) + - arrays terminated by "break" (0x9f) + - maps terminated by "break" (0xbf) + - date/time (0xc0..0xc1) + - bignum (0xc2..0xc3) + - decimal fraction (0xc4) + - bigfloat (0xc5) + - tagged items (0xc6..0xd4, 0xd8..0xdb) + - expected conversions (0xd5..0xd7) + - simple values (0xe0..0xf3, 0xf8) + - undefined (0xf7) + - half and single-precision floats (0xf9-0xfa) + - break (0xff) + @param[in] j JSON value to serialize @return MessagePack serialization as byte vector @@ -7686,9 +9250,11 @@ class basic_json vector in CBOR format.,to_cbor} @sa http://cbor.io - @sa @ref from_cbor(const std::vector&) for the analogous - deserialization + @sa @ref from_cbor(const std::vector&, const size_t) for the + analogous deserialization @sa @ref to_msgpack(const basic_json& for the related MessagePack format + + @since version 2.0.9 */ static std::vector to_cbor(const basic_json& j) { @@ -7703,12 +9269,74 @@ class basic_json Deserializes a given byte vector @a v to a JSON value using the CBOR (Concise Binary Object Representation) serialization format. + The library maps CBOR types to JSON value types as follows: + + CBOR type | JSON value type | first byte + ---------------------- | --------------- | ---------- + Integer | number_unsigned | 0x00..0x17 + Unsigned integer | number_unsigned | 0x18 + Unsigned integer | number_unsigned | 0x19 + Unsigned integer | number_unsigned | 0x1a + Unsigned integer | number_unsigned | 0x1b + Negative integer | number_integer | 0x20..0x37 + Negative integer | number_integer | 0x38 + Negative integer | number_integer | 0x39 + Negative integer | number_integer | 0x3a + Negative integer | number_integer | 0x3b + Negative integer | number_integer | 0x40..0x57 + UTF-8 string | string | 0x60..0x77 + UTF-8 string | string | 0x78 + UTF-8 string | string | 0x79 + UTF-8 string | string | 0x7a + UTF-8 string | string | 0x7b + UTF-8 string | string | 0x7f + array | array | 0x80..0x97 + array | array | 0x98 + array | array | 0x99 + array | array | 0x9a + array | array | 0x9b + array | array | 0x9f + map | object | 0xa0..0xb7 + map | object | 0xb8 + map | object | 0xb9 + map | object | 0xba + map | object | 0xbb + map | object | 0xbf + False | `false` | 0xf4 + True | `true` | 0xf5 + Nill | `null` | 0xf6 + Half-Precision Float | number_float | 0xf9 + Single-Precision Float | number_float | 0xfa + Double-Precision Float | number_float | 0xfb + + @warning The mapping is **incomplete** in the sense that not all CBOR + types can be converted to a JSON value. The following CBOR types + are not supported and will yield parse errors (parse_error.112): + - byte strings (0x40..0x5f) + - date/time (0xc0..0xc1) + - bignum (0xc2..0xc3) + - decimal fraction (0xc4) + - bigfloat (0xc5) + - tagged items (0xc6..0xd4, 0xd8..0xdb) + - expected conversions (0xd5..0xd7) + - simple values (0xe0..0xf3, 0xf8) + - undefined (0xf7) + + @warning CBOR allows map keys of any type, whereas JSON only allows + strings as keys in object values. Therefore, CBOR maps with keys + other than UTF-8 strings are rejected (parse_error.113). + + @note Any CBOR output created @ref to_cbor can be successfully parsed by + @ref from_cbor. + @param[in] v a byte vector in CBOR format + @param[in] start_index the index to start reading from @a v (0 by default) @return deserialized JSON value - @throw std::invalid_argument if unsupported features from CBOR were used in - the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from CBOR were + used in the given vector @a v or if the input is not valid CBOR + @throw parse_error.113 if a string was expected as map key, but not found @complexity Linear in the size of the byte vector @a v. @@ -7717,375 +9345,63 @@ class basic_json @sa http://cbor.io @sa @ref to_cbor(const basic_json&) for the analogous serialization - @sa @ref from_msgpack(const std::vector&) for the related - MessagePack format - */ - static basic_json from_cbor(const std::vector& v) - { - size_t i = 0; - return from_cbor_internal(v, i); - } - - /// @} - - private: - /////////////////////////// - // convenience functions // - /////////////////////////// - - /*! - @brief return the type as string - - Returns the type name as string to be used in error messages - usually to - indicate that a function was called on a wrong JSON type. - - @return basically a string representation of a the @a m_type member - - @complexity Constant. - - @since version 1.0.0 - */ - std::string type_name() const - { - switch (m_type) - { - case value_t::null: - return "null"; - case value_t::object: - return "object"; - case value_t::array: - return "array"; - case value_t::string: - return "string"; - case value_t::boolean: - return "boolean"; - case value_t::discarded: - return "discarded"; - default: - return "number"; - } - } - - /*! - @brief calculates the extra space to escape a JSON string - - @param[in] s the string to escape - @return the number of characters required to escape string @a s - - @complexity Linear in the length of string @a s. - */ - static std::size_t extra_space(const string_t& s) noexcept - { - return std::accumulate(s.begin(), s.end(), size_t{}, - [](size_t res, typename string_t::value_type c) - { - switch (c) - { - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - return res + 1; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // from c (1 byte) to \uxxxx (6 bytes) - return res + 5; - } - else - { - return res; - } - } - } - }); - } - - /*! - @brief escape a string - - Escape a string by replacing certain special characters by a sequence of - an escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. - - @param[in] s the string to escape - @return the escaped string - - @complexity Linear in the length of string @a s. - */ - static string_t escape_string(const string_t& s) - { - const auto space = extra_space(s); - if (space == 0) - { - return s; - } - - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; - - for (const auto& c : s) - { - switch (c) - { - // quotation mark (0x22) - case '"': - { - result[pos + 1] = '"'; - pos += 2; - break; - } - - // reverse solidus (0x5c) - case '\\': - { - // nothing to change - pos += 2; - break; - } - - // backspace (0x08) - case '\b': - { - result[pos + 1] = 'b'; - pos += 2; - break; - } - - // formfeed (0x0c) - case '\f': - { - result[pos + 1] = 'f'; - pos += 2; - break; - } - - // newline (0x0a) - case '\n': - { - result[pos + 1] = 'n'; - pos += 2; - break; - } - - // carriage return (0x0d) - case '\r': - { - result[pos + 1] = 'r'; - pos += 2; - break; - } - - // horizontal tab (0x09) - case '\t': - { - result[pos + 1] = 't'; - pos += 2; - break; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // convert a number 0..15 to its hex representation - // (0..f) - static const char hexify[16] = - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - - // print character c as \uxxxx - for (const char m : - { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] - }) - { - result[++pos] = m; - } - - ++pos; - } - else - { - // all other characters are added as-is - result[pos++] = c; - } - break; - } - } - } - - return result; - } - - /*! - @brief internal implementation of the serialization function - - This function is called by the public member function dump and organizes - the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. Note that - - - strings and object keys are escaped using `escape_string()` - - integer numbers are converted implicitly via `operator<<` - - floating-point numbers are converted to a string using `"%g"` format - - @param[out] o stream to write to - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] indent_step the indent level - @param[in] current_indent the current indent level (only used internally) - */ - void dump(std::ostream& o, - const bool pretty_print, - const unsigned int indent_step, - const unsigned int current_indent = 0) const - { - // variable to hold indentation for recursive calls - unsigned int new_indent = current_indent; - - switch (m_type) - { - case value_t::object: - { - if (m_value.object->empty()) - { - o << "{}"; - return; - } - - o << "{"; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) - { - if (i != m_value.object->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' ') << "\"" - << escape_string(i->first) << "\":" - << (pretty_print ? " " : ""); - i->second.dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') + "}"; - return; - } - - case value_t::array: - { - if (m_value.array->empty()) - { - o << "[]"; - return; - } - - o << "["; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) - { - if (i != m_value.array->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' '); - i->dump(o, pretty_print, indent_step, new_indent); - } + @sa @ref from_msgpack(const std::vector&, const size_t) for the + related MessagePack format - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } + @since version 2.0.9, parameter @a start_index since 2.1.1 + */ + static basic_json from_cbor(const std::vector& v, + const size_t start_index = 0) + { + size_t i = start_index; + return from_cbor_internal(v, i); + } - o << string_t(new_indent, ' ') << "]"; - return; - } + /// @} - case value_t::string: - { - o << string_t("\"") << escape_string(*m_value.string) << "\""; - return; - } + /////////////////////////// + // convenience functions // + /////////////////////////// - case value_t::boolean: - { - o << (m_value.boolean ? "true" : "false"); - return; - } + /*! + @brief return the type as string - case value_t::number_integer: - { - o << m_value.number_integer; - return; - } + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. - case value_t::number_unsigned: - { - o << m_value.number_unsigned; - return; - } + @return basically a string representation of a the @a m_type member - case value_t::number_float: - { - if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); - } - else - { - o << m_value.number_float; - } - return; - } + @complexity Constant. - case value_t::discarded: - { - o << ""; - return; - } + @liveexample{The following code exemplifies `type_name()` for all JSON + types.,type_name} - case value_t::null: + @since version 1.0.0, public since 2.1.0 + */ + std::string type_name() const + { + { + switch (m_type) { - o << "null"; - return; + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; } } } + private: ////////////////////// // member variables // @@ -8115,6 +9431,11 @@ class basic_json class primitive_iterator_t { public: + + difference_type get_value() const noexcept + { + return m_it; + } /// set iterator to a defined beginning void set_begin() noexcept { @@ -8139,16 +9460,89 @@ class basic_json return (m_it == end_value); } - /// return reference to the value to change and compare - operator difference_type& () noexcept + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept { - return m_it; + return lhs.m_it == rhs.m_it; } - /// return value to compare - constexpr operator difference_type () const noexcept + friend constexpr bool operator!=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept { - return m_it; + return !(lhs == rhs); + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + friend constexpr bool operator<=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it <= rhs.m_it; + } + + friend constexpr bool operator>(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it > rhs.m_it; + } + + friend constexpr bool operator>=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it >= rhs.m_it; + } + + primitive_iterator_t operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it) + { + return os << it.m_it; + } + + primitive_iterator_t& operator++() + { + ++m_it; + return *this; + } + + primitive_iterator_t operator++(int) + { + auto result = *this; + m_it++; + return result; + } + + primitive_iterator_t& operator--() + { + --m_it; + return *this; + } + + primitive_iterator_t operator--(int) + { + auto result = *this; + m_it--; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) + { + m_it -= n; + return *this; } private: @@ -8500,7 +9894,7 @@ class basic_json case basic_json::value_t::null: { - throw std::out_of_range("cannot get value"); + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } default: @@ -8509,10 +9903,8 @@ class basic_json { return *m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8545,10 +9937,8 @@ class basic_json { return m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8648,7 +10038,7 @@ class basic_json // if objects are not the same, the comparison is undefined if (m_object != other.m_object) { - throw std::domain_error("cannot compare iterators of different containers"); + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); } assert(m_object != nullptr); @@ -8690,7 +10080,7 @@ class basic_json // if objects are not the same, the comparison is undefined if (m_object != other.m_object) { - throw std::domain_error("cannot compare iterators of different containers"); + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); } assert(m_object != nullptr); @@ -8699,7 +10089,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot compare order of object iterators"); + JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators")); } case basic_json::value_t::array: @@ -8753,7 +10143,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use offsets with object iterators"); + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); } case basic_json::value_t::array: @@ -8815,7 +10205,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use offsets with object iterators"); + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); } case basic_json::value_t::array: @@ -8842,7 +10232,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use operator[] for object iterators"); + JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators")); } case basic_json::value_t::array: @@ -8852,19 +10242,17 @@ class basic_json case basic_json::value_t::null: { - throw std::out_of_range("cannot get value"); + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } default: { - if (m_it.primitive_iterator == -n) + if (m_it.primitive_iterator.get_value() == -n) { return *m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8881,10 +10269,8 @@ class basic_json { return m_it.object_iterator->first; } - else - { - throw std::domain_error("cannot use key() for non-object iterators"); - } + + JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators")); } /*! @@ -9039,7 +10425,9 @@ class basic_json literal_false, ///< the `false` literal literal_null, ///< the `null` literal value_string, ///< a string -- use get_string() for actual value - value_number, ///< a number -- use get_number() for actual value + value_unsigned, ///< an unsigned integer -- use get_number() for actual value + value_integer, ///< a signed integer -- use get_number() for actual value + value_float, ///< an floating point number -- use get_number() for actual value begin_array, ///< the character for array begin `[` begin_object, ///< the character for object begin `{` end_array, ///< the character for array end `]` @@ -9062,14 +10450,17 @@ class basic_json m_limit = m_content + len; } - /// a lexer from an input stream + /*! + @brief a lexer from an input stream + @throw parse_error.111 if input stream is in a bad state + */ explicit lexer(std::istream& s) : m_stream(&s), m_line_buffer() { // immediately abort if stream is erroneous if (s.fail()) { - throw std::invalid_argument("stream error"); + JSON_THROW(parse_error::create(111, 0, "bad input stream")); } // fill buffer @@ -9103,17 +10494,17 @@ class basic_json @return string representation of the code point; the length of the result string is between 1 and 4 characters. - @throw std::out_of_range if code point is > 0x10ffff; example: `"code - points above 0x10FFFF are invalid"` - @throw std::invalid_argument if the low surrogate is invalid; example: + @throw parse_error.102 if the low surrogate is invalid; example: `""missing or wrong low surrogate""` + @throw parse_error.103 if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` @complexity Constant. @see */ - static string_t to_unicode(const std::size_t codepoint1, - const std::size_t codepoint2 = 0) + string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) const { // calculate the code point from the given code points std::size_t codepoint = codepoint1; @@ -9136,7 +10527,7 @@ class basic_json } else { - throw std::invalid_argument("missing or wrong low surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing or wrong low surrogate")); } } @@ -9150,27 +10541,27 @@ class basic_json else if (codepoint <= 0x7ff) { // 2-byte characters: 110xxxxx 10xxxxxx - result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast(0xC0 | (codepoint >> 6))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else if (codepoint <= 0xffff) { // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast(0xE0 | (codepoint >> 12))); result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else if (codepoint <= 0x10ffff) { // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast(0xF0 | (codepoint >> 18))); result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else { - throw std::out_of_range("code points above 0x10FFFF are invalid"); + JSON_THROW(parse_error::create(103, get_position(), "code points above 0x10FFFF are invalid")); } return result; @@ -9191,7 +10582,9 @@ class basic_json return "null literal"; case token_type::value_string: return "string literal"; - case token_type::value_number: + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: return "number literal"; case token_type::begin_array: return "'['"; @@ -9233,10 +10626,9 @@ class basic_json Proof (by contradiction): Assume a finite input. To loop forever, the loop must never hit code with a `break` statement. The only code - snippets without a `break` statement are the continue statements for - whitespace and byte-order-marks. To loop forever, the input must be an - infinite sequence of whitespace or byte-order-marks. This contradicts - the assumption of finite input, q.e.d. + snippets without a `break` statement is the continue statement for + whitespace. To loop forever, the input must be an infinite sequence + whitespace. This contradicts the assumption of finite input, q.e.d. */ token_type scan() { @@ -9428,6 +10820,7 @@ basic_json_parser_6: goto basic_json_parser_6; } { + position += static_cast((m_cursor - m_start)); continue; } basic_json_parser_9: @@ -9464,37 +10857,47 @@ basic_json_parser_12: } if (yych <= '0') { - goto basic_json_parser_13; + goto basic_json_parser_43; } if (yych <= '9') { - goto basic_json_parser_15; + goto basic_json_parser_45; } goto basic_json_parser_5; basic_json_parser_13: yyaccept = 1; yych = *(m_marker = ++m_cursor); - if (yych <= 'D') + if (yych <= '9') { if (yych == '.') { - goto basic_json_parser_43; + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; } } else { if (yych <= 'E') { - goto basic_json_parser_44; + if (yych >= 'E') + { + goto basic_json_parser_51; + } } - if (yych == 'e') + else { - goto basic_json_parser_44; + if (yych == 'e') + { + goto basic_json_parser_51; + } } } basic_json_parser_14: { - last_token_type = token_type::value_number; + last_token_type = token_type::value_unsigned; break; } basic_json_parser_15: @@ -9513,7 +10916,7 @@ basic_json_parser_15: { if (yych == '.') { - goto basic_json_parser_43; + goto basic_json_parser_47; } goto basic_json_parser_14; } @@ -9521,11 +10924,11 @@ basic_json_parser_15: { if (yych <= 'E') { - goto basic_json_parser_44; + goto basic_json_parser_51; } if (yych == 'e') { - goto basic_json_parser_44; + goto basic_json_parser_51; } goto basic_json_parser_14; } @@ -9552,7 +10955,7 @@ basic_json_parser_23: yych = *(m_marker = ++m_cursor); if (yych == 'a') { - goto basic_json_parser_45; + goto basic_json_parser_52; } goto basic_json_parser_5; basic_json_parser_24: @@ -9560,7 +10963,7 @@ basic_json_parser_24: yych = *(m_marker = ++m_cursor); if (yych == 'u') { - goto basic_json_parser_46; + goto basic_json_parser_53; } goto basic_json_parser_5; basic_json_parser_25: @@ -9568,7 +10971,7 @@ basic_json_parser_25: yych = *(m_marker = ++m_cursor); if (yych == 'r') { - goto basic_json_parser_47; + goto basic_json_parser_54; } goto basic_json_parser_5; basic_json_parser_26: @@ -9650,13 +11053,27 @@ basic_json_parser_31: } basic_json_parser_32: m_cursor = m_marker; - if (yyaccept == 0) + if (yyaccept <= 1) { - goto basic_json_parser_5; + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } } else { - goto basic_json_parser_14; + if (yyaccept == 2) + { + goto basic_json_parser_44; + } + else + { + goto basic_json_parser_58; + } } basic_json_parser_33: ++m_cursor; @@ -9737,7 +11154,7 @@ basic_json_parser_35: } if (yych <= 'u') { - goto basic_json_parser_48; + goto basic_json_parser_55; } goto basic_json_parser_32; } @@ -9836,43 +11253,138 @@ basic_json_parser_41: } if (yych <= 0xBF) { - goto basic_json_parser_38; + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_42: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0x8F) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_43: + yyaccept = 2; + yych = *(m_marker = ++m_cursor); + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; + } + } + else + { + if (yych <= 'E') + { + if (yych >= 'E') + { + goto basic_json_parser_51; + } + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + } + } +basic_json_parser_44: + { + last_token_type = token_type::value_integer; + break; + } +basic_json_parser_45: + yyaccept = 2; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych <= '/') + { + goto basic_json_parser_44; + } + goto basic_json_parser_45; + } + else + { + if (yych <= 'E') + { + if (yych <= 'D') + { + goto basic_json_parser_44; + } + goto basic_json_parser_51; + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + goto basic_json_parser_44; + } + } +basic_json_parser_47: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_56; } goto basic_json_parser_32; -basic_json_parser_42: +basic_json_parser_48: ++m_cursor; if (m_limit <= m_cursor) { fill_line_buffer(1); // LCOV_EXCL_LINE } yych = *m_cursor; - if (yych <= 0x7F) - { - goto basic_json_parser_32; - } - if (yych <= 0x8F) - { - goto basic_json_parser_38; - } - goto basic_json_parser_32; -basic_json_parser_43: - yych = *++m_cursor; if (yych <= '/') { - goto basic_json_parser_32; + goto basic_json_parser_50; } if (yych <= '9') { - goto basic_json_parser_49; + goto basic_json_parser_48; } - goto basic_json_parser_32; -basic_json_parser_44: +basic_json_parser_50: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_51: yych = *++m_cursor; if (yych <= ',') { if (yych == '+') { - goto basic_json_parser_51; + goto basic_json_parser_59; } goto basic_json_parser_32; } @@ -9880,7 +11392,7 @@ basic_json_parser_44: { if (yych <= '-') { - goto basic_json_parser_51; + goto basic_json_parser_59; } if (yych <= '/') { @@ -9888,32 +11400,32 @@ basic_json_parser_44: } if (yych <= '9') { - goto basic_json_parser_52; + goto basic_json_parser_60; } goto basic_json_parser_32; } -basic_json_parser_45: +basic_json_parser_52: yych = *++m_cursor; if (yych == 'l') { - goto basic_json_parser_54; + goto basic_json_parser_62; } goto basic_json_parser_32; -basic_json_parser_46: +basic_json_parser_53: yych = *++m_cursor; if (yych == 'l') { - goto basic_json_parser_55; + goto basic_json_parser_63; } goto basic_json_parser_32; -basic_json_parser_47: +basic_json_parser_54: yych = *++m_cursor; if (yych == 'u') { - goto basic_json_parser_56; + goto basic_json_parser_64; } goto basic_json_parser_32; -basic_json_parser_48: +basic_json_parser_55: ++m_cursor; if (m_limit <= m_cursor) { @@ -9928,7 +11440,7 @@ basic_json_parser_48: } if (yych <= '9') { - goto basic_json_parser_57; + goto basic_json_parser_65; } goto basic_json_parser_32; } @@ -9936,7 +11448,7 @@ basic_json_parser_48: { if (yych <= 'F') { - goto basic_json_parser_57; + goto basic_json_parser_65; } if (yych <= '`') { @@ -9944,12 +11456,12 @@ basic_json_parser_48: } if (yych <= 'f') { - goto basic_json_parser_57; + goto basic_json_parser_65; } goto basic_json_parser_32; } -basic_json_parser_49: - yyaccept = 1; +basic_json_parser_56: + yyaccept = 3; m_marker = ++m_cursor; if ((m_limit - m_cursor) < 3) { @@ -9960,27 +11472,30 @@ basic_json_parser_49: { if (yych <= '/') { - goto basic_json_parser_14; + goto basic_json_parser_58; } if (yych <= '9') { - goto basic_json_parser_49; + goto basic_json_parser_56; } - goto basic_json_parser_14; } else { if (yych <= 'E') { - goto basic_json_parser_44; + goto basic_json_parser_51; } if (yych == 'e') { - goto basic_json_parser_44; + goto basic_json_parser_51; } - goto basic_json_parser_14; } -basic_json_parser_51: +basic_json_parser_58: + { + last_token_type = token_type::value_float; + break; + } +basic_json_parser_59: yych = *++m_cursor; if (yych <= '/') { @@ -9990,7 +11505,7 @@ basic_json_parser_51: { goto basic_json_parser_32; } -basic_json_parser_52: +basic_json_parser_60: ++m_cursor; if (m_limit <= m_cursor) { @@ -9999,35 +11514,35 @@ basic_json_parser_52: yych = *m_cursor; if (yych <= '/') { - goto basic_json_parser_14; + goto basic_json_parser_58; } if (yych <= '9') { - goto basic_json_parser_52; + goto basic_json_parser_60; } - goto basic_json_parser_14; -basic_json_parser_54: + goto basic_json_parser_58; +basic_json_parser_62: yych = *++m_cursor; if (yych == 's') { - goto basic_json_parser_58; + goto basic_json_parser_66; } goto basic_json_parser_32; -basic_json_parser_55: +basic_json_parser_63: yych = *++m_cursor; if (yych == 'l') { - goto basic_json_parser_59; + goto basic_json_parser_67; } goto basic_json_parser_32; -basic_json_parser_56: +basic_json_parser_64: yych = *++m_cursor; if (yych == 'e') { - goto basic_json_parser_61; + goto basic_json_parser_69; } goto basic_json_parser_32; -basic_json_parser_57: +basic_json_parser_65: ++m_cursor; if (m_limit <= m_cursor) { @@ -10042,7 +11557,7 @@ basic_json_parser_57: } if (yych <= '9') { - goto basic_json_parser_63; + goto basic_json_parser_71; } goto basic_json_parser_32; } @@ -10050,7 +11565,7 @@ basic_json_parser_57: { if (yych <= 'F') { - goto basic_json_parser_63; + goto basic_json_parser_71; } if (yych <= '`') { @@ -10058,30 +11573,30 @@ basic_json_parser_57: } if (yych <= 'f') { - goto basic_json_parser_63; + goto basic_json_parser_71; } goto basic_json_parser_32; } -basic_json_parser_58: +basic_json_parser_66: yych = *++m_cursor; if (yych == 'e') { - goto basic_json_parser_64; + goto basic_json_parser_72; } goto basic_json_parser_32; -basic_json_parser_59: +basic_json_parser_67: ++m_cursor; { last_token_type = token_type::literal_null; break; } -basic_json_parser_61: +basic_json_parser_69: ++m_cursor; { last_token_type = token_type::literal_true; break; } -basic_json_parser_63: +basic_json_parser_71: ++m_cursor; if (m_limit <= m_cursor) { @@ -10096,7 +11611,7 @@ basic_json_parser_63: } if (yych <= '9') { - goto basic_json_parser_66; + goto basic_json_parser_74; } goto basic_json_parser_32; } @@ -10104,7 +11619,7 @@ basic_json_parser_63: { if (yych <= 'F') { - goto basic_json_parser_66; + goto basic_json_parser_74; } if (yych <= '`') { @@ -10112,17 +11627,17 @@ basic_json_parser_63: } if (yych <= 'f') { - goto basic_json_parser_66; + goto basic_json_parser_74; } goto basic_json_parser_32; } -basic_json_parser_64: +basic_json_parser_72: ++m_cursor; { last_token_type = token_type::literal_false; break; } -basic_json_parser_66: +basic_json_parser_74: ++m_cursor; if (m_limit <= m_cursor) { @@ -10161,6 +11676,7 @@ basic_json_parser_66: } + position += static_cast((m_cursor - m_start)); return last_token_type; } @@ -10209,7 +11725,7 @@ basic_json_parser_66: assert(m_marker == nullptr or m_marker <= m_limit); // number of processed characters (p) - const size_t num_processed_chars = static_cast(m_start - m_content); + const auto num_processed_chars = static_cast(m_start - m_content); // offset for m_marker wrt. to m_start const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start; // number of unprocessed characters (u) @@ -10219,7 +11735,7 @@ basic_json_parser_66: if (m_stream == nullptr or m_stream->eof()) { // m_start may or may not be pointing into m_line_buffer at - // this point. We trust the standand library to do the right + // this point. We trust the standard library to do the right // thing. See http://stackoverflow.com/q/28142011/266378 m_line_buffer.assign(m_start, m_limit); @@ -10237,6 +11753,13 @@ basic_json_parser_66: m_line_buffer.erase(0, num_processed_chars); // read next line from input stream m_line_buffer_tmp.clear(); + + // check if stream is still good + if (m_stream->fail()) + { + JSON_THROW(parse_error::create(111, 0, "bad input stream")); + } + std::getline(*m_stream, m_line_buffer_tmp, '\n'); // add line with newline symbol to the line buffer @@ -10307,7 +11830,7 @@ basic_json_parser_66: m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This can be rephrased to m_cursor - m_start - 2 > x. With the precondition, we x <= 0, meaning that the loop condition holds - indefinitly if i is always decreased. However, observe that the value + indefinitely if i is always decreased. However, observe that the value of i is strictly increasing with each iteration, as it is incremented by 1 in the iteration expression and never decremented inside the loop body. Hence, the loop condition will eventually be false which @@ -10316,7 +11839,8 @@ basic_json_parser_66: @return string value of current token without opening and closing quotes - @throw std::out_of_range if to_unicode fails + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails */ string_t get_string() const { @@ -10402,7 +11926,7 @@ basic_json_parser_66: // make sure there is a subsequent unicode if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') { - throw std::invalid_argument("missing low surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing low surrogate")); } // get code yyyy from uxxxx\uyyyy @@ -10415,7 +11939,7 @@ basic_json_parser_66: else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF) { // we found a lone low surrogate - throw std::invalid_argument("missing high surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing high surrogate")); } else { @@ -10433,186 +11957,245 @@ basic_json_parser_66: return result; } - /*! - @brief parse floating point number - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + /*! + @brief parse string into a built-in arithmetic type as if the current + locale is POSIX. - @param[in,out] endptr recieves a pointer to the first character after - the number + @note in floating-point case strtod may parse past the token's end - + this is not an error - @return the floating point number + @note any leading blanks are not handled */ - long double str_to_float_t(long double* /* type */, char** endptr) const + struct strtonum { - return std::strtold(reinterpret_cast(m_start), endptr); - } + public: + strtonum(const char* start, const char* end) + : m_start(start), m_end(end) + {} - /*! - @brief parse floating point number + /*! + @return true iff parsed successfully as number of type T - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + @param[in,out] val shall contain parsed value, or undefined value + if could not parse + */ + template::value>::type> + bool to(T& val) const + { + return parse(val, std::is_integral()); + } - @param[in,out] endptr recieves a pointer to the first character after - the number + private: + const char* const m_start = nullptr; + const char* const m_end = nullptr; - @return the floating point number - */ - double str_to_float_t(double* /* type */, char** endptr) const - { - return std::strtod(reinterpret_cast(m_start), endptr); - } + // floating-point conversion - /*! - @brief parse floating point number + // overloaded wrappers for strtod/strtof/strtold + // that will be called from parse + static void strtof(float& f, const char* str, char** endptr) + { + f = std::strtof(str, endptr); + } - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + static void strtof(double& f, const char* str, char** endptr) + { + f = std::strtod(str, endptr); + } - @param[in,out] endptr recieves a pointer to the first character after - the number + static void strtof(long double& f, const char* str, char** endptr) + { + f = std::strtold(str, endptr); + } - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } + template + bool parse(T& value, /*is_integral=*/std::false_type) const + { + // replace decimal separator with locale-specific version, + // when necessary; data will point to either the original + // string, or buf, or tempstr containing the fixed string. + std::string tempstr; + std::array buf; + const size_t len = static_cast(m_end - m_start); - /*! - @brief return number value for number tokens + // lexer will reject empty numbers + assert(len > 0); - This function translates the last token into the most appropriate - number type (either integer, unsigned integer or floating point), - which is passed back to the caller via the result parameter. + // since dealing with strtod family of functions, we're + // getting the decimal point char from the C locale facilities + // instead of C++'s numpunct facet of the current std::locale + const auto loc = localeconv(); + assert(loc != nullptr); + const char decimal_point_char = (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0]; - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. + const char* data = m_start; - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). + if (decimal_point_char != '.') + { + const size_t ds_pos = static_cast(std::find(m_start, m_end, '.') - m_start); - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. - */ - void get_number(basic_json& result) const - { - assert(m_start != nullptr); + if (ds_pos != len) + { + // copy the data into the local buffer or tempstr, if + // buffer is too small; replace decimal separator, and + // update data to point to the modified bytes + if ((len + 1) < buf.size()) + { + std::copy(m_start, m_end, buf.begin()); + buf[len] = 0; + buf[ds_pos] = decimal_point_char; + data = buf.data(); + } + else + { + tempstr.assign(m_start, m_end); + tempstr[ds_pos] = decimal_point_char; + data = tempstr.c_str(); + } + } + } + + char* endptr = nullptr; + value = 0; + // this calls appropriate overload depending on T + strtof(value, data, &endptr); - const lexer::lexer_char_t* curptr = m_start; + // parsing was successful iff strtof parsed exactly the number + // of characters determined by the lexer (len) + const bool ok = (endptr == (data + len)); - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; + if (ok and (value == static_cast(0.0)) and (*data == '-')) + { + // some implementations forget to negate the zero + value = -0.0; + } - // maximum absolute value of the relevant integer type - number_unsigned_t max; + return ok; + } - // temporarily store the type to avoid unecessary bitfield access - value_t type; + // integral conversion - // look for sign - if (*curptr == '-') + signed long long parse_integral(char** endptr, /*is_signed*/std::true_type) const { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; + return std::strtoll(m_start, endptr, 10); } - else + + unsigned long long parse_integral(char** endptr, /*is_signed*/std::false_type) const + { + return std::strtoull(m_start, endptr, 10); + } + + template + bool parse(T& value, /*is_integral=*/std::true_type) const { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); + char* endptr = nullptr; + errno = 0; // these are thread-local + const auto x = parse_integral(&endptr, std::is_signed()); + + // called right overload? + static_assert(std::is_signed() == std::is_signed(), ""); + + value = static_cast(x); + + return (x == static_cast(value)) // x fits into destination T + and (x < 0) == (value < 0) // preserved sign + //and ((x != 0) or is_integral()) // strto[u]ll did nto fail + and (errno == 0) // strto[u]ll did not overflow + and (m_start < m_end) // token was not empty + and (endptr == m_end); // parsed entire token exactly } + }; + + /*! + @brief return number value for number tokens + + This function translates the last token into the most appropriate + number type (either integer, unsigned integer or floating point), + which is passed back to the caller via the result parameter. + + integral numbers that don't fit into the the range of the respective + type are parsed as number_float_t + + floating-point values do not satisfy std::isfinite predicate + are converted to value_t::null + + throws if the entire string [m_start .. m_cursor) cannot be + interpreted as a number + + @param[out] result @ref basic_json object to receive the number. + @param[in] token the type of the number token + */ + bool get_number(basic_json& result, const token_type token) const + { + assert(m_start != nullptr); + assert(m_start < m_cursor); + assert((token == token_type::value_unsigned) or + (token == token_type::value_integer) or + (token == token_type::value_float)); - // count the significant figures - for (; curptr < m_cursor; curptr++) + strtonum num_converter(reinterpret_cast(m_start), + reinterpret_cast(m_cursor)); + + switch (token) { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') + case lexer::token_type::value_unsigned: { - if (*curptr == '.') + number_unsigned_t val; + if (num_converter.to(val)) { - // don't count '.' but change to float - type = value_t::number_float; - continue; + // parsing successful + result.m_type = value_t::number_unsigned; + result.m_value = val; + return true; } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; break; } - // skip if definitely not an integer - if (type != value_t::number_float) + case lexer::token_type::value_integer: { - auto digit = static_cast(*curptr - '0'); - - // overflow if value * 10 + digit > max, move terms around - // to avoid overflow in intermediate values - if (value > (max - digit) / 10) - { - // overflow - type = value_t::number_float; - } - else + number_integer_t val; + if (num_converter.to(val)) { - // no overflow - value = value * 10 + digit; + // parsing successful + result.m_type = value_t::number_integer; + result.m_value = val; + return true; } + break; } - } - - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - // invariant: if we parsed a '-', the absolute value is between - // 0 (we allow -0) and max == -INT64_MIN - assert(value >= 0); - assert(value <= max); - if (value == max) - { - // we cannot simply negate value (== max == -INT64_MIN), - // see https://github.com/nlohmann/json/issues/389 - result.m_value.number_integer = static_cast(INT64_MIN); - } - else + default: { - // all other values can be negated safely - result.m_value.number_integer = -static_cast(value); + break; } } - else + + // parse float (either explicitly or because a previous conversion + // failed) + number_float_t val; + if (num_converter.to(val)) { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); + // parsing successful + result.m_type = value_t::number_float; + result.m_value = val; - // replace infinity and NAN by null + // throw in case of infinity or NAN if (not std::isfinite(result.m_value.number_float)) { - type = value_t::null; - result.m_value = basic_json::json_value(); + JSON_THROW(out_of_range::create(406, "number overflow parsing '" + get_token_string() + "'")); } + + return true; } - // save the type - result.m_type = type; + // couldn't parse number in any format + return false; + } + + constexpr size_t get_position() const + { + return position; } private: @@ -10634,6 +12217,8 @@ basic_json_parser_66: const lexer_char_t* m_limit = nullptr; /// the last token type token_type last_token_type = token_type::end_of_input; + /// current position in the input (read bytes) + size_t position = 0; }; /*! @@ -10650,7 +12235,10 @@ basic_json_parser_66: m_lexer(reinterpret_cast(buff), std::strlen(buff)) {} - /// a parser reading from an input stream + /*! + @brief a parser reading from an input stream + @throw parse_error.111 if input stream is in a bad state + */ parser(std::istream& is, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(is) {} @@ -10666,7 +12254,12 @@ basic_json_parser_66: static_cast(std::distance(first, last))) {} - /// public parser interface + /*! + @brief public parser interface + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ basic_json parse() { // read first token @@ -10683,7 +12276,12 @@ basic_json_parser_66: } private: - /// the actual parser + /*! + @brief the actual parser + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ basic_json parse_internal(bool keep) { auto result = basic_json(value_t::discarded); @@ -10856,9 +12454,11 @@ basic_json_parser_66: break; } - case lexer::token_type::value_number: + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: { - m_lexer.get_number(result); + m_lexer.get_number(result, last_token); get_token(); break; } @@ -10884,6 +12484,9 @@ basic_json_parser_66: return last_token; } + /*! + @throw parse_error.101 if expected token did not occur + */ void expect(typename lexer::token_type t) const { if (t != last_token) @@ -10893,10 +12496,13 @@ basic_json_parser_66: "'") : lexer::token_type_name(last_token)); error_msg += "; expected " + lexer::token_type_name(t); - throw std::invalid_argument(error_msg); + JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); } } + /*! + @throw parse_error.101 if unexpected token occurred + */ void unexpect(typename lexer::token_type t) const { if (t == last_token) @@ -10905,7 +12511,7 @@ basic_json_parser_66: error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + "'") : lexer::token_type_name(last_token)); - throw std::invalid_argument(error_msg); + JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); } } @@ -10948,12 +12554,12 @@ basic_json_parser_66: empty string is assumed which references the whole JSON value - @throw std::domain_error if reference token is nonempty and does not - begin with a slash (`/`); example: `"JSON pointer must be empty or - begin with /"` - @throw std::domain_error if a tilde (`~`) is not followed by `0` - (representing `~`) or `1` (representing `/`); example: `"escape error: - ~ must be followed with 0 or 1"` + @throw parse_error.107 if the given JSON pointer @a s is nonempty and + does not begin with a slash (`/`); see example below + + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s + is not followed by `0` (representing `~`) or `1` (representing `/`); + see example below @liveexample{The example shows the construction several valid JSON pointers as well as the exceptional behavior.,json_pointer} @@ -10996,12 +12602,15 @@ basic_json_parser_66: } private: - /// remove and return last reference pointer + /*! + @brief remove and return last reference pointer + @throw out_of_range.405 if JSON pointer has no parent + */ std::string pop_back() { if (is_root()) { - throw std::domain_error("JSON pointer has no parent"); + JSON_THROW(out_of_range::create(405, "JSON pointer has no parent")); } auto last = reference_tokens.back(); @@ -11019,7 +12628,7 @@ basic_json_parser_66: { if (is_root()) { - throw std::domain_error("JSON pointer has no parent"); + JSON_THROW(out_of_range::create(405, "JSON pointer has no parent")); } json_pointer result = *this; @@ -11031,6 +12640,9 @@ basic_json_parser_66: @brief create and return a reference to the pointed to value @complexity Linear in the number of reference tokens. + + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened */ reference get_and_create(reference j) const { @@ -11067,7 +12679,14 @@ basic_json_parser_66: case value_t::array: { // create an entry in the array - result = &result->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + result = &result->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } @@ -11080,7 +12699,7 @@ basic_json_parser_66: */ default: { - throw std::domain_error("invalid value to unflatten"); + JSON_THROW(type_error::create(313, "invalid value to unflatten")); } } } @@ -11103,9 +12722,9 @@ basic_json_parser_66: @complexity Linear in the length of the JSON pointer. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved */ reference get_unchecked(pointer ptr) const { @@ -11119,7 +12738,7 @@ basic_json_parser_66: reference_token.end(), [](const char x) { - return std::isdigit(x); + return (x >= '0' and x <= '9'); }); // change value to array for numbers or "-" or to object @@ -11148,25 +12767,32 @@ basic_json_parser_66: // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } if (reference_token == "-") { - // explicityly treat "-" as index beyond the end + // explicitly treat "-" as index beyond the end ptr = &ptr->operator[](ptr->m_value.array->size()); } else { // convert array index to number; unchecked access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11174,6 +12800,12 @@ basic_json_parser_66: return *ptr; } + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ reference get_checked(pointer ptr) const { for (const auto& reference_token : reference_tokens) @@ -11192,25 +12824,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11225,6 +12864,11 @@ basic_json_parser_66: @return const reference to the JSON value pointed to by the JSON pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved */ const_reference get_unchecked(const_pointer ptr) const { @@ -11244,25 +12888,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" cannot be used for const access - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // use unchecked array access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11270,6 +12921,12 @@ basic_json_parser_66: return *ptr; } + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ const_reference get_checked(const_pointer ptr) const { for (const auto& reference_token : reference_tokens) @@ -11288,25 +12945,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11314,7 +12978,15 @@ basic_json_parser_66: return *ptr; } - /// split the string input to reference tokens + /*! + @brief split the string input to reference tokens + + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. + + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ static std::vector split(const std::string& reference_string) { std::vector result; @@ -11328,7 +13000,7 @@ basic_json_parser_66: // check if nonempty reference string begins with slash if (reference_string[0] != '/') { - throw std::domain_error("JSON pointer must be empty or begin with '/'"); + JSON_THROW(parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'")); } // extract the reference tokens: @@ -11336,7 +13008,7 @@ basic_json_parser_66: // - start: position after the previous slash for ( // search for the first slash after the first character - size_t slash = reference_string.find_first_of("/", 1), + size_t slash = reference_string.find_first_of('/', 1), // set the beginning of the first reference token start = 1; // we can stop if start == string::npos+1 = 0 @@ -11345,16 +13017,16 @@ basic_json_parser_66: // (will eventually be 0 if slash == std::string::npos) start = slash + 1, // find next slash - slash = reference_string.find_first_of("/", start)) + slash = reference_string.find_first_of('/', start)) { // use the text between the beginning of the reference token // (start) and the last slash (slash). auto reference_token = reference_string.substr(start, slash - start); // check reference tokens are properly escaped - for (size_t pos = reference_token.find_first_of("~"); + for (size_t pos = reference_token.find_first_of('~'); pos != std::string::npos; - pos = reference_token.find_first_of("~", pos + 1)) + pos = reference_token.find_first_of('~', pos + 1)) { assert(reference_token[pos] == '~'); @@ -11363,7 +13035,7 @@ basic_json_parser_66: (reference_token[pos + 1] != '0' and reference_token[pos + 1] != '1')) { - throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); + JSON_THROW(parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'")); } } @@ -11375,7 +13047,6 @@ basic_json_parser_66: return result; } - private: /*! @brief replace all occurrences of a substring by another string @@ -11384,7 +13055,8 @@ basic_json_parser_66: @param[in] f the substring to replace with @a t @param[in] t the string to replace @a f - @pre The search string @a f must not be empty. + @pre The search string @a f must not be empty. **This precondition is + enforced with an assertion.** @since version 2.0.0 */ @@ -11484,12 +13156,17 @@ basic_json_parser_66: @param[in] value flattened JSON @return unflattened JSON + + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened */ static basic_json unflatten(const basic_json& value) { if (not value.is_object()) { - throw std::domain_error("only objects can be unflattened"); + JSON_THROW(type_error::create(314, "only objects can be unflattened")); } basic_json result; @@ -11499,7 +13176,7 @@ basic_json_parser_66: { if (not element.second.is_primitive()) { - throw std::domain_error("values in object must be primitive"); + JSON_THROW(type_error::create(315, "values in object must be primitive")); } // assign value to reference pointed to by JSON pointer; Note @@ -11513,7 +13190,18 @@ basic_json_parser_66: return result; } - private: + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return lhs.reference_tokens == rhs.reference_tokens; + } + + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return !(lhs == rhs); + } + /// the reference tokens std::vector reference_tokens {}; }; @@ -11550,9 +13238,9 @@ basic_json_parser_66: @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved @liveexample{The behavior is shown in the example.,operatorjson_pointer} @@ -11577,9 +13265,10 @@ basic_json_parser_66: @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} @@ -11600,15 +13289,30 @@ basic_json_parser_66: @return reference to the element pointed to by @a ptr - @complexity Constant. + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. - @liveexample{The behavior is shown in the example.,at_json_pointer} + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. @since version 2.0.0 + + @liveexample{The behavior is shown in the example.,at_json_pointer} */ reference at(const json_pointer& ptr) { @@ -11625,15 +13329,30 @@ basic_json_parser_66: @return reference to the element pointed to by @a ptr - @complexity Constant. + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. - @liveexample{The behavior is shown in the example.,at_json_pointer_const} + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. @since version 2.0.0 + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} */ const_reference at(const json_pointer& ptr) const { @@ -11648,7 +13367,7 @@ basic_json_parser_66: primitive. The original JSON value can be restored using the @ref unflatten() function. - @return an object that maps JSON pointers to primitve values + @return an object that maps JSON pointers to primitive values @note Empty objects and arrays are flattened to `null` and will not be reconstructed correctly by the @ref unflatten() function. @@ -11689,6 +13408,9 @@ basic_json_parser_66: @complexity Linear in the size the JSON value. + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitve + @liveexample{The following code shows how a flattened JSON object is unflattened into the original nested JSON object.,unflatten} @@ -11715,7 +13437,7 @@ basic_json_parser_66: [JSON Patch](http://jsonpatch.com) defines a JSON document structure for expressing a sequence of operations to apply to a JSON) document. With - this funcion, a JSON Patch is applied to the current JSON value by + this function, a JSON Patch is applied to the current JSON value by executing all operations from the patch. @param[in] json_patch JSON patch document @@ -11726,12 +13448,23 @@ basic_json_parser_66: any case, the original value is not changed: the patch is applied to a copy of the value. - @throw std::out_of_range if a JSON pointer inside the patch could not - be resolved successfully in the current JSON value; example: `"key baz - not found"` - @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + @throw parse_error.104 if the JSON patch does not consist of an array of + objects + + @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory attributes are missing); example: `"operation add must have member path"` + @throw out_of_range.401 if an array index is out of range. + + @throw out_of_range.403 if a JSON pointer inside the patch could not be + resolved successfully in the current JSON value; example: `"key baz not + found"` + + @throw out_of_range.405 if JSON pointer has no parent ("add", "remove", + "move") + + @throw other_error.501 if "test" operation was unsuccessful + @complexity Linear in the size of the JSON value and the length of the JSON patch. As usually only a fraction of the JSON value is affected by the patch, the complexity can usually be neglected. @@ -11754,7 +13487,7 @@ basic_json_parser_66: // the valid JSON Patch operations enum class patch_operations {add, remove, replace, move, copy, test, invalid}; - const auto get_op = [](const std::string op) + const auto get_op = [](const std::string & op) { if (op == "add") { @@ -11828,7 +13561,7 @@ basic_json_parser_66: if (static_cast(idx) > parent.size()) { // avoid undefined behavior - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } else { @@ -11866,7 +13599,7 @@ basic_json_parser_66: } else { - throw std::out_of_range("key '" + last_path + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found")); } } else if (parent.is_array()) @@ -11876,14 +13609,13 @@ basic_json_parser_66: } }; - // type check + // type check: top level value must be an array if (not json_patch.is_array()) { - // a JSON patch must be an array of objects - throw std::invalid_argument("JSON patch must be an array of objects"); + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); } - // iterate and apply th eoperations + // iterate and apply the operations for (const auto& val : json_patch) { // wrapper to get a value for an operation @@ -11900,23 +13632,23 @@ basic_json_parser_66: // check if desired value is present if (it == val.m_value.object->end()) { - throw std::invalid_argument(error_msg + " must have member '" + member + "'"); + JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'")); } // check if result is of type string if (string_type and not it->second.is_string()) { - throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); + JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'")); } // no error: return value return it->second; }; - // type check + // type check: every element of the array must be an object if (not val.is_object()) { - throw std::invalid_argument("JSON patch must be an array of objects"); + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); } // collect mandatory members @@ -11975,13 +13707,13 @@ basic_json_parser_66: case patch_operations::test: { bool success = false; - try + JSON_TRY { // check if "value" matches the one at "path" // the "path" location must exist - use at() success = (result.at(ptr) == get_value("test", "value", false)); } - catch (std::out_of_range&) + JSON_CATCH (out_of_range&) { // ignore out of range errors: success remains false } @@ -11989,7 +13721,7 @@ basic_json_parser_66: // throw an exception if test fails if (not success) { - throw std::domain_error("unsuccessful: " + val.dump()); + JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump())); } break; @@ -11999,7 +13731,7 @@ basic_json_parser_66: { // op must be "add", "remove", "replace", "move", "copy", or // "test" - throw std::invalid_argument("operation value '" + op + "' is invalid"); + JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid")); } } } @@ -12022,8 +13754,8 @@ basic_json_parser_66: @note Currently, only `remove`, `add`, and `replace` operations are generated. - @param[in] source JSON value to copare from - @param[in] target JSON value to copare against + @param[in] source JSON value to compare from + @param[in] target JSON value to compare against @param[in] path helper value to create JSON pointers @return a JSON patch to convert the @a source to @a target @@ -12174,7 +13906,6 @@ basic_json_parser_66: /// @} }; - ///////////// // presets // ///////////// @@ -12188,7 +13919,7 @@ uses the standard template types. @since version 1.0.0 */ using json = basic_json<>; -} +} // namespace nlohmann /////////////////////// @@ -12229,7 +13960,23 @@ struct hash return h(j.dump()); } }; -} + +/// specialization for std::less +template <> +struct less<::nlohmann::detail::value_t> +{ + /*! + @brief compare two value_t enum values + @since version 3.0.0 + */ + bool operator()(nlohmann::detail::value_t lhs, + nlohmann::detail::value_t rhs) const noexcept + { + return nlohmann::detail::operator<(lhs, rhs); + } +}; + +} // namespace std /*! @brief user-defined string literal for JSON values @@ -12271,5 +14018,14 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop #endif +#if defined(__clang__) + #pragma GCC diagnostic pop +#endif + +// clean up +#undef JSON_CATCH +#undef JSON_THROW +#undef JSON_TRY +#undef JSON_DEPRECATED #endif diff --git a/make-linux.mk b/make-linux.mk index 811d4a6e..5fb489bb 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -8,7 +8,7 @@ ifeq ($(origin CXX),default) endif INCLUDES?= -DEFS?=-D_FORTIFY_SOURCE=2 +DEFS?= LDLIBS?= DESTDIR?= @@ -55,14 +55,15 @@ endif ifeq ($(ZT_DEBUG),1) override DEFS+=-DZT_TRACE - override CFLAGS+=-Wall -g -O -pthread $(INCLUDES) $(DEFS) - override CXXFLAGS+=-Wall -g -O -std=c++11 -pthread $(INCLUDES) $(DEFS) + override CFLAGS+=-Wall -g -pthread $(INCLUDES) $(DEFS) + override CXXFLAGS+=-Wall -g -std=c++11 -pthread $(INCLUDES) $(DEFS) override LDFLAGS+= STRIP?=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box! -node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CXXFLAGS=-Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else + override DEFS+=-D_FORTIFY_SOURCE=2 CFLAGS?=-O3 -fstack-protector override CFLAGS+=-Wall -fPIE -pthread $(INCLUDES) -DNDEBUG $(DEFS) CXXFLAGS?=-O3 -fstack-protector diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 93be64dd..f01da38e 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -72,6 +72,8 @@ public: _thingCount(0), _issuedTo(issuedTo) { + memset(_thingTypes,0,sizeof(_thingTypes)); + memset(_thingValues,0,sizeof(_thingValues)); } inline uint64_t networkId() const { return _networkId; } diff --git a/root-watcher/schema.sql b/root-watcher/schema.sql index bdb3a1cf..ade0fa3e 100644 --- a/root-watcher/schema.sql +++ b/root-watcher/schema.sql @@ -1,7 +1,6 @@ /* Schema for ZeroTier root watcher log database */ -/* If you cluster this DB using any PG clustering scheme that uses logs, you must remove UNLOGGED here! */ -CREATE UNLOGGED TABLE "Peer" +CREATE TABLE "Peer" ( "ztAddress" BIGINT NOT NULL, "timestamp" BIGINT NOT NULL, diff --git a/selftest.cpp b/selftest.cpp index 33e65f2c..e23afd6e 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -49,7 +49,6 @@ #include "osdep/OSUtils.hpp" #include "osdep/Phy.hpp" -#include "osdep/Http.hpp" #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" @@ -1019,51 +1018,6 @@ static int testPhy() return 0; } -/* -static int testHttp() -{ - std::map requestHeaders,responseHeaders; - std::string responseBody; - - InetAddress downloadZerotierDotCom; - std::vector rr(OSUtils::resolve("download.zerotier.com")); - if (rr.empty()) { - std::cout << "[http] Resolve of download.zerotier.com failed, skipping." << std::endl; - return 0; - } else { - for(std::vector::iterator r(rr.begin());r!=rr.end();++r) { - std::cout << "[http] download.zerotier.com: " << r->toString() << std::endl; - if (r->isV4()) - downloadZerotierDotCom = *r; - } - } - downloadZerotierDotCom.setPort(80); - - std::cout << "[http] GET http://download.zerotier.com/dev/1k @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - requestHeaders["Host"] = "download.zerotier.com"; - unsigned int sc = Http::GET(1024 * 1024 * 16,60000,reinterpret_cast(&downloadZerotierDotCom),"/dev/1k",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " " << responseBody.length() << " bytes "; - if (sc == 0) - std::cout << "ERROR: " << responseBody << std::endl; - else std::cout << "DONE" << std::endl; - - std::cout << "[http] GET http://download.zerotier.com/dev/4m @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - requestHeaders["Host"] = "download.zerotier.com"; - sc = Http::GET(1024 * 1024 * 16,60000,reinterpret_cast(&downloadZerotierDotCom),"/dev/4m",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " " << responseBody.length() << " bytes "; - if (sc == 0) - std::cout << "ERROR: " << responseBody << std::endl; - else std::cout << "DONE" << std::endl; - - downloadZerotierDotCom = InetAddress("1.0.0.1/1234"); - std::cout << "[http] GET @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - sc = Http::GET(1024 * 1024 * 16,2500,reinterpret_cast(&downloadZerotierDotCom),"/dev/4m",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " (should be 0, time out)" << std::endl; - - return 0; -} -*/ - #ifdef __WINDOWS__ int __cdecl _tmain(int argc, _TCHAR* argv[]) #else @@ -1127,7 +1081,6 @@ int main(int argc,char **argv) r |= testIdentity(); r |= testCertificate(); r |= testPhy(); - //r |= testHttp(); //*/ if (r) -- cgit v1.2.3 From 62578a21625025a2e2681c8da6126270ead47aa7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 27 Apr 2017 19:36:03 -0700 Subject: Reduce memory use and a bit of cleanup. --- controller/EmbeddedNetworkController.cpp | 28 ++++++++------ controller/EmbeddedNetworkController.hpp | 11 ++++++ controller/JSONDB.cpp | 63 ++++++++++++++++++-------------- controller/JSONDB.hpp | 8 ++-- 4 files changed, 67 insertions(+), 43 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index a41a4a94..61aa23ae 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -719,6 +719,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["lastModified"] = now; json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + _removeMemberNonPersistedFields(member); _db.saveNetworkMember(nwid,address,member); _pushMemberUpdate(now,nwid,member); } @@ -1017,10 +1018,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy - if (true) { + if (network != origNetwork) { json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); network["lastModified"] = now; + _removeNetworkNonPersistedFields(network); _db.saveNetwork(nwid,network); // Send an update to all members of the network @@ -1187,6 +1189,10 @@ void EmbeddedNetworkController::_request( const Identity &identity, const Dictionary &metaData) { + char nwids[24]; + JSONDB::NetworkSummaryInfo ns; + json network,member,origMember; + if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) return; @@ -1200,17 +1206,13 @@ void EmbeddedNetworkController::_request( lrt = now; } - char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network,member; - JSONDB::NetworkSummaryInfo ns; if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; } - - const bool newMember = (member.size() == 0); - json origMember(member); // for detecting modification later + origMember = member; + const bool newMember = ((!member.is_object())||(member.size() == 0)); _initMember(member); { @@ -1235,10 +1237,12 @@ void EmbeddedNetworkController::_request( } // These are always the same, but make sure they are set - const std::string addrs(identity.address().toString()); - member["id"] = addrs; - member["address"] = addrs; - member["nwid"] = nwids; + { + const std::string addrs(identity.address().toString()); + member["id"] = addrs; + member["address"] = addrs; + member["nwid"] = nwids; + } // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; @@ -1347,6 +1351,7 @@ void EmbeddedNetworkController::_request( if (!authorizedBy) { if (origMember != member) { member["lastModified"] = now; + _removeMemberNonPersistedFields(member); _db.saveNetworkMember(nwid,identity.address().toInt(),member); } _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); @@ -1697,6 +1702,7 @@ void EmbeddedNetworkController::_request( if (member != origMember) { member["lastModified"] = now; + _removeMemberNonPersistedFields(member); _db.saveNetworkMember(nwid,identity.address().toInt(),member); } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 8a220139..64ea6884 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -157,10 +157,21 @@ private: network["activeMemberCount"] = ns.activeMemberCount; network["totalMemberCount"] = ns.totalMemberCount; } + inline void _removeNetworkNonPersistedFields(nlohmann::json &network) + { + network.erase("clock"); + network.erase("authorizedMemberCount"); + network.erase("activeMemberCount"); + network.erase("totalMemberCount"); + } inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now) { member["clock"] = now; } + inline void _removeMemberNonPersistedFields(nlohmann::json &member) + { + member.erase("clock"); + } const uint64_t _startTime; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 01eb50cd..8cd957b6 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -110,24 +110,23 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) bool JSONDB::hasNetwork(const uint64_t networkId) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - return (i != _networks.end()); + return (_networks.find(networkId) != _networks.end()); } bool JSONDB::getNetwork(const uint64_t networkId,nlohmann::json &config) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return false; - config = i->second.config; + config = nlohmann::json::from_msgpack(i->second.config); return true; } bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return false; ns = i->second.summaryInfo; @@ -137,14 +136,14 @@ bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo & int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return 0; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + const std::unordered_map< uint64_t,std::vector >::const_iterator j(i->second.members.find(nodeId)); if (j == i->second.members.end()) return 1; - networkConfig = i->second.config; - memberConfig = j->second; + networkConfig = nlohmann::json::from_msgpack(i->second.config); + memberConfig = nlohmann::json::from_msgpack(j->second); ns = i->second.summaryInfo; return 3; } @@ -152,13 +151,13 @@ int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,n bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return false; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + const std::unordered_map< uint64_t,std::vector >::const_iterator j(i->second.members.find(nodeId)); if (j == i->second.members.end()) return false; - memberConfig = j->second; + memberConfig = nlohmann::json::from_msgpack(j->second); return true; } @@ -169,7 +168,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC writeRaw(n,OSUtils::jsonDump(networkConfig)); { Mutex::Lock _l(_networks_m); - _networks[networkId].config = networkConfig; + _networks[networkId].config = nlohmann::json::to_msgpack(networkConfig); } _recomputeSummaryInfo(networkId); } @@ -181,7 +180,7 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co writeRaw(n,OSUtils::jsonDump(memberConfig)); { Mutex::Lock _l(_networks_m); - _networks[networkId].members[nodeId] = memberConfig; + _networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig); } _recomputeSummaryInfo(networkId); } @@ -192,10 +191,10 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) std::vector memberIds; { Mutex::Lock _l(_networks_m); - std::unordered_map::iterator i(_networks.find(networkId)); + const std::unordered_map::iterator i(_networks.find(networkId)); if (i == _networks.end()) return _EMPTY_JSON; - for(std::unordered_map::iterator m(i->second.members.begin());m!=i->second.members.end();++m) + for(std::unordered_map< uint64_t,std::vector >::iterator m(i->second.members.begin());m!=i->second.members.end();++m) memberIds.push_back(m->first); } for(std::vector::iterator m(memberIds.begin());m!=memberIds.end();++m) @@ -221,7 +220,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) std::unordered_map::iterator i(_networks.find(networkId)); if (i == _networks.end()) return _EMPTY_JSON; // sanity check, shouldn't happen - nlohmann::json tmp(i->second.config); + nlohmann::json tmp(nlohmann::json::from_msgpack(i->second.config)); _networks.erase(i); return tmp; } @@ -248,7 +247,7 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ std::unordered_map::iterator i(_networks.find(networkId)); if (i == _networks.end()) return _EMPTY_JSON; - std::unordered_map::iterator j(i->second.members.find(nodeId)); + std::unordered_map< uint64_t,std::vector >::iterator j(i->second.members.find(nodeId)); if (j == i->second.members.end()) return _EMPTY_JSON; nlohmann::json tmp(j->second); @@ -288,13 +287,15 @@ void JSONDB::threadMain() ns.totalMemberCount = 0; ns.mostRecentDeauthTime = 0; - for(std::unordered_map::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { try { - if (OSUtils::jsonBool(m->second["authorized"],false)) { + nlohmann::json member(nlohmann::json::from_msgpack(m->second)); + + if (OSUtils::jsonBool(member["authorized"],false)) { ++ns.authorizedMemberCount; try { - const nlohmann::json &mlog = m->second["recentLog"]; + const nlohmann::json &mlog = member["recentLog"]; if ((mlog.is_array())&&(mlog.size() > 0)) { const nlohmann::json &mlog1 = mlog[0]; if (mlog1.is_object()) { @@ -305,12 +306,12 @@ void JSONDB::threadMain() } catch ( ... ) {} try { - if (OSUtils::jsonBool(m->second["activeBridge"],false)) + if (OSUtils::jsonBool(member["activeBridge"],false)) ns.activeBridges.push_back(Address(m->first)); } catch ( ... ) {} try { - const nlohmann::json &mips = m->second["ipAssignments"]; + const nlohmann::json &mips = member["ipAssignments"]; if (mips.is_array()) { for(unsigned long i=0;isecond["lastDeauthorizedTime"],0ULL)); + ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,OSUtils::jsonInt(member["lastDeauthorizedTime"],0ULL)); } catch ( ... ) {} } ++ns.totalMemberCount; @@ -342,6 +343,8 @@ void JSONDB::threadMain() bool JSONDB::_load(const std::string &p) { if (_httpAddr) { + // In HTTP harnessed mode we download our entire working data set on startup. + std::string body; std::map headers; const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); @@ -360,12 +363,12 @@ bool JSONDB::_load(const std::string &p) if ((id.length() == 16)&&(objtype == "network")) { const uint64_t nwid = Utils::hexStrToU64(id.c_str()); if (nwid) - _networks[nwid].config = j; + _networks[nwid].config = nlohmann::json::to_msgpack(j); } else if ((id.length() == 10)&&(objtype == "member")) { const uint64_t mid = Utils::hexStrToU64(id.c_str()); const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); if ((mid)&&(nwid)) - _networks[nwid].members[mid] = j; + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); } } } @@ -374,7 +377,10 @@ bool JSONDB::_load(const std::string &p) } catch ( ... ) {} // invalid JSON, so maybe incomplete request } return false; + } else { + // In regular mode we recursively read it from controller.d/ on disk + std::vector dl(OSUtils::listDirectory(p.c_str(),true)); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { @@ -389,14 +395,14 @@ bool JSONDB::_load(const std::string &p) const uint64_t nwid = Utils::strToU64(id.c_str()); if (nwid) { Mutex::Lock _l(_networks_m); - _networks[nwid].config = j; + _networks[nwid].config = nlohmann::json::to_msgpack(j); } } else if ((id.length() == 10)&&(objtype == "member")) { const uint64_t mid = Utils::strToU64(id.c_str()); const uint64_t nwid = Utils::strToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); if ((mid)&&(nwid)) { Mutex::Lock _l(_networks_m); - _networks[nwid].members[mid] = j; + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); } } } catch ( ... ) {} @@ -406,6 +412,7 @@ bool JSONDB::_load(const std::string &p) } } return true; + } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 8db5cc48..ba16d97b 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -99,9 +99,9 @@ public: Mutex::Lock _l(_networks_m); std::unordered_map::const_iterator i(_networks.find(networkId)); if (i != _networks.end()) { - for(std::unordered_map::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { try { - func(networkId,m->first,m->second); + func(networkId,m->first,nlohmann::json::from_msgpack(m->second)); } catch ( ... ) {} } } @@ -126,10 +126,10 @@ private: struct _NW { _NW() : summaryInfoLastComputed(0) {} - nlohmann::json config; + std::vector config; NetworkSummaryInfo summaryInfo; uint64_t summaryInfoLastComputed; - std::unordered_map members; + std::unordered_map< uint64_t,std::vector > members; }; std::unordered_map _networks; -- cgit v1.2.3 From a9ce77358484e41cd6bac42594e4eeb045a788cb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 28 Apr 2017 19:58:21 -0700 Subject: Remove lastModified field in config which literally nothing uses anywhere, and prevent some unnecessary writes. --- controller/EmbeddedNetworkController.cpp | 23 ++++++++--------------- controller/migrate-sqlite/migrate.js | 2 -- 2 files changed, 8 insertions(+), 17 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 61aa23ae..70640ff5 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -715,20 +715,18 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["address"] = addrs; // legacy member["nwid"] = nwids; + _removeMemberNonPersistedFields(member); if (member != origMember) { - member["lastModified"] = now; json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - _removeMemberNonPersistedFields(member); _db.saveNetworkMember(nwid,address,member); _pushMemberUpdate(now,nwid,member); } - // Add non-persisted fields - member["clock"] = now; - + _addMemberNonPersistedFields(member,now); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; + return 200; } else if ((path.size() == 3)&&(path[2] == "test")) { @@ -1018,11 +1016,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy + _removeNetworkNonPersistedFields(network); if (network != origNetwork) { json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - network["lastModified"] = now; - _removeNetworkNonPersistedFields(network); _db.saveNetwork(nwid,network); // Send an update to all members of the network @@ -1349,11 +1346,9 @@ void EmbeddedNetworkController::_request( // If they are not authorized, STOP! if (!authorizedBy) { - if (origMember != member) { - member["lastModified"] = now; - _removeMemberNonPersistedFields(member); + _removeMemberNonPersistedFields(member); + if (origMember != member) _db.saveNetworkMember(nwid,identity.address().toInt(),member); - } _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); return; } @@ -1700,11 +1695,9 @@ void EmbeddedNetworkController::_request( return; } - if (member != origMember) { - member["lastModified"] = now; - _removeMemberNonPersistedFields(member); + _removeMemberNonPersistedFields(member); + if (member != origMember) _db.saveNetworkMember(nwid,identity.address().toInt(),member); - } _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } diff --git a/controller/migrate-sqlite/migrate.js b/controller/migrate-sqlite/migrate.js index ac9678a7..2f2462c9 100644 --- a/controller/migrate-sqlite/migrate.js +++ b/controller/migrate-sqlite/migrate.js @@ -95,7 +95,6 @@ async.series([function(nextStep) { creationTime: parseInt(row.creationTime)||0, enableBroadcast: !!row.enableBroadcast, ipAssignmentPools: [], - lastModified: Date.now(), multicastLimit: row.multicastLimit||32, name: row.name||'', private: !!row.private, @@ -177,7 +176,6 @@ async.series([function(nextStep) { ipAssignments: [], lastAuthorizedTime: (row.authorized) ? Date.now() : 0, lastDeauthorizedTime: (row.authorized) ? 0 : Date.now(), - lastModified: Date.now(), lastRequestMetaData: '', noAutoAssignIps: false, nwid: row.networkId, -- cgit v1.2.3 From 718e1d6c082453bfbab8b900f5ffde42047fc814 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 1 May 2017 13:21:26 -0700 Subject: Finish removing constantly changing stuff from controller. --- controller/EmbeddedNetworkController.cpp | 160 ++++++++++++++++++------------- controller/EmbeddedNetworkController.hpp | 34 ++++++- controller/JSONDB.hpp | 13 +++ controller/README.md | 22 ++--- node/Dictionary.hpp | 2 + 5 files changed, 143 insertions(+), 88 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 70640ff5..3ec02454 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -518,7 +518,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( json member; if (!_db.getNetworkMember(nwid,address,member)) return 404; - _addMemberNonPersistedFields(member,OSUtils::now()); + _addMemberNonPersistedFields(nwid,address,member,OSUtils::now()); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; } else { @@ -569,6 +569,26 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } // else 404 + } else if ((path.size() == 1)&&(path[0] == "memberStatus")) { + + const uint64_t now = OSUtils::now(); + Mutex::Lock _l(_memberStatus_m); + responseBody.push_back('{'); + _db.eachId([this,&responseBody,&now](uint64_t networkId,uint64_t nodeId) { + char tmp[64]; + auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); + Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%s", + (responseBody.length() > 1) ? "," : "", + (unsigned long long)networkId, + (unsigned long long)nodeId, + ((ms != _memberStatus.end())&&(ms->second.online(now))) ? "true" : "false"); + responseBody.append(tmp); + }); + responseBody.push_back('}'); + responseContentType = "application/json"; + + return 200; + } else { char tmp[4096]; @@ -649,10 +669,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!newAuth) { Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); - Mutex::Lock _l(_lastRequestTime_m); - for(std::map< std::pair,uint64_t >::iterator i(_lastRequestTime.begin());i!=_lastRequestTime.end();++i) { - if ((now - i->second) < ZT_NETWORK_AUTOCONF_DELAY) - _node->ncSendRevocation(Address(i->first.first),rev); + + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == nwid)&&(i->second.online(now))) + _node->ncSendRevocation(Address(i->first.nodeId),rev); } } } @@ -720,10 +741,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetworkMember(nwid,address,member); - _pushMemberUpdate(now,nwid,member); + + // Push update to member if online + try { + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,address)]; + if ((ms.online(now))&&(ms.lastRequestMetaData)) + request(nwid,InetAddress(),0,ms.identity,ms.lastRequestMetaData); + } catch ( ... ) {} } - _addMemberNonPersistedFields(member,now); + _addMemberNonPersistedFields(nwid,address,member,now); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; @@ -1022,10 +1050,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetwork(nwid,network); - // Send an update to all members of the network - _db.eachMember(nwid,[this,&now,&nwid](uint64_t networkId,uint64_t nodeId,const json &obj) { - this->_pushMemberUpdate(now,nwid,obj); - }); + // Send an update to all members of the network that are online + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == nwid)&&(i->second.online(now))&&(i->second.lastRequestMetaData)) + request(nwid,InetAddress(),0,i->second.identity,i->second.lastRequestMetaData); + } } JSONDB::NetworkSummaryInfo ns; @@ -1044,7 +1074,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json testRec; const uint64_t now = OSUtils::now(); testRec["clock"] = now; - testRec["uptime"] = (now - _startTime); + testRec["startTime"] = _startTime; testRec["content"] = b; responseBody = OSUtils::jsonDump(testRec); _db.writeRaw("pong",responseBody); @@ -1075,6 +1105,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( const uint64_t address = Utils::hexStrToU64(path[3].c_str()); json member = _db.eraseNetworkMember(nwid,address); + + { + Mutex::Lock _l(_memberStatus_m); + _memberStatus.erase(_MemberStatusKey(nwid,address)); + } + if (!member.size()) return 404; responseBody = OSUtils::jsonDump(member); @@ -1083,6 +1119,16 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( } } else { json network = _db.eraseNetwork(nwid); + + { + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();) { + if (i->first.networkId == nwid) + _memberStatus.erase(i++); + else ++i; + } + } + if (!network.size()) return 404; responseBody = OSUtils::jsonDump(network); @@ -1106,6 +1152,7 @@ void EmbeddedNetworkController::threadMain() _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); } catch ( ... ) {} delete qe; + if (_running) { uint64_t now = OSUtils::now(); if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { @@ -1196,11 +1243,11 @@ void EmbeddedNetworkController::_request( const uint64_t now = OSUtils::now(); if (requestPacketId) { - Mutex::Lock _l(_lastRequestTime_m); - uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; - if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())]; + if ((now - ms.lastRequestTime) <= ZT_NETCONF_MIN_REQUEST_PERIOD) return; - lrt = now; + ms.lastRequestTime = now; } Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); @@ -1315,37 +1362,37 @@ void EmbeddedNetworkController::_request( member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } - // Log this request - if (requestPacketId) { // only log if this is a request, not for generated pushes - json rlEntry = json::object(); - rlEntry["ts"] = now; - rlEntry["auth"] = (authorizedBy) ? true : false; - rlEntry["authBy"] = (authorizedBy) ? authorizedBy : ""; - rlEntry["vMajor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["vMinor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["vRev"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["vProto"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - if (fromAddr) - rlEntry["fromAddr"] = fromAddr.toString(); - - json recentLog = json::array(); - recentLog.push_back(rlEntry); - json &oldLog = member["recentLog"]; - if (oldLog.is_array()) { - for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) - break; - } - } - member["recentLog"] = recentLog; + if (authorizedBy) { + // Update version info and meta-data if authorized and if this is a genuine request + if (requestPacketId) { + const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + const uint64_t vMinor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + const uint64_t vRev = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + const uint64_t vProto = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - // Also only do this on real requests - member["lastRequestMetaData"] = metaData.data(); - } + member["vMajor"] = vMajor; + member["vMinor"] = vMinor; + member["vRev"] = vRev; + member["vProto"] = vProto; - // If they are not authorized, STOP! - if (!authorizedBy) { + { + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())]; + + ms.vMajor = (int)vMajor; + ms.vMinor = (int)vMinor; + ms.vRev = (int)vRev; + ms.vProto = (int)vProto; + ms.lastRequestMetaData = metaData; + ms.identity = identity; + + if (fromAddr) + ms.physicalAddr = fromAddr; + member["physicalAddr"] = ms.physicalAddr.toString(); + } + } + } else { + // If they are not authorized, STOP! _removeMemberNonPersistedFields(member); if (origMember != member) _db.saveNetworkMember(nwid,identity.address().toInt(),member); @@ -1702,27 +1749,4 @@ void EmbeddedNetworkController::_request( _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } -void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member) -{ - try { - const std::string idstr = member["identity"]; - const std::string mdstr = member["lastRequestMetaData"]; - if ((idstr.length() > 0)&&(mdstr.length() > 0)) { - const Identity id(idstr); - bool online; - { - Mutex::Lock _l(_lastRequestTime_m); - std::map< std::pair,uint64_t >::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); - online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); - } - if (online) { - Dictionary metaData(mdstr.c_str()); - try { - this->request(nwid,InetAddress(),0,id,metaData); - } catch ( ... ) {} - } - } - } catch ( ... ) {} -} - } // namespace ZeroTier diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 64ea6884..faf7f029 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -107,7 +107,6 @@ private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); - void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); // These init objects with default and static/informational fields inline void _initMember(nlohmann::json &member) @@ -164,9 +163,11 @@ private: network.erase("activeMemberCount"); network.erase("totalMemberCount"); } - inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now) + inline void _addMemberNonPersistedFields(uint64_t nwid,uint64_t nodeId,nlohmann::json &member,uint64_t now) { member["clock"] = now; + Mutex::Lock _l(_memberStatus_m); + member["online"] = _memberStatus[_MemberStatusKey(nwid,nodeId)].online(now); } inline void _removeMemberNonPersistedFields(nlohmann::json &member) { @@ -191,8 +192,33 @@ private: std::list< ZT_CircuitTest > _tests; Mutex _tests_m; - std::map< std::pair,uint64_t > _lastRequestTime; // last request time by - Mutex _lastRequestTime_m; + struct _MemberStatusKey + { + _MemberStatusKey() : networkId(0),nodeId(0) {} + _MemberStatusKey(const uint64_t nwid,const uint64_t nid) : networkId(nwid),nodeId(nid) {} + uint64_t networkId; + uint64_t nodeId; + inline bool operator==(const _MemberStatusKey &k) const { return ((k.networkId == networkId)&&(k.nodeId == nodeId)); } + }; + struct _MemberStatus + { + _MemberStatus() : lastRequestTime(0),vMajor(-1),vMinor(-1),vRev(-1),vProto(-1) {} + uint64_t lastRequestTime; + int vMajor,vMinor,vRev,vProto; + Dictionary lastRequestMetaData; + Identity identity; + InetAddress physicalAddr; // last known physical address + inline bool online(const uint64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); } + }; + struct _MemberStatusHash + { + inline std::size_t operator()(const _MemberStatusKey &networkIdNodeId) const + { + return (std::size_t)(networkIdNodeId.networkId + networkIdNodeId.nodeId); + } + }; + std::unordered_map< _MemberStatusKey,_MemberStatus,_MemberStatusHash > _memberStatus; + Mutex _memberStatus_m; }; } // namespace ZeroTier diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index ba16d97b..530f9632 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -107,6 +107,19 @@ public: } } + template + inline void eachId(F func) + { + Mutex::Lock _l(_networks_m); + for(std::unordered_map::const_iterator i(_networks.begin());i!=_networks.end();++i) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { + try { + func(i->first,m->first); + } catch ( ... ) {} + } + } + } + void threadMain() throw(); diff --git a/controller/README.md b/controller/README.md index db8d0153..3519eb11 100644 --- a/controller/README.md +++ b/controller/README.md @@ -227,22 +227,12 @@ This returns an object containing all currently online members and the most rece | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | | ipAssignments | array[string] | Managed IP address assignments | YES | -| memberRevision | integer | Member revision counter | no | -| recentLog | array[object] | Recent member activity log; see below | no | +| revision | integer | Member revision counter | no | +| vMajor | integer | Most recently known major version | no | +| vMinor | integer | Most recently known minor version | no | +| vRev | integer | Most recently known revision | no | +| vProto | integer | Most recently known protocl version | no | +| physicalAddr | string | Last known physical IP/port or null if none | no | Note that managed IP assignments are only used if they fall within a managed route. Otherwise they are ignored. -**Recent log object format:** - -| Field | Type | Description | -| --------------------- | ------------- | ------------------------------------------------- | -| ts | integer | Time of request, ms since epoch | -| auth | boolean | Was member authorized? | -| authBy | string | How was member authorized? | -| vMajor | integer | Client major version or -1 if unknown | -| vMinor | integer | Client minor version or -1 if unknown | -| vRev | integer | Client revision or -1 if unknown | -| vProto | integer | ZeroTier protocol version reported by client | -| fromAddr | string | Physical address if known | - -The controller can only know a member's `fromAddr` if it's able to establish a direct path to it. Members behind very restrictive firewalls may not have this information since the controller will be receiving the member's requests by way of a relay. ZeroTier does not back-trace IP paths as packets are relayed since this would add a lot of protocol overhead. diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index e212e453..4413d628 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -99,6 +99,8 @@ public: return *this; } + inline operator bool() const { return (_d[0] != 0); } + /** * Load a dictionary from a C-string * -- cgit v1.2.3 From bcc67999023c06f97402b4eef70443b39e091e65 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 1 May 2017 15:23:21 -0700 Subject: Send member uptime in pong posts. --- controller/EmbeddedNetworkController.cpp | 49 +++++++++++++++----------------- controller/EmbeddedNetworkController.hpp | 12 +++++++- 2 files changed, 34 insertions(+), 27 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3ec02454..514e06d1 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -569,26 +569,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } // else 404 - } else if ((path.size() == 1)&&(path[0] == "memberStatus")) { - - const uint64_t now = OSUtils::now(); - Mutex::Lock _l(_memberStatus_m); - responseBody.push_back('{'); - _db.eachId([this,&responseBody,&now](uint64_t networkId,uint64_t nodeId) { - char tmp[64]; - auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); - Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%s", - (responseBody.length() > 1) ? "," : "", - (unsigned long long)networkId, - (unsigned long long)nodeId, - ((ms != _memberStatus.end())&&(ms->second.online(now))) ? "true" : "false"); - responseBody.append(tmp); - }); - responseBody.push_back('}'); - responseContentType = "application/json"; - - return 200; - } else { char tmp[4096]; @@ -1071,14 +1051,31 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } else if (path[0] == "ping") { - json testRec; const uint64_t now = OSUtils::now(); - testRec["clock"] = now; - testRec["startTime"] = _startTime; - testRec["content"] = b; - responseBody = OSUtils::jsonDump(testRec); - _db.writeRaw("pong",responseBody); + bool first = true; + std::string pong("{\"memberStatus\":{"); + { + Mutex::Lock _l(_memberStatus_m); + _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { + char tmp[64]; + auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); + Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%s", + (first) ? "" : ",", + (unsigned long long)networkId, + (unsigned long long)nodeId, + ((ms != _memberStatus.end())&&(ms->second.online(now))) ? "true" : "false"); + pong.append(tmp); + first = false; + }); + } + char tmp2[256]; + Utils::snprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + pong.append(tmp2); + _db.writeRaw("pong",pong); + + responseBody = "{}"; responseContentType = "application/json"; + return 200; } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index faf7f029..0a6b8176 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -114,7 +114,6 @@ private: if (!member.count("authorized")) member["authorized"] = false; if (!member.count("authHistory")) member["authHistory"] = nlohmann::json::array(); if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); - if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array(); if (!member.count("activeBridge")) member["activeBridge"] = false; if (!member.count("tags")) member["tags"] = nlohmann::json::array(); if (!member.count("capabilities")) member["capabilities"] = nlohmann::json::array(); @@ -123,6 +122,11 @@ private: if (!member.count("revision")) member["revision"] = 0ULL; if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL; if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL; + if (!member.count("vMajor")) member["vMajor"] = -1; + if (!member.count("vMinor")) member["vMinor"] = -1; + if (!member.count("vRev")) member["vRev"] = -1; + if (!member.count("vProto")) member["vProto"] = -1; + if (!member.count("physicalAddr")) member["physicalAddr"] = nlohmann::json(); member["objtype"] = "member"; } inline void _initNetwork(nlohmann::json &network) @@ -162,6 +166,8 @@ private: network.erase("authorizedMemberCount"); network.erase("activeMemberCount"); network.erase("totalMemberCount"); + // legacy fields + network.erase("lastModified"); } inline void _addMemberNonPersistedFields(uint64_t nwid,uint64_t nodeId,nlohmann::json &member,uint64_t now) { @@ -172,6 +178,10 @@ private: inline void _removeMemberNonPersistedFields(nlohmann::json &member) { member.erase("clock"); + // legacy fields + member.erase("recentLog"); + member.erase("lastModified"); + member.erase("lastRequestMetaData"); } const uint64_t _startTime; -- cgit v1.2.3 From faf7b81c0173149230bc562ecf09b488f03af02e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 1 May 2017 15:48:52 -0700 Subject: Send last requets time in status, and do not set physicalAddr unless we know one. --- controller/EmbeddedNetworkController.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 514e06d1..9ffc0ea2 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1059,11 +1059,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { char tmp[64]; auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); - Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%s", + Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, - ((ms != _memberStatus.end())&&(ms->second.online(now))) ? "true" : "false"); + (ms != _memberStatus.end()) ? (unsigned long long)ms->second.lastRequestTime : 0ULL); pong.append(tmp); first = false; }); @@ -1385,7 +1385,8 @@ void EmbeddedNetworkController::_request( if (fromAddr) ms.physicalAddr = fromAddr; - member["physicalAddr"] = ms.physicalAddr.toString(); + if (ms.physicalAddr) + member["physicalAddr"] = ms.physicalAddr.toString(); } } } else { -- cgit v1.2.3 From 132643cd4ae03e00e44ab8779ac2779b400fbf25 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 2 May 2017 10:34:33 -0700 Subject: Fix small issue preventing pongs if there are never seen members. --- controller/EmbeddedNetworkController.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 9ffc0ea2..9bcddac8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1058,12 +1058,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Mutex::Lock _l(_memberStatus_m); _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { char tmp[64]; + uint64_t lrt = 0ULL; auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); + if (ms != _memberStatus.end()) + lrt = ms->second.lastRequestTime; Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, - (ms != _memberStatus.end()) ? (unsigned long long)ms->second.lastRequestTime : 0ULL); + (unsigned long long)lrt); pong.append(tmp); first = false; }); -- cgit v1.2.3 From fde99e2fcfda41c352862a7c56911f879179db90 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 2 May 2017 12:08:53 -0700 Subject: Fix for post size limit in Http library. --- controller/JSONDB.cpp | 8 ++++---- osdep/Http.cpp | 35 ++++++++++++++++++----------------- 2 files changed, 22 insertions(+), 21 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 8cd957b6..31c0cc05 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -97,7 +97,7 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) Utils::snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); reqHeaders["Content-Length"] = tmp; reqHeaders["Content-Type"] = "application/json"; - const unsigned int sc = Http::PUT(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); + const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); return (sc == 200); } else { const std::string path(_genPath(n,true)); @@ -208,7 +208,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) // Deletion is currently done by Central in harnessed mode //std::map headers; //std::string body; - //Http::DEL(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); } else { const std::string path(_genPath(n,false)); if (path.length()) @@ -235,7 +235,7 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ // Deletion is currently done by the caller in Central harnessed mode //std::map headers; //std::string body; - //Http::DEL(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); } else { const std::string path(_genPath(n,false)); if (path.length()) @@ -347,7 +347,7 @@ bool JSONDB::_load(const std::string &p) std::string body; std::map headers; - const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + const unsigned int sc = Http::GET(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); if (sc == 200) { try { nlohmann::json dbImg(OSUtils::jsonParse(body)); diff --git a/osdep/Http.cpp b/osdep/Http.cpp index d2540071..f1d3bfe2 100644 --- a/osdep/Http.cpp +++ b/osdep/Http.cpp @@ -112,12 +112,12 @@ struct HttpPhyHandler inline void phyOnTcpWritable(PhySocket *sock,void **uptr) { - if (writePtr < writeSize) { - long n = phy->streamSend(sock,writeBuf + writePtr,writeSize - writePtr,true); + if (writePtr < (unsigned long)writeBuf.length()) { + long n = phy->streamSend(sock,writeBuf.data() + writePtr,(unsigned long)writeBuf.length() - writePtr,true); if (n > 0) writePtr += n; } - if (writePtr >= writeSize) + if (writePtr >= (unsigned long)writeBuf.length()) phy->setNotifyWritable(sock,false); } @@ -135,8 +135,7 @@ struct HttpPhyHandler unsigned long messageSize; unsigned long writePtr; uint64_t lastActivity; - unsigned long writeSize; - char writeBuf[32768]; + std::string writeBuf; unsigned long maxResponseSize; std::map *responseHeaders; @@ -244,24 +243,26 @@ unsigned int Http::_do( handler.lastActivity = OSUtils::now(); try { - handler.writeSize = Utils::snprintf(handler.writeBuf,sizeof(handler.writeBuf),"%s %s HTTP/1.1\r\n",method,path); - for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) - handler.writeSize += Utils::snprintf(handler.writeBuf + handler.writeSize,sizeof(handler.writeBuf) - handler.writeSize,"%s: %s\r\n",h->first.c_str(),h->second.c_str()); - handler.writeSize += Utils::snprintf(handler.writeBuf + handler.writeSize,sizeof(handler.writeBuf) - handler.writeSize,"\r\n"); - if ((requestBody)&&(requestBodyLength)) { - if ((handler.writeSize + requestBodyLength) > sizeof(handler.writeBuf)) { - responseBody = "request too large"; - return 0; - } - memcpy(handler.writeBuf + handler.writeSize,requestBody,requestBodyLength); - handler.writeSize += requestBodyLength; + char tmp[1024]; + Utils::snprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); + handler.writeBuf.append(tmp); + for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) { + Utils::snprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); + handler.writeBuf.append(tmp); } + handler.writeBuf.append("\r\n"); + if ((requestBody)&&(requestBodyLength)) + handler.writeBuf.append((const char *)requestBody,requestBodyLength); } catch ( ... ) { responseBody = "request too large"; return 0; } - handler.maxResponseSize = maxResponseSize; + if (maxResponseSize) { + handler.maxResponseSize = maxResponseSize; + } else { + handler.maxResponseSize = 2147483647; + } handler.responseHeaders = &responseHeaders; handler.responseBody = &responseBody; handler.error = false; -- cgit v1.2.3 From 625e3e8e259637ecf45b2f7738b18c9d33975852 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 2 May 2017 13:53:47 -0700 Subject: Tiny optimization to prealloc string space. --- controller/EmbeddedNetworkController.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 9bcddac8..90dd8f52 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1056,6 +1056,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( std::string pong("{\"memberStatus\":{"); { Mutex::Lock _l(_memberStatus_m); + pong.reserve(64 * _memberStatus.size()); _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { char tmp[64]; uint64_t lrt = 0ULL; -- cgit v1.2.3 From 8e19188f49fe33679ebc565cd5f6d184e7b87842 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 2 May 2017 16:58:51 -0700 Subject: Do the sometimes rather big "pong" in a background worker thread. --- controller/EmbeddedNetworkController.cpp | 79 +++++++++++++++----------------- controller/EmbeddedNetworkController.hpp | 19 ++++++++ 2 files changed, 57 insertions(+), 41 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 90dd8f52..91b8671e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -467,26 +467,14 @@ void EmbeddedNetworkController::request( { if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) return; - - { - Mutex::Lock _l(_threads_m); - if (_threads.size() == 0) { - long hwc = (long)(std::thread::hardware_concurrency() / 2); - if (hwc < 1) - hwc = 1; - else if (hwc > 16) - hwc = 16; - for(long i=0;inwid = nwid; qe->requestPacketId = requestPacketId; qe->fromAddr = fromAddr; qe->identity = identity; qe->metaData = metaData; + qe->type = _RQEntry::RQENTRY_TYPE_REQUEST; _queue.post(qe); } @@ -1051,33 +1039,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } else if (path[0] == "ping") { - const uint64_t now = OSUtils::now(); - bool first = true; - std::string pong("{\"memberStatus\":{"); - { - Mutex::Lock _l(_memberStatus_m); - pong.reserve(64 * _memberStatus.size()); - _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { - char tmp[64]; - uint64_t lrt = 0ULL; - auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); - if (ms != _memberStatus.end()) - lrt = ms->second.lastRequestTime; - Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", - (first) ? "" : ",", - (unsigned long long)networkId, - (unsigned long long)nodeId, - (unsigned long long)lrt); - pong.append(tmp); - first = false; - }); - } - char tmp2[256]; - Utils::snprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); - pong.append(tmp2); - _db.writeRaw("pong",pong); + _startThreads(); + _RQEntry *qe = new _RQEntry; + qe->type = _RQEntry::RQENTRY_TYPE_PING; + _queue.post(qe); - responseBody = "{}"; + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"{\"clock\":%llu}",(unsigned long long)now); + responseBody = tmp; responseContentType = "application/json"; return 200; @@ -1150,7 +1119,35 @@ void EmbeddedNetworkController::threadMain() _RQEntry *qe = (_RQEntry *)0; while ((_running)&&((qe = _queue.get()))) { try { - _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); + if (qe->type == _RQEntry::RQENTRY_TYPE_REQUEST) { + _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); + } else if (qe->type == _RQEntry::RQENTRY_TYPE_PING) { + const uint64_t now = OSUtils::now(); + bool first = true; + std::string pong("{\"memberStatus\":{"); + { + Mutex::Lock _l(_memberStatus_m); + pong.reserve(64 * _memberStatus.size()); + _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { + char tmp[64]; + uint64_t lrt = 0ULL; + auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); + if (ms != _memberStatus.end()) + lrt = ms->second.lastRequestTime; + Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", + (first) ? "" : ",", + (unsigned long long)networkId, + (unsigned long long)nodeId, + (unsigned long long)lrt); + pong.append(tmp); + first = false; + }); + } + char tmp2[256]; + Utils::snprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + pong.append(tmp2); + _db.writeRaw("pong",pong); + } } catch ( ... ) {} delete qe; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 0a6b8176..ade7eb20 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include "../node/Constants.hpp" @@ -103,11 +104,29 @@ private: InetAddress fromAddr; Identity identity; Dictionary metaData; + enum { + RQENTRY_TYPE_REQUEST = 0, + RQENTRY_TYPE_PING = 1 + } type; }; static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); + inline void _startThreads() + { + Mutex::Lock _l(_threads_m); + if (_threads.size() == 0) { + long hwc = (long)std::thread::hardware_concurrency(); + if (hwc < 1) + hwc = 1; + else if (hwc > 16) + hwc = 16; + for(long i=0;i Date: Tue, 2 May 2017 17:28:18 -0700 Subject: Echo back ping payload. --- controller/EmbeddedNetworkController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 91b8671e..16839c18 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1045,7 +1045,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _queue.post(qe); char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"{\"clock\":%llu}",(unsigned long long)now); + Utils::snprintf(tmp,sizeof(tmp),"{\"clock\":%llu,\"ping\":%s}",(unsigned long long)now,OSUtils::jsonDump(b).c_str()); responseBody = tmp; responseContentType = "application/json"; -- cgit v1.2.3 From 39db45e144275092716ad612d58c7b9ba1bc8eed Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 3 May 2017 09:48:08 -0700 Subject: Fix crash on exit (sometimes) in controller. --- controller/EmbeddedNetworkController.cpp | 9 +++------ osdep/BlockingQueue.hpp | 27 +++++++++++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 16839c18..8ff8eb83 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -436,19 +436,16 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa EmbeddedNetworkController::~EmbeddedNetworkController() { - _running = false; std::vector t; { Mutex::Lock _l(_threads_m); + _running = false; t = _threads; } if (t.size() > 0) { - for(unsigned long i=0,j=(unsigned long)(t.size() * 4);i::iterator i(t.begin());i!=t.end();++i) Thread::join(*i); - */ } } @@ -1117,7 +1114,7 @@ void EmbeddedNetworkController::threadMain() { uint64_t lastCircuitTestCheck = 0; _RQEntry *qe = (_RQEntry *)0; - while ((_running)&&((qe = _queue.get()))) { + while ((_running)&&(_queue.get(qe))) { try { if (qe->type == _RQEntry::RQENTRY_TYPE_REQUEST) { _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); diff --git a/osdep/BlockingQueue.hpp b/osdep/BlockingQueue.hpp index 34abcb67..43ae7435 100644 --- a/osdep/BlockingQueue.hpp +++ b/osdep/BlockingQueue.hpp @@ -42,7 +42,7 @@ template class BlockingQueue { public: - BlockingQueue(void) {} + BlockingQueue(void) : r(true) {} inline void post(T t) { @@ -51,19 +51,34 @@ public: c.notify_one(); } - inline T get(void) + inline void stop(void) + { + std::lock_guard lock(m); + r = false; + c.notify_all(); + } + + /** + * @param value Value to set to next queue item if return value is true + * @return False if stop() has been called, true otherwise + */ + inline bool get(T &value) { std::unique_lock lock(m); - while(q.empty()) + if (!r) return false; + while (q.empty()) { c.wait(lock); - T val = q.front(); + if (!r) return false; + } + value = q.front(); q.pop(); - return val; + return true; } private: + volatile bool r; std::queue q; - mutable std::mutex m; + std::mutex m; std::condition_variable c; }; -- cgit v1.2.3 From 909a14fb484178ba29bcc0bc4a3cb22baff1d040 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 May 2017 17:22:24 -0700 Subject: MTU support in controller. --- controller/EmbeddedNetworkController.cpp | 2 ++ controller/EmbeddedNetworkController.hpp | 1 + include/ZeroTierOne.h | 5 +++++ 3 files changed, 8 insertions(+) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8ff8eb83..e8f54fdb 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -809,6 +809,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("enableBroadcast")) network["enableBroadcast"] = OSUtils::jsonBool(b["enableBroadcast"],false); if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = OSUtils::jsonBool(b["allowPassiveBridging"],false); if (b.count("multicastLimit")) network["multicastLimit"] = OSUtils::jsonInt(b["multicastLimit"],32ULL); + if (b.count("mtu")) network["mtu"] = std::max(std::min((unsigned int)OSUtils::jsonInt(b["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU); if (b.count("v4AssignMode")) { json nv4m; @@ -1424,6 +1425,7 @@ void EmbeddedNetworkController::_request( if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; if (OSUtils::jsonBool(network["allowPassiveBridging"],false)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; Utils::scopy(nc->name,sizeof(nc->name),OSUtils::jsonString(network["name"],"").c_str()); + nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU); nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index ade7eb20..4f4660f8 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -162,6 +162,7 @@ private: if (!network.count("tags")) network["tags"] = nlohmann::json::array(); if (!network.count("routes")) network["routes"] = nlohmann::json::array(); if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); + if (!network.count("mtu")) network["mtu"] = ZT_DEFAULT_MTU; if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment network["rules"] = {{ diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 21adbe02..17d6d67e 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -59,6 +59,11 @@ extern "C" { */ #define ZT_DEFAULT_PORT 9993 +/** + * Minimum MTU, which is the minimum allowed by IPv6 and several specs + */ +#define ZT_MIN_MTU 1280 + /** * Maximum MTU for ZeroTier virtual networks */ -- cgit v1.2.3 From d90560735397036d216c63fe1d89a10f54154460 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 May 2017 08:35:55 -0700 Subject: Small optimizations. --- controller/EmbeddedNetworkController.cpp | 27 ++++++++++++++++----------- controller/JSONDB.hpp | 9 +++++++++ 2 files changed, 25 insertions(+), 11 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index e8f54fdb..3e9a28b8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -487,9 +487,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( if ((path.size() >= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - json network; if (!_db.getNetwork(nwid,network)) return 404; @@ -499,6 +496,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( if (path[2] == "member") { if (path.size() >= 4) { + // Get member + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); json member; if (!_db.getNetworkMember(nwid,address,member)) @@ -506,24 +505,29 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( _addMemberNonPersistedFields(nwid,address,member,OSUtils::now()); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; + } else { + // List members and their revisions + responseBody = "{"; + responseBody.reserve((_db.memberCount(nwid) + 1) * 32); _db.eachMember(nwid,[&responseBody](uint64_t networkId,uint64_t nodeId,const json &member) { if ((member.is_object())&&(member.size() > 0)) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(OSUtils::jsonString(member["id"],"0")); - responseBody.append("\":"); - responseBody.append(OSUtils::jsonString(member["revision"],"0")); + char tmp[128]; + Utils::snprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); + responseBody.append(tmp); } }); responseBody.push_back('}'); responseContentType = "application/json"; + } return 200; } // else 404 } else { + // Get network const uint64_t now = OSUtils::now(); JSONDB::NetworkSummaryInfo ns; @@ -535,12 +539,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } } else if (path.size() == 1) { + // List networks std::vector networkIds(_db.networkIds()); - std::sort(networkIds.begin(),networkIds.end()); - char tmp[64]; - responseBody.push_back('['); + responseBody = "["; + responseBody.reserve((networkIds.size() + 1) * 24); for(std::vector::const_iterator i(networkIds.begin());i!=networkIds.end();++i) { if (responseBody.length() > 1) responseBody.push_back(','); @@ -555,6 +559,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } // else 404 } else { + // Controller status char tmp[4096]; Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); @@ -1125,7 +1130,7 @@ void EmbeddedNetworkController::threadMain() std::string pong("{\"memberStatus\":{"); { Mutex::Lock _l(_memberStatus_m); - pong.reserve(64 * _memberStatus.size()); + pong.reserve(48 * (_memberStatus.size() + 1)); _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { char tmp[64]; uint64_t lrt = 0ULL; diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 530f9632..e164a14e 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -93,6 +93,15 @@ public: return r; } + inline unsigned long memberCount(const uint64_t networkId) + { + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i != _networks.end()) + return (unsigned long)i->second.members.size(); + return 0; + } + template inline void eachMember(const uint64_t networkId,F func) { -- cgit v1.2.3 From 426b7c2c767e797f586d38870f5e737b5cc9ba14 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 11 May 2017 14:13:38 -0700 Subject: Stupid bug is stupid. --- controller/JSONDB.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 31c0cc05..3b98a7ac 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -392,14 +392,14 @@ bool JSONDB::_load(const std::string &p) std::string objtype(OSUtils::jsonString(j["objtype"],"")); if ((id.length() == 16)&&(objtype == "network")) { - const uint64_t nwid = Utils::strToU64(id.c_str()); + const uint64_t nwid = Utils::hexStrToU64(const char *s)(id.c_str()); if (nwid) { Mutex::Lock _l(_networks_m); _networks[nwid].config = nlohmann::json::to_msgpack(j); } } else if ((id.length() == 10)&&(objtype == "member")) { - const uint64_t mid = Utils::strToU64(id.c_str()); - const uint64_t nwid = Utils::strToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); + const uint64_t mid = Utils::hexStrToU64(id.c_str()); + const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); if ((mid)&&(nwid)) { Mutex::Lock _l(_networks_m); _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); -- cgit v1.2.3 From 236a952458a547e71390c2b846ef1ee52ffc52a8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 11 May 2017 14:26:46 -0700 Subject: typo --- controller/JSONDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 3b98a7ac..e0dd1742 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -392,7 +392,7 @@ bool JSONDB::_load(const std::string &p) std::string objtype(OSUtils::jsonString(j["objtype"],"")); if ((id.length() == 16)&&(objtype == "network")) { - const uint64_t nwid = Utils::hexStrToU64(const char *s)(id.c_str()); + const uint64_t nwid = Utils::hexStrToU64(id.c_str()); if (nwid) { Mutex::Lock _l(_networks_m); _networks[nwid].config = nlohmann::json::to_msgpack(j); -- cgit v1.2.3 From 2ec88e800877cfbc7f007d21f10429bc1b493006 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 30 May 2017 10:19:45 -0700 Subject: Remove old circuit test code. Rules engine will let us do this much better and more simply. --- controller/EmbeddedNetworkController.cpp | 125 ----------------- controller/EmbeddedNetworkController.hpp | 7 - include/ZeroTierOne.h | 226 ------------------------------- node/IncomingPacket.cpp | 196 +-------------------------- node/IncomingPacket.hpp | 2 - node/Node.cpp | 88 ------------ node/Node.hpp | 6 - node/Packet.cpp | 2 - node/Packet.hpp | 116 +--------------- 9 files changed, 4 insertions(+), 764 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3e9a28b8..e2eaa788 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -726,59 +726,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( responseContentType = "application/json"; return 200; - } else if ((path.size() == 3)&&(path[2] == "test")) { - - Mutex::Lock _l(_tests_m); - - _tests.push_back(ZT_CircuitTest()); - ZT_CircuitTest *const test = &(_tests.back()); - memset(test,0,sizeof(ZT_CircuitTest)); - - Utils::getSecureRandom(&(test->testId),sizeof(test->testId)); - test->credentialNetworkId = nwid; - test->ptr = (void *)this; - json hops = b["hops"]; - if (hops.is_array()) { - for(unsigned long i=0;ihops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - } - ++test->hopCount; - } else if (hops2.is_string()) { - std::string s = hops2; - test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - ++test->hopCount; - } - } - } - test->reportAtEveryHop = (OSUtils::jsonBool(b["reportAtEveryHop"],true) ? 1 : 0); - - if (!test->hopCount) { - _tests.pop_back(); - responseBody = "{ \"message\": \"a test must contain at least one hop\" }"; - responseContentType = "application/json"; - return 400; - } - - test->timestamp = OSUtils::now(); - - if (_node) { - _node->circuitTestBegin((void *)0,test,&(EmbeddedNetworkController::_circuitTestCallback)); - } else { - _tests.pop_back(); - return 500; - } - - char json[512]; - Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\",\"timestamp\":%llu}",test->testId,test->timestamp); - responseBody = json; - responseContentType = "application/json"; - - return 200; - } // else 404 } else { @@ -1118,7 +1065,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::threadMain() throw() { - uint64_t lastCircuitTestCheck = 0; _RQEntry *qe = (_RQEntry *)0; while ((_running)&&(_queue.get(qe))) { try { @@ -1153,80 +1099,9 @@ void EmbeddedNetworkController::threadMain() } } catch ( ... ) {} delete qe; - - if (_running) { - uint64_t now = OSUtils::now(); - if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - lastCircuitTestCheck = now; - Mutex::Lock _l(_tests_m); - for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { - if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - _node->circuitTestEnd(&(*i)); - _tests.erase(i++); - } else ++i; - } - } - } } } -void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) -{ - char tmp[2048],id[128]; - EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); - - if ((!test)||(!report)||(!test->credentialNetworkId)) return; // sanity check - - const uint64_t now = OSUtils::now(); - Utils::snprintf(id,sizeof(id),"network/%.16llx/test/%.16llx-%.16llx-%.10llx-%.10llx",test->credentialNetworkId,test->testId,now,report->upstream,report->current); - Utils::snprintf(tmp,sizeof(tmp), - "{\"id\": \"%s\"," - "\"objtype\": \"circuit_test\"," - "\"timestamp\": %llu," - "\"networkId\": \"%.16llx\"," - "\"testId\": \"%.16llx\"," - "\"upstream\": \"%.10llx\"," - "\"current\": \"%.10llx\"," - "\"receivedTimestamp\": %llu," - "\"sourcePacketId\": \"%.16llx\"," - "\"flags\": %llu," - "\"sourcePacketHopCount\": %u," - "\"errorCode\": %u," - "\"vendor\": %d," - "\"protocolVersion\": %u," - "\"majorVersion\": %u," - "\"minorVersion\": %u," - "\"revision\": %u," - "\"platform\": %d," - "\"architecture\": %d," - "\"receivedOnLocalAddress\": \"%s\"," - "\"receivedFromRemoteAddress\": \"%s\"," - "\"receivedFromLinkQuality\": %f}", - id + 30, // last bit only, not leading path - (unsigned long long)test->timestamp, - (unsigned long long)test->credentialNetworkId, - (unsigned long long)test->testId, - (unsigned long long)report->upstream, - (unsigned long long)report->current, - (unsigned long long)now, - (unsigned long long)report->sourcePacketId, - (unsigned long long)report->flags, - report->sourcePacketHopCount, - report->errorCode, - (int)report->vendor, - report->protocolVersion, - report->majorVersion, - report->minorVersion, - report->revision, - (int)report->platform, - (int)report->architecture, - reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), - reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str(), - ((double)report->receivedFromLinkQuality / (double)ZT_PATH_LINK_QUALITY_MAX)); - - self->_db.writeRaw(id,std::string(tmp)); -} - void EmbeddedNetworkController::_request( uint64_t nwid, const InetAddress &fromAddr, diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 4f4660f8..1589ea71 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -45,9 +45,6 @@ #include "JSONDB.hpp" -// TTL for circuit tests -#define ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION 120000 - namespace ZeroTier { class Node; @@ -110,7 +107,6 @@ private: } type; }; - static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); inline void _startThreads() @@ -219,9 +215,6 @@ private: NetworkController::Sender *_sender; Identity _signingId; - std::list< ZT_CircuitTest > _tests; - Mutex _tests_m; - struct _MemberStatusKey { _MemberStatusKey() : networkId(0),nodeId(0) {} diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 17d6d67e..5126c5a2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -760,7 +760,6 @@ typedef struct */ uint64_t expiration; - struct { uint64_t from; uint64_t to; @@ -1105,197 +1104,6 @@ typedef struct unsigned long peerCount; } ZT_PeerList; -/** - * ZeroTier circuit test configuration and path - */ -typedef struct { - /** - * Test ID -- an arbitrary 64-bit identifier - */ - uint64_t testId; - - /** - * Timestamp -- sent with test and echoed back by each reporter - */ - uint64_t timestamp; - - /** - * Originator credential: network ID - * - * If this is nonzero, a network ID will be set for this test and - * the originator must be its primary network controller. This is - * currently the only authorization method available, so it must - * be set to run a test. - */ - uint64_t credentialNetworkId; - - /** - * Hops in circuit test (a.k.a. FIFO for graph traversal) - */ - struct { - /** - * Hop flags (currently unused, must be zero) - */ - unsigned int flags; - - /** - * Number of addresses in this hop (max: ZT_CIRCUIT_TEST_MAX_HOP_BREADTH) - */ - unsigned int breadth; - - /** - * 40-bit ZeroTier addresses (most significant 24 bits ignored) - */ - uint64_t addresses[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH]; - } hops[ZT_CIRCUIT_TEST_MAX_HOPS]; - - /** - * Number of hops (max: ZT_CIRCUIT_TEST_MAX_HOPS) - */ - unsigned int hopCount; - - /** - * If non-zero, circuit test will report back at every hop - */ - int reportAtEveryHop; - - /** - * An arbitrary user-settable pointer - */ - void *ptr; - - /** - * Pointer for internal use -- initialize to zero and do not modify - */ - void *_internalPtr; -} ZT_CircuitTest; - -/** - * Circuit test result report - */ -typedef struct { - /** - * Sender of report (current hop) - */ - uint64_t current; - - /** - * Previous hop - */ - uint64_t upstream; - - /** - * 64-bit test ID - */ - uint64_t testId; - - /** - * Timestamp from original test (echoed back at each hop) - */ - uint64_t timestamp; - - /** - * 64-bit packet ID of packet received by the reporting device - */ - uint64_t sourcePacketId; - - /** - * Flags - */ - uint64_t flags; - - /** - * ZeroTier protocol-level hop count of packet received by reporting device (>0 indicates relayed) - */ - unsigned int sourcePacketHopCount; - - /** - * Error code (currently unused, will be zero) - */ - unsigned int errorCode; - - /** - * Remote device vendor ID - */ - enum ZT_Vendor vendor; - - /** - * Remote device protocol compliance version - */ - unsigned int protocolVersion; - - /** - * Software major version - */ - unsigned int majorVersion; - - /** - * Software minor version - */ - unsigned int minorVersion; - - /** - * Software revision - */ - unsigned int revision; - - /** - * Platform / OS - */ - enum ZT_Platform platform; - - /** - * System architecture - */ - enum ZT_Architecture architecture; - - /** - * Local device address on which packet was received by reporting device - * - * This may have ss_family equal to zero (null address) if unspecified. - */ - struct sockaddr_storage receivedOnLocalAddress; - - /** - * Remote address from which reporter received the test packet - * - * This may have ss_family set to zero (null address) if unspecified. - */ - struct sockaddr_storage receivedFromRemoteAddress; - - /** - * Path link quality of physical path over which test was received - */ - int receivedFromLinkQuality; - - /** - * Next hops to which packets are being or will be sent by the reporter - * - * In addition to reporting back, the reporter may send the test on if - * there are more recipients in the FIFO. If it does this, it can report - * back the address(es) that make up the next hop and the physical address - * for each if it has one. The physical address being null/unspecified - * typically indicates that no direct path exists and the next packet - * will be relayed. - */ - struct { - /** - * 40-bit ZeroTier address - */ - uint64_t address; - - /** - * Physical address or null address (ss_family == 0) if unspecified or unknown - */ - struct sockaddr_storage physicalAddress; - } nextHops[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH]; - - /** - * Number of next hops reported in nextHops[] - */ - unsigned int nextHopCount; -} ZT_CircuitTestReport; - /** * A cluster member's status */ @@ -1957,40 +1765,6 @@ int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t type */ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance); -/** - * Initiate a VL1 circuit test - * - * This sends an initial VERB_CIRCUIT_TEST and reports results back to the - * supplied callback until circuitTestEnd() is called. The supplied - * ZT_CircuitTest structure should be initially zeroed and then filled - * in with settings and hops. - * - * It is the caller's responsibility to call circuitTestEnd() and then - * to dispose of the test structure. Otherwise this node will listen - * for results forever. - * - * @param node Node instance - * @param tptr Thread pointer to pass to functions/callbacks resulting from this call - * @param test Test configuration - * @param reportCallback Function to call each time a report is received - * @return OK or error if, for example, test is too big for a packet or support isn't compiled in - */ -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *)); - -/** - * Stop listening for results to a given circuit test - * - * This does not free the 'test' structure. The caller may do that - * after calling this method to unregister it. - * - * Any reports that are received for a given test ID after it is - * terminated are ignored. - * - * @param node Node instance - * @param test Test configuration to unregister - */ -void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test); - /** * Initialize cluster operation * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 131659f9..9140c502 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -115,8 +115,6 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,tPtr,peer); case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,tPtr,peer); case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,tPtr,peer); - case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,tPtr,peer); - case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,tPtr,peer); case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,tPtr,peer); } } else { @@ -1252,196 +1250,6 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt return true; } -bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) -{ - try { - const Address originatorAddress(field(ZT_PACKET_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - SharedPtr originator(RR->topology->getPeer(tPtr,originatorAddress)); - if (!originator) { - RR->sw->requestWhois(tPtr,originatorAddress); - return false; - } - - const unsigned int flags = at(ZT_PACKET_IDX_PAYLOAD + 5); - const uint64_t timestamp = at(ZT_PACKET_IDX_PAYLOAD + 7); - const uint64_t testId = at(ZT_PACKET_IDX_PAYLOAD + 15); - - // Tracks total length of variable length fields, initialized to originator credential length below - unsigned int vlf; - - // Originator credentials -- right now only a network ID for which the originator is controller or is authorized by controller is allowed - const unsigned int originatorCredentialLength = vlf = at(ZT_PACKET_IDX_PAYLOAD + 23); - uint64_t originatorCredentialNetworkId = 0; - if (originatorCredentialLength >= 1) { - switch((*this)[ZT_PACKET_IDX_PAYLOAD + 25]) { - case 0x01: { // 64-bit network ID, originator must be controller - if (originatorCredentialLength >= 9) - originatorCredentialNetworkId = at(ZT_PACKET_IDX_PAYLOAD + 26); - } break; - default: break; - } - } - - // Add length of "additional fields," which are currently unused - vlf += at(ZT_PACKET_IDX_PAYLOAD + 25 + vlf); - - // Verify signature -- only tests signed by their originators are allowed - const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); - if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { - TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - return true; - } - vlf += signatureLength; - - // Save this length so we can copy the immutable parts of this test - // into the one we send along to next hops. - const unsigned int lengthOfSignedPortionAndSignature = 29 + vlf; - - // Add length of second "additional fields" section. - vlf += at(ZT_PACKET_IDX_PAYLOAD + 29 + vlf); - - uint64_t reportFlags = 0; - - // Check credentials (signature already verified) - if (originatorCredentialNetworkId) { - SharedPtr network(RR->node->network(originatorCredentialNetworkId)); - if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - return true; - } - if (network->gate(tPtr,peer)) - reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH; - } else { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - return true; - } - - const uint64_t now = RR->node->now(); - - unsigned int breadth = 0; - Address nextHop[256]; // breadth is a uin8_t, so this is the max - InetAddress nextHopBestPathAddress[256]; - unsigned int remainingHopsPtr = ZT_PACKET_IDX_PAYLOAD + 33 + vlf; - if ((ZT_PACKET_IDX_PAYLOAD + 31 + vlf) < size()) { - // unsigned int nextHopFlags = (*this)[ZT_PACKET_IDX_PAYLOAD + 31 + vlf] - breadth = (*this)[ZT_PACKET_IDX_PAYLOAD + 32 + vlf]; - for(unsigned int h=0;h nhp(RR->topology->getPeer(tPtr,nextHop[h])); - if (nhp) { - SharedPtr nhbp(nhp->getBestPath(now,false)); - if ((nhbp)&&(nhbp->alive(now))) - nextHopBestPathAddress[h] = nhbp->address(); - } - } - } - - // Report back to originator, depending on flags and whether we are last hop - if ( ((flags & 0x01) != 0) || ((breadth == 0)&&((flags & 0x02) != 0)) ) { - Packet outp(originatorAddress,RR->identity.address(),Packet::VERB_CIRCUIT_TEST_REPORT); - outp.append((uint64_t)timestamp); - outp.append((uint64_t)testId); - outp.append((uint64_t)0); // field reserved for future use - outp.append((uint8_t)ZT_VENDOR_ZEROTIER); - outp.append((uint8_t)ZT_PROTO_VERSION); - outp.append((uint8_t)ZEROTIER_ONE_VERSION_MAJOR); - outp.append((uint8_t)ZEROTIER_ONE_VERSION_MINOR); - outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); - outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED); - outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED); - outp.append((uint16_t)0); // error code, currently unused - outp.append((uint64_t)reportFlags); - outp.append((uint64_t)packetId()); - peer->address().appendTo(outp); - outp.append((uint8_t)hops()); - _path->localAddress().serialize(outp); - _path->address().serialize(outp); - outp.append((uint16_t)_path->linkQuality()); - outp.append((uint8_t)breadth); - for(unsigned int h=0;hsw->send(tPtr,outp,true); - } - - // If there are next hops, forward the test along through the graph - if (breadth > 0) { - Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST); - outp.append(field(ZT_PACKET_IDX_PAYLOAD,lengthOfSignedPortionAndSignature),lengthOfSignedPortionAndSignature); - outp.append((uint16_t)0); // no additional fields - if (remainingHopsPtr < size()) - outp.append(field(remainingHopsPtr,size() - remainingHopsPtr),size() - remainingHopsPtr); - - for(unsigned int h=0;hidentity.address() != nextHop[h]) { // next hops that loop back to the current hop are not valid - outp.newInitializationVector(); - outp.setDestination(nextHop[h]); - RR->sw->send(tPtr,outp,true); - } - } - } - - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); - } - return true; -} - -bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) -{ - try { - ZT_CircuitTestReport report; - memset(&report,0,sizeof(report)); - - report.current = peer->address().toInt(); - report.upstream = Address(field(ZT_PACKET_IDX_PAYLOAD + 52,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); - report.testId = at(ZT_PACKET_IDX_PAYLOAD + 8); - report.timestamp = at(ZT_PACKET_IDX_PAYLOAD); - report.sourcePacketId = at(ZT_PACKET_IDX_PAYLOAD + 44); - report.flags = at(ZT_PACKET_IDX_PAYLOAD + 36); - report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 57]; // end of fixed length headers: 58 - report.errorCode = at(ZT_PACKET_IDX_PAYLOAD + 34); - report.vendor = (enum ZT_Vendor)((*this)[ZT_PACKET_IDX_PAYLOAD + 24]); - report.protocolVersion = (*this)[ZT_PACKET_IDX_PAYLOAD + 25]; - report.majorVersion = (*this)[ZT_PACKET_IDX_PAYLOAD + 26]; - report.minorVersion = (*this)[ZT_PACKET_IDX_PAYLOAD + 27]; - report.revision = at(ZT_PACKET_IDX_PAYLOAD + 28); - report.platform = (enum ZT_Platform)at(ZT_PACKET_IDX_PAYLOAD + 30); - report.architecture = (enum ZT_Architecture)at(ZT_PACKET_IDX_PAYLOAD + 32); - - const unsigned int receivedOnLocalAddressLen = reinterpret_cast(&(report.receivedOnLocalAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58); - const unsigned int receivedFromRemoteAddressLen = reinterpret_cast(&(report.receivedFromRemoteAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen); - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen + receivedFromRemoteAddressLen; - if (report.protocolVersion >= 9) { - report.receivedFromLinkQuality = at(ptr); ptr += 2; - } else { - report.receivedFromLinkQuality = ZT_PATH_LINK_QUALITY_MAX; - ptr += at(ptr) + 2; // this field was once an 'extended field length' reserved field, which was always set to 0 - } - - report.nextHopCount = (*this)[ptr++]; - if (report.nextHopCount > ZT_CIRCUIT_TEST_MAX_HOP_BREADTH) // sanity check, shouldn't be possible - report.nextHopCount = ZT_CIRCUIT_TEST_MAX_HOP_BREADTH; - for(unsigned int h=0;h(&(report.nextHops[h].physicalAddress))->deserialize(*this,ptr); - } - - RR->node->postCircuitTestReport(&report); - - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); - } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); - } - return true; -} - bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { @@ -1453,9 +1261,9 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + TRACE("dropped USER_MESSAGE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 43a1ea10..11b60712 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -138,8 +138,6 @@ private: bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); - bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); - bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid); diff --git a/node/Node.cpp b/node/Node.cpp index 6d7eea43..911c9c4b 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -503,64 +503,6 @@ void Node::setNetconfMaster(void *networkControllerInstance) RR->localNetworkController->init(RR->identity,this); } -ZT_ResultCode Node::circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) -{ - if (test->hopCount > 0) { - try { - Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST); - RR->identity.address().appendTo(outp); - outp.append((uint16_t)((test->reportAtEveryHop != 0) ? 0x03 : 0x02)); - outp.append((uint64_t)test->timestamp); - outp.append((uint64_t)test->testId); - outp.append((uint16_t)0); // originator credential length, updated later - if (test->credentialNetworkId) { - outp.append((uint8_t)0x01); - outp.append((uint64_t)test->credentialNetworkId); - outp.setAt(ZT_PACKET_IDX_PAYLOAD + 23,(uint16_t)9); - } - outp.append((uint16_t)0); - C25519::Signature sig(RR->identity.sign(reinterpret_cast(outp.data()) + ZT_PACKET_IDX_PAYLOAD,outp.size() - ZT_PACKET_IDX_PAYLOAD)); - outp.append((uint16_t)sig.size()); - outp.append(sig.data,(unsigned int)sig.size()); - outp.append((uint16_t)0); // originator doesn't need an extra credential, since it's the originator - for(unsigned int h=1;hhopCount;++h) { - outp.append((uint8_t)0); - outp.append((uint8_t)(test->hops[h].breadth & 0xff)); - for(unsigned int a=0;ahops[h].breadth;++a) - Address(test->hops[h].addresses[a]).appendTo(outp); - } - - for(unsigned int a=0;ahops[0].breadth;++a) { - outp.newInitializationVector(); - outp.setDestination(Address(test->hops[0].addresses[a])); - RR->sw->send(tptr,outp,true); - } - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet - } - } - - { - test->_internalPtr = reinterpret_cast(reportCallback); - Mutex::Lock _l(_circuitTests_m); - if (std::find(_circuitTests.begin(),_circuitTests.end(),test) == _circuitTests.end()) - _circuitTests.push_back(test); - } - - return ZT_RESULT_OK; -} - -void Node::circuitTestEnd(ZT_CircuitTest *test) -{ - Mutex::Lock _l(_circuitTests_m); - for(;;) { - std::vector< ZT_CircuitTest * >::iterator ct(std::find(_circuitTests.begin(),_circuitTests.end(),test)); - if (ct == _circuitTests.end()) - break; - else _circuitTests.erase(ct); - } -} - ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -715,20 +657,6 @@ uint64_t Node::prng() return z + y; } -void Node::postCircuitTestReport(const ZT_CircuitTestReport *report) -{ - std::vector< ZT_CircuitTest * > toNotify; - { - Mutex::Lock _l(_circuitTests_m); - for(std::vector< ZT_CircuitTest * >::iterator i(_circuitTests.begin());i!=_circuitTests.end();++i) { - if ((*i)->testId == report->testId) - toNotify.push_back(*i); - } - } - for(std::vector< ZT_CircuitTest * >::iterator i(toNotify.begin());i!=toNotify.end();++i) - (reinterpret_cast((*i)->_internalPtr))(reinterpret_cast(this),*i,report); -} - void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count) { RR->topology->setTrustedPaths(reinterpret_cast(networks),ids,count); @@ -1070,22 +998,6 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) } catch ( ... ) {} } -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) -{ - try { - return reinterpret_cast(node)->circuitTestBegin(tptr,test,reportCallback); - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - -void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test) -{ - try { - reinterpret_cast(node)->circuitTestEnd(test); - } catch ( ... ) {} -} - enum ZT_ResultCode ZT_Node_clusterInit( ZT_Node *node, unsigned int myId, diff --git a/node/Node.hpp b/node/Node.hpp index 95587161..57b5489e 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -117,8 +117,6 @@ public: void clearLocalInterfaceAddresses(); int sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); void setNetconfMaster(void *networkControllerInstance); - ZT_ResultCode circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); - void circuitTestEnd(ZT_CircuitTest *test); ZT_ResultCode clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -219,7 +217,6 @@ public: inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } uint64_t prng(); - void postCircuitTestReport(const ZT_CircuitTestReport *report); void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); World planet() const; @@ -309,9 +306,6 @@ private: std::vector< std::pair< uint64_t, SharedPtr > > _networks; Mutex _networks_m; - std::vector< ZT_CircuitTest * > _circuitTests; - Mutex _circuitTests_m; - std::vector _directPaths; Mutex _directPaths_m; diff --git a/node/Packet.cpp b/node/Packet.cpp index e778e3bb..6e1b36ac 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -1082,8 +1082,6 @@ const char *Packet::verbString(Verb v) case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; - case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST"; - case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT"; case VERB_USER_MESSAGE: return "USER_MESSAGE"; } return "(unknown)"; diff --git a/node/Packet.hpp b/node/Packet.hpp index 1de679e7..a76d4180 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -61,7 +61,7 @@ * 4 - 0.6.0 ... 1.0.6 * + BREAKING CHANGE: New identity format based on hashcash design * 5 - 1.1.0 ... 1.1.5 - * + Supports circuit test, proof of work, and echo + * + Supports echo * + Supports in-band world (root server definition) updates * + Clustering! (Though this will work with protocol v4 clients.) * + Otherwise backward compatible with protocol v4 @@ -954,119 +954,7 @@ public: */ VERB_PUSH_DIRECT_PATHS = 0x10, - /** - * Source-routed circuit test message: - * <[5] address of originator of circuit test> - * <[2] 16-bit flags> - * <[8] 64-bit timestamp> - * <[8] 64-bit test ID (arbitrary, set by tester)> - * <[2] 16-bit originator credential length (includes type)> - * [[1] originator credential type (for authorizing test)] - * [[...] originator credential] - * <[2] 16-bit length of additional fields> - * [[...] additional fields] - * [ ... end of signed portion of request ... ] - * <[2] 16-bit length of signature of request> - * <[...] signature of request by originator> - * <[2] 16-bit length of additional fields> - * [[...] additional fields] - * <[...] next hop(s) in path> - * - * Flags: - * 0x01 - Report back to originator at all hops - * 0x02 - Report back to originator at last hop - * - * Originator credential types: - * 0x01 - 64-bit network ID for which originator is controller - * - * Path record format: - * <[1] 8-bit flags (unused, must be zero)> - * <[1] 8-bit breadth (number of next hops)> - * <[...] one or more ZeroTier addresses of next hops> - * - * The circuit test allows a device to send a message that will traverse - * the network along a specified path, with each hop optionally reporting - * back to the tester via VERB_CIRCUIT_TEST_REPORT. - * - * Each circuit test packet includes a digital signature by the originator - * of the request, as well as a credential by which that originator claims - * authorization to perform the test. Currently this signature is ed25519, - * but in the future flags might be used to indicate an alternative - * algorithm. For example, the originator might be a network controller. - * In this case the test might be authorized if the recipient is a member - * of a network controlled by it, and if the previous hop(s) are also - * members. Each hop may include its certificate of network membership. - * - * Circuit test paths consist of a series of records. When a node receives - * an authorized circuit test, it: - * - * (1) Reports back to circuit tester as flags indicate - * (2) Reads and removes the next hop from the packet's path - * (3) Sends the packet along to next hop(s), if any. - * - * It is perfectly legal for a path to contain the same hop more than - * once. In fact, this can be a very useful test to determine if a hop - * can be reached bidirectionally and if so what that connectivity looks - * like. - * - * The breadth field in source-routed path records allows a hop to forward - * to more than one recipient, allowing the tester to specify different - * forms of graph traversal in a test. - * - * There is no hard limit to the number of hops in a test, but it is - * practically limited by the maximum size of a (possibly fragmented) - * ZeroTier packet. - * - * Support for circuit tests is optional. If they are not supported, the - * node should respond with an UNSUPPORTED_OPERATION error. If a circuit - * test request is not authorized, it may be ignored or reported as - * an INVALID_REQUEST. No OK messages are generated, but TEST_REPORT - * messages may be sent (see below). - * - * ERROR packet format: - * <[8] 64-bit timestamp (echoed from original> - * <[8] 64-bit test ID (echoed from original)> - */ - VERB_CIRCUIT_TEST = 0x11, - - /** - * Circuit test hop report: - * <[8] 64-bit timestamp (echoed from original test)> - * <[8] 64-bit test ID (echoed from original test)> - * <[8] 64-bit reserved field (set to 0, currently unused)> - * <[1] 8-bit vendor ID (set to 0, currently unused)> - * <[1] 8-bit reporter protocol version> - * <[1] 8-bit reporter software major version> - * <[1] 8-bit reporter software minor version> - * <[2] 16-bit reporter software revision> - * <[2] 16-bit reporter OS/platform or 0 if not specified> - * <[2] 16-bit reporter architecture or 0 if not specified> - * <[2] 16-bit error code (set to 0, currently unused)> - * <[8] 64-bit report flags> - * <[8] 64-bit packet ID of received CIRCUIT_TEST packet> - * <[5] upstream ZeroTier address from which CIRCUIT_TEST was received> - * <[1] 8-bit packet hop count of received CIRCUIT_TEST> - * <[...] local wire address on which packet was received> - * <[...] remote wire address from which packet was received> - * <[2] 16-bit path link quality of path over which packet was received> - * <[1] 8-bit number of next hops (breadth)> - * <[...] next hop information> - * - * Next hop information record format: - * <[5] ZeroTier address of next hop> - * <[...] current best direct path address, if any, 0 if none> - * - * Report flags: - * 0x1 - Upstream peer in circuit test path allowed in path (e.g. network COM valid) - * - * Circuit test reports can be sent by hops in a circuit test to report - * back results. They should include information about the sender as well - * as about the paths to which next hops are being sent. - * - * If a test report is received and no circuit test was sent, it should be - * ignored. This message generates no OK or ERROR response. - */ - VERB_CIRCUIT_TEST_REPORT = 0x12, + // 0x11, 0x12 -- deprecated /** * A message with arbitrary user-definable content: -- cgit v1.2.3 From 9b2e08dd09b88e94b459a84764d47590531a5ef9 Mon Sep 17 00:00:00 2001 From: Travis LaDuke Date: Mon, 26 Jun 2017 11:27:07 -0700 Subject: Update README.md We've had multiple questions in the community chat regarding posting new network and getting the curl syntax correct. --- controller/README.md | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'controller') diff --git a/controller/README.md b/controller/README.md index db8d0153..b21b3058 100644 --- a/controller/README.md +++ b/controller/README.md @@ -69,6 +69,12 @@ By making queries to this path you can create, configure, and delete networks. D When POSTing new networks take care that their IDs are not in use, otherwise you may overwrite an existing one. To create a new network with a random unused ID, POST to `/controller/network/##########______`. The #'s are the controller's 10-digit ZeroTier address and they're followed by six underscores. Check the `nwid` field of the returned JSON object for your network's newly allocated ID. Subsequent POSTs to this network must refer to its actual path. +Example: + +`curl -X POST --header "X-ZT1-Auth: secret" -d '{"name":"my network"}' http://localhost:9993/controller/network/305f406058______` + +**Network object format:** + | Field | Type | Description | Writable | | --------------------- | ------------- | ------------------------------------------------- | -------- | | id | string | 16-digit network ID | no | -- cgit v1.2.3 From 355cce3938a815feba1085569263ae0225cebfa6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Jun 2017 11:31:29 -0700 Subject: Rename Utils::snprintf due to it being a #define on one platform. --- attic/DBM.cpp | 243 ------------------------------- attic/DBM.hpp | 168 --------------------- controller/EmbeddedNetworkController.cpp | 26 ++-- controller/JSONDB.cpp | 10 +- node/Address.hpp | 4 +- node/Dictionary.hpp | 4 +- node/InetAddress.cpp | 8 +- node/MAC.hpp | 2 +- node/MulticastGroup.hpp | 2 +- node/Network.cpp | 4 +- node/NetworkConfig.cpp | 2 +- node/Node.cpp | 2 +- node/Utils.cpp | 5 +- node/Utils.hpp | 3 +- one.cpp | 20 +-- osdep/BSDEthernetTap.cpp | 16 +- osdep/Http.cpp | 4 +- osdep/LinuxEthernetTap.cpp | 10 +- osdep/OSUtils.cpp | 6 +- osdep/OSXEthernetTap.cpp | 14 +- osdep/PortMapper.cpp | 4 +- osdep/WindowsEthernetTap.cpp | 10 +- selftest.cpp | 2 +- service/ClusterDefinition.hpp | 2 +- service/OneService.cpp | 62 ++++---- service/SoftwareUpdater.cpp | 2 +- 26 files changed, 111 insertions(+), 524 deletions(-) delete mode 100644 attic/DBM.cpp delete mode 100644 attic/DBM.hpp (limited to 'controller') diff --git a/attic/DBM.cpp b/attic/DBM.cpp deleted file mode 100644 index 54f017e0..00000000 --- a/attic/DBM.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#include "DBM.hpp" - -#include "../version.h" - -#include "../node/Salsa20.hpp" -#include "../node/Poly1305.hpp" -#include "../node/SHA512.hpp" - -#include "../osdep/OSUtils.hpp" - -#define ZT_STORED_OBJECT_TYPE__CLUSTER_NODE_STATUS (ZT_STORED_OBJECT__MAX_TYPE_ID + 1) -#define ZT_STORED_OBJECT_TYPE__CLUSTER_DEFINITION (ZT_STORED_OBJECT__MAX_TYPE_ID + 2) - -namespace ZeroTier { - -// We generate the cluster ID from our address and version info since this is -// not at all designed to allow interoperation between versions (or endians) -// in the same cluster. -static inline uint64_t _mkClusterId(const Address &myAddress) -{ - uint64_t x = ZEROTIER_ONE_VERSION_MAJOR; - x <<= 8; - x += ZEROTIER_ONE_VERSION_MINOR; - x <<= 8; - x += ZEROTIER_ONE_VERSION_REVISION; - x <<= 40; - x ^= myAddress.toInt(); -#if __BYTE_ORDER == __BIG_ENDIAN - ++x; -#endif; - return x; -} - -void DBM::onUpdate(uint64_t from,const _MapKey &k,const _MapValue &v,uint64_t rev) -{ - char p[4096]; - char tmp[ZT_DBM_MAX_VALUE_SIZE]; - if (_persistentPath((ZT_StoredObjectType)k.type,k.key,p,sizeof(p))) { - // Reduce unnecessary disk writes - FILE *f = fopen(p,"r"); - if (f) { - long n = (long)fread(tmp,1,sizeof(tmp),f); - fclose(f); - if ((n == (long)v.len)&&(!memcmp(v.data,tmp,n))) - return; - } - - // Write to disk if file has changed or was not already present - f = fopen(p,"w"); - if (f) { - if (fwrite(data,len,1,f) != 1) - fprintf(stderr,"WARNING: error writing to %s (I/O error)" ZT_EOL_S,p); - fclose(f); - if (type == ZT_STORED_OBJECT_IDENTITY_SECRET) - OSUtils::lockDownFile(p,false); - } else { - fprintf(stderr,"WARNING: error writing to %s (cannot open)" ZT_EOL_S,p); - } - } -} - -void DBM::onDelete(uint64_t from,const _MapKey &k) -{ - char p[4096]; - if (_persistentPath((ZT_StoredObjectType)k.type,k.key,p,sizeof(p))) - OSUtils::rm(p); -} - -DBM::_vsdm_cryptor::_vsdm_cryptor(const Identity &secretIdentity) -{ - uint8_t s512[64]; - SHA512::hash(h512,secretIdentity.privateKeyPair().priv.data,ZT_C25519_PRIVATE_KEY_LEN); - memcpy(_key,s512,sizeof(_key)); -} - -void DBM::_vsdm_cryptor::encrypt(void *d,unsigned long l) -{ - if (l >= 24) { // sanity check - uint8_t key[32]; - uint8_t authKey[32]; - uint8_t auth[16]; - - uint8_t *const iv = reinterpret_cast(d) + (l - 16); - Utils::getSecureRandom(iv,16); - memcpy(key,_key,32); - for(unsigned long i=0;i<8;++i) - _key[i] ^= iv[i]; - - Salsa20 s20(key,iv + 8); - memset(authKey,0,32); - s20.crypt12(authKey,authKey,32); - s20.crypt12(d,d,l - 24); - - Poly1305::compute(auth,d,l - 24,authKey); - memcpy(reinterpret_cast(d) + (l - 24),auth,8); - } -} - -bool DBM::_vsdm_cryptor::decrypt(void *d,unsigned long l) -{ - if (l >= 24) { // sanity check - uint8_t key[32]; - uint8_t authKey[32]; - uint8_t auth[16]; - - uint8_t *const iv = reinterpret_cast(d) + (l - 16); - memcpy(key,_key,32); - for(unsigned long i=0;i<8;++i) - _key[i] ^= iv[i]; - - Salsa20 s20(key,iv + 8); - memset(authKey,0,32); - s20.crypt12(authKey,authKey,32); - - Poly1305::compute(auth,d,l - 24,authKey); - if (!Utils::secureEq(reinterpret_cast(d) + (l - 24),auth,8)) - return false; - - s20.crypt12(d,d,l - 24); - - return true; - } - return false; -} - -DBM::DBM(const Identity &secretIdentity,uint64_t clusterMemberId,const std::string &basePath,Node *node) : - _basePath(basePath), - _node(node), - _startTime(OSUtils::now()), - _m(_mkClusterId(secretIdentity.address()),clusterMemberId,false,_vsdm_cryptor(secretIdentity),_vsdm_watcher(this)) -{ -} - -DBM::~DBM() -{ -} - -void DBM::put(const ZT_StoredObjectType type,const uint64_t key,const void *data,unsigned int len) -{ - char p[4096]; - if (_m.put(_MapKey(key,(uint16_t)type),Value(OSUtils::now(),(uint16_t)len,data))) { - if (_persistentPath(type,key,p,sizeof(p))) { - FILE *f = fopen(p,"w"); - if (f) { - if (fwrite(data,len,1,f) != 1) - fprintf(stderr,"WARNING: error writing to %s (I/O error)" ZT_EOL_S,p); - fclose(f); - if (type == ZT_STORED_OBJECT_IDENTITY_SECRET) - OSUtils::lockDownFile(p,false); - } else { - fprintf(stderr,"WARNING: error writing to %s (cannot open)" ZT_EOL_S,p); - } - } - } -} - -bool DBM::get(const ZT_StoredObjectType type,const uint64_t key,Value &value) -{ - char p[4096]; - if (_m.get(_MapKey(key,(uint16_t)type),value)) - return true; - if (_persistentPath(type,key,p,sizeof(p))) { - FILE *f = fopen(p,"r"); - if (f) { - long n = (long)fread(value.data,1,sizeof(value.data),f); - value.len = (n > 0) ? (uint16_t)n : (uint16_t)0; - fclose(f); - value.ts = OSUtils::getLastModified(p); - _m.put(_MapKey(key,(uint16_t)type),value); - return true; - } - } - return false; -} - -void DBM::del(const ZT_StoredObjectType type,const uint64_t key) -{ - char p[4096]; - _m.del(_MapKey(key,(uint16_t)type)); - if (_persistentPath(type,key,p,sizeof(p))) - OSUtils::rm(p); -} - -void DBM::clean() -{ -} - -bool DBM::_persistentPath(const ZT_StoredObjectType type,const uint64_t key,char *p,unsigned int maxlen) -{ - switch(type) { - case ZT_STORED_OBJECT_IDENTITY_PUBLIC: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "identity.public",_basePath.c_str()); - return true; - case ZT_STORED_OBJECT_IDENTITY_SECRET: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "identity.secret",_basePath.c_str()); - return true; - case ZT_STORED_OBJECT_IDENTITY: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "iddb.d" ZT_PATH_SEPARATOR_S "%.10llx",_basePath.c_str(),key); - return true; - case ZT_STORED_OBJECT_NETWORK_CONFIG: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.conf",_basePath.c_str(),key); - return true; - case ZT_STORED_OBJECT_PLANET: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "planet",_basePath.c_str()); - return true; - case ZT_STORED_OBJECT_MOON: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "moons.d" ZT_PATH_SEPARATOR_S "%.16llx.moon",_basePath.c_str(),key); - return true; - case (ZT_StoredObjectType)ZT_STORED_OBJECT_TYPE__CLUSTER_DEFINITION: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "cluster",_basePath.c_str()); - return true; - default: - return false; - } -} - -} // namespace ZeroTier diff --git a/attic/DBM.hpp b/attic/DBM.hpp deleted file mode 100644 index c6d5b8c0..00000000 --- a/attic/DBM.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_DBM_HPP___ -#define ZT_DBM_HPP___ - -#include -#include -#include -#include - -#include - -#include "../node/Constants.hpp" -#include "../node/Mutex.hpp" -#include "../node/Utils.hpp" -#include "../node/Identity.hpp" -#include "../node/Peer.hpp" - -#include "../ext/vsdm/vsdm.hpp" - -// The Peer is the largest structure we persist here -#define ZT_DBM_MAX_VALUE_SIZE sizeof(Peer) - -namespace ZeroTier { - -class Node; -class DBM; - -class DBM -{ -public: - ZT_PACKED_STRUCT(struct Value - { - Value(const uint64_t t,const uint16_t l,const void *d) : - ts(t), - l(l) - { - memcpy(data,d,l); - } - uint64_t ts; - uint16_t len; - uint8_t data[ZT_DBM_MAX_VALUE_SIZE]; - }); - -private: - ZT_PACKED_STRUCT(struct _MapKey - { - _MapKey() : obj(0),type(0) {} - _MapKey(const uint16_t t,const uint64_t o) : obj(o),type(t) {} - uint64_t obj; - uint16_t type; - inline bool operator==(const _MapKey &k) const { return ((obj == k.obj)&&(type == k.type)); } - }); - struct _MapHasher - { - inline std::size_t operator()(const _MapKey &k) const { return (std::size_t)((k.obj ^ (k.obj >> 32)) + (uint64_t)k.type); } - }; - - void onUpdate(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev); - void onDelete(uint64_t from,const _MapKey &k); - - class _vsdm_watcher - { - public: - _vsdm_watcher(DBM *p) : _parent(p) {} - inline void add(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev) { _parent->onUpdate(from,k,v,rev); } - inline void update(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev) { _parent->onUpdate(from,k,v,rev); } - inline void del(uint64_t from,const _MapKey &k) { _parent->onDelete(from,k); } - private: - DBM *_parent; - }; - class _vsdm_serializer - { - public: - static inline unsigned long objectSize(const _MapKey &k) { return 10; } - static inline unsigned long objectSize(const Value &v) { return (10 + v.len); } - static inline const char *objectData(const _MapKey &k) { return reinterpret_cast(&k); } - static inline const char *objectData(const Value &v) { return reinterpret_cast(&v); } - static inline bool objectDeserialize(const char *d,unsigned long l,_MapKey &k) - { - if (l == 10) { - memcpy(&k,d,10); - return true; - } - return false; - } - static inline bool objectDeserialize(const char *d,unsigned long l,Value &v) - { - if ((l >= 10)&&(l <= (10 + ZT_DBM_MAX_VALUE_SIZE))) { - memcpy(&v,d,l); - return true; - } - return false; - } - }; - class _vsdm_cryptor - { - public: - _vsdm_cryptor(const Identity &secretIdentity); - static inline unsigned long overhead() { return 24; } - void encrypt(void *d,unsigned long l); - bool decrypt(void *d,unsigned long l); - uint8_t _key[32]; - }; - - typedef vsdm< _MapKey,Value,16384,_vsdm_watcher,_vsdm_serializer,_vsdm_cryptor,_MapHasher > _Map; - - friend class _Map; - -public: - ZT_PACKED_STRUCT(struct ClusterPeerStatus - { - uint64_t startTime; - uint64_t currentTime; - uint64_t clusterPeersConnected; - uint64_t ztPeersConnected; - uint16_t platform; - uint16_t arch; - }); - - DBM(const Identity &secretIdentity,uint64_t clusterMemberId,const std::string &basePath,Node *node); - - ~DBM(); - - void put(const ZT_StoredObjectType type,const uint64_t key,const void *data,unsigned int len); - - bool get(const ZT_StoredObjectType type,const uint64_t key,Value &value); - - void del(const ZT_StoredObjectType type,const uint64_t key); - - void clean(); - -private: - bool DBM::_persistentPath(const ZT_StoredObjectType type,const uint64_t key,char *p,unsigned int maxlen); - - const std::string _basePath; - Node *const _node; - uint64_t _startTime; - _Map _m; -}; - -} // namespace ZeroTier - -#endif diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index e2eaa788..85c759e7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -122,12 +122,12 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: r["type"] = "MATCH_MAC_DEST"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: @@ -179,7 +179,7 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); r["mask"] = tmp; break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: @@ -514,7 +514,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( _db.eachMember(nwid,[&responseBody](uint64_t networkId,uint64_t nodeId,const json &member) { if ((member.is_object())&&(member.size() > 0)) { char tmp[128]; - Utils::snprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); responseBody.append(tmp); } }); @@ -548,7 +548,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( for(std::vector::const_iterator i(networkIds.begin());i!=networkIds.end();++i) { if (responseBody.length() > 1) responseBody.push_back(','); - Utils::snprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); + Utils::ztsnprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); responseBody.append(tmp); } responseBody.push_back(']'); @@ -562,7 +562,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( // Controller status char tmp[4096]; - Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + Utils::ztsnprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); responseBody = tmp; responseContentType = "application/json"; return 200; @@ -603,14 +603,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if ((path.size() >= 2)&&(path[1].length() == 16)) { uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { uint64_t address = Utils::hexStrToU64(path[3].c_str()); char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); + Utils::ztsnprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); json member; _db.getNetworkMember(nwid,address,member); @@ -748,7 +748,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!nwid) return 503; } - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); json network; _db.getNetwork(nwid,network); @@ -995,7 +995,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _queue.post(qe); char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"{\"clock\":%llu,\"ping\":%s}",(unsigned long long)now,OSUtils::jsonDump(b).c_str()); + Utils::ztsnprintf(tmp,sizeof(tmp),"{\"clock\":%llu,\"ping\":%s}",(unsigned long long)now,OSUtils::jsonDump(b).c_str()); responseBody = tmp; responseContentType = "application/json"; @@ -1083,7 +1083,7 @@ void EmbeddedNetworkController::threadMain() auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); if (ms != _memberStatus.end()) lrt = ms->second.lastRequestTime; - Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", + Utils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, @@ -1093,7 +1093,7 @@ void EmbeddedNetworkController::threadMain() }); } char tmp2[256]; - Utils::snprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + Utils::ztsnprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); pong.append(tmp2); _db.writeRaw("pong",pong); } @@ -1126,7 +1126,7 @@ void EmbeddedNetworkController::_request( ms.lastRequestTime = now; } - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index e0dd1742..acf23700 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -94,7 +94,7 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) std::string body; std::map reqHeaders; char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); reqHeaders["Content-Length"] = tmp; reqHeaders["Content-Type"] = "application/json"; const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); @@ -164,7 +164,7 @@ bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlo void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) { char n[64]; - Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig)); { Mutex::Lock _l(_networks_m); @@ -176,7 +176,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig) { char n[256]; - Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); writeRaw(n,OSUtils::jsonDump(memberConfig)); { Mutex::Lock _l(_networks_m); @@ -202,7 +202,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) } char n[256]; - Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); if (_httpAddr) { // Deletion is currently done by Central in harnessed mode @@ -229,7 +229,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo) { char n[256]; - Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); if (_httpAddr) { // Deletion is currently done by the caller in Central harnessed mode diff --git a/node/Address.hpp b/node/Address.hpp index 9d2d1734..98e32858 100644 --- a/node/Address.hpp +++ b/node/Address.hpp @@ -144,7 +144,7 @@ public: inline std::string toString() const { char buf[16]; - Utils::snprintf(buf,sizeof(buf),"%.10llx",(unsigned long long)_a); + Utils::ztsnprintf(buf,sizeof(buf),"%.10llx",(unsigned long long)_a); return std::string(buf); }; @@ -154,7 +154,7 @@ public: */ inline void toString(char *buf,unsigned int len) const { - Utils::snprintf(buf,len,"%.10llx",(unsigned long long)_a); + Utils::ztsnprintf(buf,len,"%.10llx",(unsigned long long)_a); } /** diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 4413d628..0b000f13 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -391,7 +391,7 @@ public: inline bool add(const char *key,uint64_t value) { char tmp[32]; - Utils::snprintf(tmp,sizeof(tmp),"%llx",(unsigned long long)value); + Utils::ztsnprintf(tmp,sizeof(tmp),"%llx",(unsigned long long)value); return this->add(key,tmp,-1); } @@ -401,7 +401,7 @@ public: inline bool add(const char *key,const Address &a) { char tmp[32]; - Utils::snprintf(tmp,sizeof(tmp),"%.10llx",(unsigned long long)a.toInt()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",(unsigned long long)a.toInt()); return this->add(key,tmp,-1); } diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 0fbb2d68..17d7c72e 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -152,7 +152,7 @@ std::string InetAddress::toString() const char buf[128]; switch(ss_family) { case AF_INET: - Utils::snprintf(buf,sizeof(buf),"%d.%d.%d.%d/%d", + Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d/%d", (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], @@ -161,7 +161,7 @@ std::string InetAddress::toString() const ); return std::string(buf); case AF_INET6: - Utils::snprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x/%d", + Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x/%d", (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), @@ -190,7 +190,7 @@ std::string InetAddress::toIpString() const char buf[128]; switch(ss_family) { case AF_INET: - Utils::snprintf(buf,sizeof(buf),"%d.%d.%d.%d", + Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d", (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], @@ -198,7 +198,7 @@ std::string InetAddress::toIpString() const ); return std::string(buf); case AF_INET6: - Utils::snprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x", + Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x", (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), diff --git a/node/MAC.hpp b/node/MAC.hpp index e7717d99..db50aeb1 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -178,7 +178,7 @@ public: */ inline void toString(char *buf,unsigned int len) const { - Utils::snprintf(buf,len,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)(*this)[0],(int)(*this)[1],(int)(*this)[2],(int)(*this)[3],(int)(*this)[4],(int)(*this)[5]); + Utils::ztsnprintf(buf,len,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)(*this)[0],(int)(*this)[1],(int)(*this)[2],(int)(*this)[3],(int)(*this)[4],(int)(*this)[5]); } /** diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index 4240db67..7cbec2e0 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -100,7 +100,7 @@ public: inline std::string toString() const { char buf[64]; - Utils::snprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%.8lx",(unsigned int)_mac[0],(unsigned int)_mac[1],(unsigned int)_mac[2],(unsigned int)_mac[3],(unsigned int)_mac[4],(unsigned int)_mac[5],(unsigned long)_adi); + Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%.8lx",(unsigned int)_mac[0],(unsigned int)_mac[1],(unsigned int)_mac[2],(unsigned int)_mac[3],(unsigned int)_mac[4],(unsigned int)_mac[5],(unsigned long)_adi); return std::string(buf); } diff --git a/node/Network.cpp b/node/Network.cpp index 12deeeb7..8c6f2ce8 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -51,7 +51,7 @@ namespace ZeroTier { namespace { #ifdef ZT_RULES_ENGINE_DEBUGGING -#define FILTER_TRACE(f,...) { Utils::snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } +#define FILTER_TRACE(f,...) { Utils::ztsnprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } static const char *_rtn(const ZT_VirtualNetworkRuleType rt) { switch(rt) { @@ -1261,7 +1261,7 @@ void Network::requestConfiguration(void *tPtr) nconf->rules[13].t = (uint8_t)ZT_NETWORK_RULE_ACTION_DROP; nconf->type = ZT_NETWORK_TYPE_PUBLIC; - Utils::snprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); + Utils::ztsnprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); this->setConfiguration(tPtr,*nconf,false); delete nconf; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index c39f6cab..65101c3a 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -94,7 +94,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (ets.length() > 0) ets.push_back(','); char tmp2[16]; - Utils::snprintf(tmp2,sizeof(tmp2),"%x",et); + Utils::ztsnprintf(tmp2,sizeof(tmp2),"%x",et); ets.append(tmp2); } et = 0; diff --git a/node/Node.cpp b/node/Node.cpp index 39325b65..ab49e63b 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -742,7 +742,7 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) va_end(ap); tmp2[sizeof(tmp2)-1] = (char)0; - Utils::snprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2); + Utils::ztsnprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2); postEvent((void *)0,ZT_EVENT_TRACE,tmp1); } #endif // ZT_TRACE diff --git a/node/Utils.cpp b/node/Utils.cpp index d69e5335..d2321e16 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -244,8 +244,7 @@ bool Utils::scopy(char *dest,unsigned int len,const char *src) return true; } -unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...) - throw(std::length_error) +unsigned int Utils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...) { va_list ap; @@ -256,7 +255,7 @@ unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...) if ((n >= (int)len)||(n < 0)) { if (len) buf[len - 1] = (char)0; - throw std::length_error("buf[] overflow in Utils::snprintf"); + throw std::length_error("buf[] overflow"); } return (unsigned int)n; diff --git a/node/Utils.hpp b/node/Utils.hpp index 25a90055..212ef247 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -244,8 +244,7 @@ public: * @param ... Format arguments * @throws std::length_error buf[] too short (buf[] will still be left null-terminated) */ - static unsigned int snprintf(char *buf,unsigned int len,const char *fmt,...) - throw(std::length_error); + static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...); /** * Count the number of bits set in an integer diff --git a/one.cpp b/one.cpp index 93504cfb..cbf09121 100644 --- a/one.cpp +++ b/one.cpp @@ -260,9 +260,9 @@ static int cli(int argc,char **argv) if (hd) { char p[4096]; #ifdef __APPLE__ - Utils::snprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); + Utils::ztsnprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); #else - Utils::snprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); + Utils::ztsnprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); #endif OSUtils::readFile(p,authToken); } @@ -278,7 +278,7 @@ static int cli(int argc,char **argv) InetAddress addr; { char addrtmp[256]; - Utils::snprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); + Utils::ztsnprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); addr = InetAddress(addrtmp); } @@ -366,7 +366,7 @@ static int cli(int argc,char **argv) std::string addr = path["address"]; const uint64_t now = OSUtils::now(); const double lq = (path.count("linkQuality")) ? (double)path["linkQuality"] : -1.0; - Utils::snprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); bestPath = tmp; break; } @@ -378,7 +378,7 @@ static int cli(int argc,char **argv) int64_t vmin = p["versionMinor"]; int64_t vrev = p["versionRev"]; if (vmaj >= 0) { - Utils::snprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + Utils::ztsnprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); } else { ver[0] = '-'; ver[1] = (char)0; @@ -527,9 +527,9 @@ static int cli(int argc,char **argv) const uint64_t seed = Utils::hexStrToU64(arg2.c_str()); if ((worldId)&&(seed)) { char jsons[1024]; - Utils::snprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); + Utils::ztsnprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); char cl[128]; - Utils::snprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -579,11 +579,11 @@ static int cli(int argc,char **argv) if (eqidx != std::string::npos) { if ((arg2.substr(0,eqidx) == "allowManaged")||(arg2.substr(0,eqidx) == "allowGlobal")||(arg2.substr(0,eqidx) == "allowDefault")) { char jsons[1024]; - Utils::snprintf(jsons,sizeof(jsons),"{\"%s\":%s}", + Utils::ztsnprintf(jsons,sizeof(jsons),"{\"%s\":%s}", arg2.substr(0,eqidx).c_str(), (((arg2.substr(eqidx,2) == "=t")||(arg2.substr(eqidx,2) == "=1")) ? "true" : "false")); char cl[128]; - Utils::snprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -864,7 +864,7 @@ static int idtool(int argc,char **argv) Buffer wbuf; w.serialize(wbuf); char fn[128]; - Utils::snprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); + Utils::ztsnprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); OSUtils::writeFile(fn,wbuf.data(),wbuf.size()); printf("wrote %s (signed world with timestamp %llu)" ZT_EOL_S,fn,(unsigned long long)now); } diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index 5bb5fbd1..f07f9e5a 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -114,8 +114,8 @@ BSDEthernetTap::BSDEthernetTap( std::vector devFiles(OSUtils::listDirectory("/dev")); for(int i=9993;i<(9993+128);++i) { - Utils::snprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::snprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); if (std::find(devFiles.begin(),devFiles.end(),std::string(tmpdevname)) == devFiles.end()) { long cpid = (long)vfork(); if (cpid == 0) { @@ -152,8 +152,8 @@ BSDEthernetTap::BSDEthernetTap( /* Other BSDs like OpenBSD only have a limited number of tap devices that cannot be renamed */ for(int i=0;i<64;++i) { - Utils::snprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::snprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { _dev = tmpdevname; @@ -171,9 +171,9 @@ BSDEthernetTap::BSDEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::snprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::snprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::snprintf(metstr,sizeof(metstr),"%u",_metric); + Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -385,7 +385,7 @@ void BSDEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%u",mtu); + Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/Http.cpp b/osdep/Http.cpp index f1d3bfe2..3c556f44 100644 --- a/osdep/Http.cpp +++ b/osdep/Http.cpp @@ -244,10 +244,10 @@ unsigned int Http::_do( try { char tmp[1024]; - Utils::snprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); handler.writeBuf.append(tmp); for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) { - Utils::snprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); handler.writeBuf.append(tmp); } handler.writeBuf.append("\r\n"); diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index ccaa92ef..fc5199f1 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -97,7 +97,7 @@ LinuxEthernetTap::LinuxEthernetTap( char procpath[128],nwids[32]; struct stat sbuf; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally @@ -134,7 +134,7 @@ LinuxEthernetTap::LinuxEthernetTap( std::map::const_iterator gdmEntry = globalDeviceMap.find(nwids); if (gdmEntry != globalDeviceMap.end()) { Utils::scopy(ifr.ifr_name,sizeof(ifr.ifr_name),gdmEntry->second.c_str()); - Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); recalledDevice = (stat(procpath,&sbuf) != 0); } @@ -142,8 +142,8 @@ LinuxEthernetTap::LinuxEthernetTap( #ifdef __SYNOLOGY__ int devno = 50; do { - Utils::snprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); - Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + Utils::ztsnprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); + Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); // try zt#++ until we find one that does not exist #else char devno = 0; @@ -158,7 +158,7 @@ LinuxEthernetTap::LinuxEthernetTap( _base32_5_to_8(reinterpret_cast(tmp2) + 5,tmp3 + 10); tmp3[15] = (char)0; memcpy(ifr.ifr_name,tmp3,16); - Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); #endif } diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 53e8bb97..bf0dc04d 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -134,7 +134,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) if (date.QuadPart > 0) { date.QuadPart -= adjust.QuadPart; if ((uint64_t)((date.QuadPart / 10000000) * 1000) < olderThan) { - Utils::snprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); + Utils::ztsnprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); if (DeleteFileA(tmp)) ++cleaned; } @@ -157,7 +157,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) break; if (dptr) { if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_REG)) { - Utils::snprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); if (stat(tmp,&st) == 0) { uint64_t mt = (uint64_t)(st.st_mtime); if ((mt > 0)&&((mt * 1000) < olderThan)) { @@ -446,7 +446,7 @@ std::string OSUtils::jsonString(const nlohmann::json &jv,const char *dfl) return jv; } else if (jv.is_number()) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); + Utils::ztsnprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); return tmp; } else if (jv.is_boolean()) { return ((bool)jv ? std::string("1") : std::string("0")); diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp index f5e1c43f..e082408e 100644 --- a/osdep/OSXEthernetTap.cpp +++ b/osdep/OSXEthernetTap.cpp @@ -336,7 +336,7 @@ OSXEthernetTap::OSXEthernetTap( char devpath[64],ethaddr[64],mtustr[32],metstr[32],nwids[32]; struct stat stattmp; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _gl(globalTapCreateLock); @@ -391,13 +391,13 @@ OSXEthernetTap::OSXEthernetTap( // Open the first unused tap device if we didn't recall a previous one. if (!recalledDevice) { for(int i=0;i<64;++i) { - Utils::snprintf(devpath,sizeof(devpath),"/dev/zt%d",i); + Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i); if (stat(devpath,&stattmp)) throw std::runtime_error("no more TAP devices available"); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { char foo[16]; - Utils::snprintf(foo,sizeof(foo),"zt%d",i); + Utils::ztsnprintf(foo,sizeof(foo),"zt%d",i); _dev = foo; break; } @@ -413,9 +413,9 @@ OSXEthernetTap::OSXEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::snprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::snprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::snprintf(metstr,sizeof(metstr),"%u",_metric); + Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -636,7 +636,7 @@ void OSXEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%u",mtu); + Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index 99286172..df868e7a 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -205,7 +205,7 @@ public: memset(externalip,0,sizeof(externalip)); memset(&urls,0,sizeof(urls)); memset(&data,0,sizeof(data)); - Utils::snprintf(inport,sizeof(inport),"%d",localPort); + Utils::ztsnprintf(inport,sizeof(inport),"%d",localPort); if ((UPNP_GetValidIGD(devlist,&urls,&data,lanaddr,sizeof(lanaddr)))&&(lanaddr[0])) { #ifdef ZT_PORTMAPPER_TRACE @@ -220,7 +220,7 @@ public: int tryPort = (int)localPort + tries; if (tryPort >= 65535) tryPort = (tryPort - 65535) + 1025; - Utils::snprintf(outport,sizeof(outport),"%u",tryPort); + Utils::ztsnprintf(outport,sizeof(outport),"%u",tryPort); // First check and see if this port is already mapped to the // same unique name. If so, keep this mapping and don't try diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index c5d82d8e..b96ad791 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -484,7 +484,7 @@ WindowsEthernetTap::WindowsEthernetTap( char tag[24]; // We "tag" registry entries with the network ID to identify persistent devices - Utils::snprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); + Utils::ztsnprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); Mutex::Lock _l(_systemTapInitLock); @@ -601,10 +601,10 @@ WindowsEthernetTap::WindowsEthernetTap( if (_netCfgInstanceId.length() > 0) { char tmps[64]; - unsigned int tmpsl = Utils::snprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; + unsigned int tmpsl = Utils::ztsnprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"NetworkAddress",REG_SZ,tmps,tmpsl); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MAC",REG_SZ,tmps,tmpsl); - tmpsl = Utils::snprintf(tmps, sizeof(tmps), "%d", mtu); + tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MTU",REG_SZ,tmps,tmpsl); DWORD tmp = 0; @@ -879,7 +879,7 @@ void WindowsEthernetTap::setMtu(unsigned int mtu) HKEY nwAdapters; if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\Class\\{4D36E972-E325-11CE-BFC1-08002BE10318}", 0, KEY_READ | KEY_WRITE, &nwAdapters) == ERROR_SUCCESS) { char tmps[64]; - unsigned int tmpsl = Utils::snprintf(tmps, sizeof(tmps), "%d", mtu); + unsigned int tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters, _mySubkeyName.c_str(), "MTU", REG_SZ, tmps, tmpsl); RegCloseKey(nwAdapters); } @@ -902,7 +902,7 @@ void WindowsEthernetTap::threadMain() HANDLE wait4[3]; OVERLAPPED tapOvlRead,tapOvlWrite; - Utils::snprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); + Utils::ztsnprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); try { while (_run) { diff --git a/selftest.cpp b/selftest.cpp index 8175d708..ff171aa3 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -831,7 +831,7 @@ static int testOther() memset(key, 0, sizeof(key)); memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { - Utils::snprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); + Utils::ztsnprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); int r = rand() % 128; for(int x=0;x lines(OSUtils::split(cf.c_str(),"\r\n","","")); for(std::vector::iterator l(lines.begin());l!=lines.end();++l) { diff --git a/service/OneService.cpp b/service/OneService.cpp index 644454bc..993fb116 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -210,10 +210,10 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc, case ZT_NETWORK_TYPE_PUBLIC: ntype = "PUBLIC"; break; } - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nc->nwid); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",nc->nwid); nj["id"] = tmp; nj["nwid"] = tmp; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); nj["mac"] = tmp; nj["name"] = nc->name; nj["status"] = nstatus; @@ -260,12 +260,12 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) case ZT_PEER_ROLE_PLANET: prole = "PLANET"; break; } - Utils::snprintf(tmp,sizeof(tmp),"%.10llx",peer->address); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); pj["address"] = tmp; pj["versionMajor"] = peer->versionMajor; pj["versionMinor"] = peer->versionMinor; pj["versionRev"] = peer->versionRev; - Utils::snprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); + Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); pj["version"] = tmp; pj["latency"] = peer->latency; pj["role"] = prole; @@ -289,7 +289,7 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) static void _moonToJson(nlohmann::json &mj,const World &world) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",world.id()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",world.id()); mj["id"] = tmp; mj["timestamp"] = world.timestamp(); mj["signature"] = Utils::hex(world.signature().data,(unsigned int)world.signature().size()); @@ -687,7 +687,7 @@ public: // Save primary port to a file so CLIs and GUIs can learn it easily char portstr[64]; - Utils::snprintf(portstr,sizeof(portstr),"%u",_ports[0]); + Utils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]); OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr)); // Attempt to bind to a secondary port chosen from our ZeroTier address. @@ -725,7 +725,7 @@ public: } if (_ports[2]) { char uniqueName[64]; - Utils::snprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); + Utils::ztsnprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); _portMapper = new PortMapper(_ports[2],uniqueName); } } @@ -1069,7 +1069,7 @@ public: n->second.settings = settings; char nlcpath[4096]; - Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); + Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); FILE *out = fopen(nlcpath,"w"); if (out) { fprintf(out,"allowManaged=%d\n",(int)n->second.settings.allowManaged); @@ -1188,7 +1188,7 @@ public: ZT_NodeStatus status; _node->status(&status); - Utils::snprintf(tmp,sizeof(tmp),"%.10llx",status.address); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",status.address); res["address"] = tmp; res["publicIdentity"] = status.publicIdentity; res["online"] = (bool)(status.online != 0); @@ -1197,7 +1197,7 @@ public: res["versionMinor"] = ZEROTIER_ONE_VERSION_MINOR; res["versionRev"] = ZEROTIER_ONE_VERSION_REVISION; res["versionBuild"] = ZEROTIER_ONE_VERSION_BUILD; - Utils::snprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); + Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); res["version"] = tmp; res["clock"] = OSUtils::now(); @@ -1373,7 +1373,7 @@ public: if ((scode != 200)&&(seed != 0)) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",id); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",id); res["id"] = tmp; res["roots"] = json::array(); res["timestamp"] = 0; @@ -1617,7 +1617,7 @@ public: std::string h = controllerDbHttpHost; _controllerDbPath.append(h); char dbp[128]; - Utils::snprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort); + Utils::ztsnprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort); _controllerDbPath.push_back(':'); _controllerDbPath.append(dbp); if (controllerDbHttpPath.is_string()) { @@ -1711,7 +1711,7 @@ public: if (syncRoutes) { char tapdev[64]; #ifdef __WINDOWS__ - Utils::snprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); + Utils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); #else Utils::scopy(tapdev,sizeof(tapdev),n.tap->deviceName().c_str()); #endif @@ -1933,24 +1933,24 @@ public: bool secure = false; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); secure = true; break; case ZT_STATE_OBJECT_PEER_IDENTITY: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); secure = true; break; case ZT_STATE_OBJECT_PLANET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); break; default: p[0] = (char)0; @@ -2022,7 +2022,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -2235,7 +2235,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -2402,7 +2402,7 @@ public: if (!n.tap) { try { char friendlyName[128]; - Utils::snprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); + Utils::ztsnprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); n.tap = new EthernetTap( _homePath.c_str(), @@ -2416,7 +2416,7 @@ public: *nuptr = (void *)&n; char nlcpath[256]; - Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); std::string nlcbuf; if (OSUtils::readFile(nlcpath,nlcbuf)) { Dictionary<4096> nc; @@ -2502,7 +2502,7 @@ public: #endif if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) { char nlcpath[256]; - Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); OSUtils::rm(nlcpath); } } else { @@ -2554,22 +2554,22 @@ public: char p[4096]; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); break; case ZT_STATE_OBJECT_PEER_IDENTITY: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_PLANET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); break; default: return -1; @@ -2765,7 +2765,7 @@ public: default: scodestr = "Error"; break; } - Utils::snprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + Utils::ztsnprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", scode, scodestr, contentType.c_str(), diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index d94beab5..e0519827 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -284,7 +284,7 @@ bool SoftwareUpdater::check(const uint64_t now) if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { _lastCheckTime = now; char tmp[512]; - const unsigned int len = Utils::snprintf(tmp,sizeof(tmp), + const unsigned int len = Utils::ztsnprintf(tmp,sizeof(tmp), "%c{\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "\":%d," -- cgit v1.2.3 From d2415dee00914ab3fd7016758f4184d46bb407a5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 16:11:11 -0700 Subject: Cleanup. --- controller/EmbeddedNetworkController.cpp | 104 +++++++++-------- controller/JSONDB.cpp | 14 +-- node/Address.hpp | 15 +-- node/CertificateOfMembership.cpp | 7 +- node/Dictionary.hpp | 6 +- node/Identity.cpp | 26 +++-- node/Identity.hpp | 18 +-- node/InetAddress.cpp | 195 ++++++++++++++----------------- node/InetAddress.hpp | 94 ++++++--------- node/MAC.hpp | 83 +++---------- node/MulticastGroup.hpp | 12 -- node/Network.cpp | 14 ++- node/NetworkConfig.cpp | 12 +- node/Node.cpp | 51 ++------ node/Node.hpp | 4 - node/RuntimeEnvironment.hpp | 10 +- node/Topology.cpp | 8 +- node/Utils.cpp | 114 ++++-------------- node/Utils.hpp | 182 +++++++++++++++++++++-------- one.cpp | 55 +++++---- osdep/BSDEthernetTap.cpp | 19 +-- osdep/Http.cpp | 4 +- osdep/LinuxEthernetTap.cpp | 33 +++--- osdep/ManagedRoute.cpp | 11 +- osdep/OSUtils.cpp | 28 ++++- osdep/OSUtils.hpp | 15 ++- osdep/OSXEthernetTap.cpp | 20 ++-- osdep/PortMapper.cpp | 4 +- osdep/WindowsEthernetTap.cpp | 10 +- selftest.cpp | 31 +++-- service/OneService.cpp | 94 +++++++-------- service/SoftwareUpdater.cpp | 5 +- 32 files changed, 620 insertions(+), 678 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 85c759e7..b57a37e8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -76,19 +76,19 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_ACTION_TEE: r["type"] = "ACTION_TEE"; - r["address"] = Address(rule.v.fwd.address).toString(); + r["address"] = Address(rule.v.fwd.address).toString(tmp); r["flags"] = (unsigned int)rule.v.fwd.flags; r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_WATCH: r["type"] = "ACTION_WATCH"; - r["address"] = Address(rule.v.fwd.address).toString(); + r["address"] = Address(rule.v.fwd.address).toString(tmp); r["flags"] = (unsigned int)rule.v.fwd.flags; r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; - r["address"] = Address(rule.v.fwd.address).toString(); + r["address"] = Address(rule.v.fwd.address).toString(tmp); r["flags"] = (unsigned int)rule.v.fwd.flags; break; case ZT_NETWORK_RULE_ACTION_BREAK: @@ -102,11 +102,11 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) switch(rt) { case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); + r["zt"] = Address(rule.v.zt).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); + r["zt"] = Address(rule.v.zt).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: r["type"] = "MATCH_VLAN_ID"; @@ -122,29 +122,29 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: r["type"] = "MATCH_MAC_DEST"; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: r["type"] = "MATCH_IPV4_SOURCE"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: r["type"] = "MATCH_IPV4_DEST"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: r["type"] = "MATCH_IPV6_SOURCE"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: r["type"] = "MATCH_IPV6_DEST"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IP_TOS: r["type"] = "MATCH_IP_TOS"; @@ -179,7 +179,7 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); r["mask"] = tmp; break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: @@ -312,28 +312,28 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_IPV4_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; - InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0").c_str()); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV4_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; - InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0").c_str()); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV6_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; - InetAddress ip(OSUtils::jsonString(r["ip"],"::0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"::0").c_str()); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IPV6_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; - InetAddress ip(OSUtils::jsonString(r["ip"],"::0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"::0").c_str()); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; @@ -514,7 +514,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( _db.eachMember(nwid,[&responseBody](uint64_t networkId,uint64_t nodeId,const json &member) { if ((member.is_object())&&(member.size() > 0)) { char tmp[128]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); responseBody.append(tmp); } }); @@ -548,7 +548,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( for(std::vector::const_iterator i(networkIds.begin());i!=networkIds.end();++i) { if (responseBody.length() > 1) responseBody.push_back(','); - Utils::ztsnprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); responseBody.append(tmp); } responseBody.push_back(']'); @@ -562,7 +562,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( // Controller status char tmp[4096]; - Utils::ztsnprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); responseBody = tmp; responseContentType = "application/json"; return 200; @@ -603,14 +603,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if ((path.size() >= 2)&&(path[1].length() == 16)) { uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { uint64_t address = Utils::hexStrToU64(path[3].c_str()); char addrs[24]; - Utils::ztsnprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); + OSUtils::ztsnprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); json member; _db.getNetworkMember(nwid,address,member); @@ -655,9 +655,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json mipa(json::array()); for(unsigned long i=0;i()); + InetAddress t(target.get().c_str()); InetAddress v; - if (via.is_string()) v.fromString(via.get()); + if (via.is_string()) v.fromString(via.get().c_str()); if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.netmaskBitsValid()) ) { json tmp; - tmp["target"] = t.toString(); + char tmp2[64]; + tmp["target"] = t.toString(tmp2); if (v.ss_family == t.ss_family) - tmp["via"] = v.toIpString(); + tmp["via"] = v.toIpString(tmp2); else tmp["via"] = json(); nrts.push_back(tmp); } @@ -840,12 +842,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i_memberStatus.find(_MemberStatusKey(networkId,nodeId)); if (ms != _memberStatus.end()) lrt = ms->second.lastRequestTime; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, @@ -1093,7 +1096,7 @@ void EmbeddedNetworkController::threadMain() }); } char tmp2[256]; - Utils::ztsnprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + OSUtils::ztsnprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); pong.append(tmp2); _db.writeRaw("pong",pong); } @@ -1126,7 +1129,7 @@ void EmbeddedNetworkController::_request( ms.lastRequestTime = now; } - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; @@ -1152,13 +1155,15 @@ void EmbeddedNetworkController::_request( } } else { // If we do not yet know this member's identity, learn it. - member["identity"] = identity.toString(false); + char idtmp[1024]; + member["identity"] = identity.toString(false,idtmp); } } // These are always the same, but make sure they are set { - const std::string addrs(identity.address().toString()); + char tmpid[128]; + const std::string addrs(identity.address().toString(tmpid)); member["id"] = addrs; member["address"] = addrs; member["nwid"] = nwids; @@ -1264,8 +1269,9 @@ void EmbeddedNetworkController::_request( if (fromAddr) ms.physicalAddr = fromAddr; + char tmpip[64]; if (ms.physicalAddr) - member["physicalAddr"] = ms.physicalAddr.toString(); + member["physicalAddr"] = ms.physicalAddr.toString(tmpip); } } } else { @@ -1427,9 +1433,9 @@ void EmbeddedNetworkController::_request( json &target = route["target"]; json &via = route["via"]; if (target.is_string()) { - const InetAddress t(target.get()); + const InetAddress t(target.get().c_str()); InetAddress v; - if (via.is_string()) v.fromString(via.get()); + if (via.is_string()) v.fromString(via.get().c_str()); if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { ZT_VirtualNetworkRoute *r = &(nc->routes[nc->routeCount]); *(reinterpret_cast(&(r->target))) = t; @@ -1462,7 +1468,7 @@ void EmbeddedNetworkController::_request( if (!ipAssignments[i].is_string()) continue; std::string ips = ipAssignments[i]; - InetAddress ip(ips); + InetAddress ip(ips.c_str()); // IP assignments are only pushed if there is a corresponding local route. We also now get the netmask bits from // this route, ignoring the netmask bits field of the assigned IP itself. Using that was worthless and a source @@ -1492,8 +1498,8 @@ void EmbeddedNetworkController::_request( for(unsigned long p=0;((p 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip6)) ) { - ipAssignments.push_back(ip6.toIpString()); + char tmpip[64]; + ipAssignments.push_back(ip6.toIpString(tmpip)); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) @@ -1552,8 +1559,8 @@ void EmbeddedNetworkController::_request( for(unsigned long p=0;((p(&ipRangeStartIA)->sin_addr.s_addr)); uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); @@ -1586,7 +1593,8 @@ void EmbeddedNetworkController::_request( // If it's routed, then try to claim and assign it and if successful end loop const InetAddress ip4(Utils::hton(ip),0); if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip4)) ) { - ipAssignments.push_back(ip4.toIpString()); + char tmpip[64]; + ipAssignments.push_back(ip4.toIpString(tmpip)); member["ipAssignments"] = ipAssignments; if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { struct sockaddr_in *const v4ip = reinterpret_cast(&(nc->staticIps[nc->staticIpCount++])); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index acf23700..97a217a1 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -39,7 +39,7 @@ JSONDB::JSONDB(const std::string &basePath) : std::size_t hnsep = hn.find_last_of(':'); if (hnsep != std::string::npos) hn[hnsep] = '/'; - _httpAddr.fromString(hn); + _httpAddr.fromString(hn.c_str()); if (hnend != std::string::npos) _basePath = _basePath.substr(7 + hnend); if (_basePath.length() == 0) @@ -94,7 +94,7 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) std::string body; std::map reqHeaders; char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); reqHeaders["Content-Length"] = tmp; reqHeaders["Content-Type"] = "application/json"; const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); @@ -164,7 +164,7 @@ bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlo void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) { char n[64]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig)); { Mutex::Lock _l(_networks_m); @@ -176,7 +176,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig) { char n[256]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); writeRaw(n,OSUtils::jsonDump(memberConfig)); { Mutex::Lock _l(_networks_m); @@ -202,7 +202,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) } char n[256]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); if (_httpAddr) { // Deletion is currently done by Central in harnessed mode @@ -229,7 +229,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo) { char n[256]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); if (_httpAddr) { // Deletion is currently done by the caller in Central harnessed mode @@ -314,7 +314,7 @@ void JSONDB::threadMain() const nlohmann::json &mips = member["ipAssignments"]; if (mips.is_array()) { for(unsigned long i=0;iadd(key,tmp,-1); + return this->add(key,Utils::hex(value,tmp),-1); } /** @@ -401,8 +400,7 @@ public: inline bool add(const char *key,const Address &a) { char tmp[32]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",(unsigned long long)a.toInt()); - return this->add(key,tmp,-1); + return this->add(key,Utils::hex(a.toInt(),tmp),-1); } /** diff --git a/node/Identity.cpp b/node/Identity.cpp index ba77aa47..3b00b4c0 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -136,19 +136,23 @@ bool Identity::locallyValidate() const (digest[63] == addrb[4])); } -std::string Identity::toString(bool includePrivate) const +char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_LENGTH]) const { - std::string r; - - r.append(_address.toString()); - r.append(":0:"); // 0 == ZT_OBJECT_TYPE_IDENTITY - r.append(Utils::hex(_publicKey.data,(unsigned int)_publicKey.size())); + char *p = buf; + Utils::hex10(_address.toInt(),p); + p += 10; + *(p++) = ':'; + *(p++) = '0'; + *(p++) = ':'; + Utils::hex(_publicKey.data,ZT_C25519_PUBLIC_KEY_LEN,p); + p += ZT_C25519_PUBLIC_KEY_LEN * 2; if ((_privateKey)&&(includePrivate)) { - r.push_back(':'); - r.append(Utils::hex(_privateKey->data,(unsigned int)_privateKey->size())); + *(p++) = ':'; + Utils::hex(_privateKey->data,ZT_C25519_PRIVATE_KEY_LEN,p); + p += ZT_C25519_PRIVATE_KEY_LEN * 2; } - - return r; + *(p++) = (char)0; + return buf; } bool Identity::fromString(const char *str) @@ -157,7 +161,7 @@ bool Identity::fromString(const char *str) return false; char *saveptr = (char *)0; - char tmp[1024]; + char tmp[ZT_IDENTITY_STRING_BUFFER_LENGTH]; if (!Utils::scopy(tmp,sizeof(tmp),str)) return false; diff --git a/node/Identity.hpp b/node/Identity.hpp index 79e17f4d..5804b9f8 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -29,7 +29,6 @@ #include #include -#include #include "Constants.hpp" #include "Array.hpp" @@ -39,6 +38,8 @@ #include "Buffer.hpp" #include "SHA512.hpp" +#define ZT_IDENTITY_STRING_BUFFER_LENGTH 384 + namespace ZeroTier { /** @@ -66,16 +67,7 @@ public: { } - Identity(const char *str) - throw(std::invalid_argument) : - _privateKey((C25519::Private *)0) - { - if (!fromString(str)) - throw std::invalid_argument(std::string("invalid string-serialized identity: ") + str); - } - - Identity(const std::string &str) - throw(std::invalid_argument) : + Identity(const char *str) : _privateKey((C25519::Private *)0) { if (!fromString(str)) @@ -277,9 +269,10 @@ public: * Serialize to a more human-friendly string * * @param includePrivate If true, include private key (if it exists) + * @param buf Buffer to store string * @return ASCII string representation of identity */ - std::string toString(bool includePrivate) const; + char *toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_LENGTH]) const; /** * Deserialize a human-friendly string @@ -291,7 +284,6 @@ public: * @return True if deserialization appears successful */ bool fromString(const char *str); - inline bool fromString(const std::string &str) { return fromString(str.c_str()); } /** * @return C25519 public key diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 17d7c72e..f7585bdb 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -5,7 +5,7 @@ * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * (at your oion) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -40,7 +40,6 @@ const InetAddress InetAddress::LO4((const void *)("\x7f\x00\x00\x01"),4,0); const InetAddress InetAddress::LO6((const void *)("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01"),16,0); InetAddress::IpScope InetAddress::ipScope() const - throw() { switch(ss_family) { @@ -111,27 +110,7 @@ InetAddress::IpScope InetAddress::ipScope() const return IP_SCOPE_NONE; } -void InetAddress::set(const std::string &ip,unsigned int port) - throw() -{ - memset(this,0,sizeof(InetAddress)); - if (ip.find(':') != std::string::npos) { - struct sockaddr_in6 *sin6 = reinterpret_cast(this); - ss_family = AF_INET6; - sin6->sin6_port = Utils::hton((uint16_t)port); - if (inet_pton(AF_INET6,ip.c_str(),(void *)&(sin6->sin6_addr.s6_addr)) <= 0) - memset(this,0,sizeof(InetAddress)); - } else if (ip.find('.') != std::string::npos) { - struct sockaddr_in *sin = reinterpret_cast(this); - ss_family = AF_INET; - sin->sin_port = Utils::hton((uint16_t)port); - if (inet_pton(AF_INET,ip.c_str(),(void *)&(sin->sin_addr.s_addr)) <= 0) - memset(this,0,sizeof(InetAddress)); - } -} - void InetAddress::set(const void *ipBytes,unsigned int ipLen,unsigned int port) - throw() { memset(this,0,sizeof(InetAddress)); if (ipLen == 4) { @@ -147,90 +126,98 @@ void InetAddress::set(const void *ipBytes,unsigned int ipLen,unsigned int port) } } -std::string InetAddress::toString() const +char *InetAddress::toString(char buf[64]) const { - char buf[128]; - switch(ss_family) { - case AF_INET: - Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d/%d", - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[3], - (int)Utils::ntoh((uint16_t)(reinterpret_cast(this)->sin_port)) - ); - return std::string(buf); - case AF_INET6: - Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x/%d", - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[3]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[4]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[5]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[6]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[7]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[8]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[9]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[10]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[11]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[12]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[13]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[14]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[15]), - (int)Utils::ntoh((uint16_t)(reinterpret_cast(this)->sin6_port)) - ); - return std::string(buf); + char *p = toIpString(buf); + if (*p) { + while (*p) ++p; + *(p++) = '/'; + Utils::decimal(port(),p); } - return std::string(); + return buf; } -std::string InetAddress::toIpString() const +char *InetAddress::toIpString(char buf[64]) const { - char buf[128]; switch(ss_family) { - case AF_INET: - Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d", - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[3] - ); - return std::string(buf); - case AF_INET6: - Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x", - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[3]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[4]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[5]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[6]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[7]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[8]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[9]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[10]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[11]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[12]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[13]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[14]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[15]) - ); - return std::string(buf); + case AF_INET: { + const uint8_t *a = reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)); + char *p = buf; + for(int i=0;;++i) { + Utils::decimal((unsigned long)a[i],p); + if (i != 3) { + while (*p) ++p; + *(p++) = '.'; + } else break; + } + } break; + + case AF_INET6: { + uint16_t a[8]; + memcpy(a,reinterpret_cast(this)->sin6_addr.s6_addr,16); + char *p = buf; + for(int i=0;i<8;++i) { + Utils::hex(Utils::ntoh(a[i]),p); + p[4] = (i == 7) ? (char)0 : ':'; + p += 5; + } + } break; + + default: + buf[0] = (char)0; + break; } - return std::string(); + return buf; } -void InetAddress::fromString(const std::string &ipSlashPort) +bool InetAddress::fromString(const char *ipSlashPort) { - const std::size_t slashAt = ipSlashPort.find('/'); - if (slashAt == std::string::npos) { - set(ipSlashPort,0); + char buf[64]; + + memset(this,0,sizeof(InetAddress)); + + if (!*ipSlashPort) + return true; + if (!Utils::scopy(buf,sizeof(buf),ipSlashPort)) + return false; + + char *portAt = buf; + while ((*portAt)&&(*portAt != '/')) + ++portAt; + unsigned int port = 0; + if (*portAt) { + *(portAt++) = (char)0; + port = Utils::strToUInt(portAt) & 0xffff; + } + + if (strchr(buf,':')) { + uint16_t a[8]; + unsigned int b = 0; + char *saveptr = (char *)0; + for(char *s=Utils::stok(buf,":",&saveptr);((s)&&(b<8));s=Utils::stok((char *)0,":",&saveptr)) + a[b++] = Utils::hton((uint16_t)(Utils::hexStrToUInt(s) & 0xffff)); + + struct sockaddr_in6 *const in6 = reinterpret_cast(this); + in6->sin6_family = AF_INET6; + memcpy(in6->sin6_addr.s6_addr,a,16); + in6->sin6_port = Utils::hton((uint16_t)port); + + return true; + } else if (strchr(buf,'.')) { + uint8_t a[4]; + unsigned int b = 0; + char *saveptr = (char *)0; + for(char *s=Utils::stok(buf,".",&saveptr);((s)&&(b<4));s=Utils::stok((char *)0,".",&saveptr)) + a[b++] = (uint8_t)(Utils::strToUInt(s) & 0xff); + + struct sockaddr_in *const in = reinterpret_cast(this); + in->sin_family = AF_INET; + memcpy(&(in->sin_addr.s_addr),a,4); + in->sin_port = Utils::hton((uint16_t)port); + + return true; } else { - long p = strtol(ipSlashPort.substr(slashAt+1).c_str(),(char **)0,10); - if ((p > 0)&&(p <= 0xffff)) - set(ipSlashPort.substr(0,slashAt),(unsigned int)p); - else set(ipSlashPort.substr(0,slashAt),0); + return false; } } @@ -244,14 +231,13 @@ InetAddress InetAddress::netmask() const case AF_INET6: { uint64_t nm[2]; const unsigned int bits = netmaskBits(); - if(bits) { - nm[0] = Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits)))); - nm[1] = Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits)))); - } - else { - nm[0] = 0; - nm[1] = 0; - } + if(bits) { + nm[0] = Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits)))); + nm[1] = Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits)))); + } else { + nm[0] = 0; + nm[1] = 0; + } memcpy(reinterpret_cast(&r)->sin6_addr.s6_addr,nm,16); } break; } @@ -338,7 +324,6 @@ bool InetAddress::containsAddress(const InetAddress &addr) const } bool InetAddress::isNetwork() const - throw() { switch(ss_family) { case AF_INET: { @@ -371,7 +356,6 @@ bool InetAddress::isNetwork() const } bool InetAddress::operator==(const InetAddress &a) const - throw() { if (ss_family == a.ss_family) { switch(ss_family) { @@ -395,7 +379,6 @@ bool InetAddress::operator==(const InetAddress &a) const } bool InetAddress::operator<(const InetAddress &a) const - throw() { if (ss_family < a.ss_family) return true; diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 4cb9a4dc..dd055084 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -31,8 +31,6 @@ #include #include -#include - #include "Constants.hpp" #include "../include/ZeroTierOne.h" #include "Utils.hpp" @@ -85,25 +83,22 @@ struct InetAddress : public sockaddr_storage IP_SCOPE_PRIVATE = 7 // 10.x.x.x, 192.168.x.x, etc. }; - InetAddress() throw() { memset(this,0,sizeof(InetAddress)); } - InetAddress(const InetAddress &a) throw() { memcpy(this,&a,sizeof(InetAddress)); } - InetAddress(const InetAddress *a) throw() { memcpy(this,a,sizeof(InetAddress)); } - InetAddress(const struct sockaddr_storage &ss) throw() { *this = ss; } - InetAddress(const struct sockaddr_storage *ss) throw() { *this = ss; } - InetAddress(const struct sockaddr &sa) throw() { *this = sa; } - InetAddress(const struct sockaddr *sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in &sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in *sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in6 &sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in6 *sa) throw() { *this = sa; } - InetAddress(const void *ipBytes,unsigned int ipLen,unsigned int port) throw() { this->set(ipBytes,ipLen,port); } - InetAddress(const uint32_t ipv4,unsigned int port) throw() { this->set(&ipv4,4,port); } - InetAddress(const std::string &ip,unsigned int port) throw() { this->set(ip,port); } - InetAddress(const std::string &ipSlashPort) throw() { this->fromString(ipSlashPort); } - InetAddress(const char *ipSlashPort) throw() { this->fromString(std::string(ipSlashPort)); } + InetAddress() { memset(this,0,sizeof(InetAddress)); } + InetAddress(const InetAddress &a) { memcpy(this,&a,sizeof(InetAddress)); } + InetAddress(const InetAddress *a) { memcpy(this,a,sizeof(InetAddress)); } + InetAddress(const struct sockaddr_storage &ss) { *this = ss; } + InetAddress(const struct sockaddr_storage *ss) { *this = ss; } + InetAddress(const struct sockaddr &sa) { *this = sa; } + InetAddress(const struct sockaddr *sa) { *this = sa; } + InetAddress(const struct sockaddr_in &sa) { *this = sa; } + InetAddress(const struct sockaddr_in *sa) { *this = sa; } + InetAddress(const struct sockaddr_in6 &sa) { *this = sa; } + InetAddress(const struct sockaddr_in6 *sa) { *this = sa; } + InetAddress(const void *ipBytes,unsigned int ipLen,unsigned int port) { this->set(ipBytes,ipLen,port); } + InetAddress(const uint32_t ipv4,unsigned int port) { this->set(&ipv4,4,port); } + InetAddress(const char *ipSlashPort) { this->fromString(ipSlashPort); } inline InetAddress &operator=(const InetAddress &a) - throw() { if (&a != this) memcpy(this,&a,sizeof(InetAddress)); @@ -111,7 +106,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const InetAddress *a) - throw() { if (a != this) memcpy(this,a,sizeof(InetAddress)); @@ -119,7 +113,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_storage &ss) - throw() { if (reinterpret_cast(&ss) != this) memcpy(this,&ss,sizeof(InetAddress)); @@ -127,7 +120,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_storage *ss) - throw() { if (reinterpret_cast(ss) != this) memcpy(this,ss,sizeof(InetAddress)); @@ -135,7 +127,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in &sa) - throw() { if (reinterpret_cast(&sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -145,7 +136,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in *sa) - throw() { if (reinterpret_cast(sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -155,7 +145,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in6 &sa) - throw() { if (reinterpret_cast(&sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -165,7 +154,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in6 *sa) - throw() { if (reinterpret_cast(sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -175,7 +163,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr &sa) - throw() { if (reinterpret_cast(&sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -192,7 +179,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr *sa) - throw() { if (reinterpret_cast(sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -211,17 +197,7 @@ struct InetAddress : public sockaddr_storage /** * @return IP scope classification (e.g. loopback, link-local, private, global) */ - IpScope ipScope() const - throw(); - - /** - * Set from a string-format IP and a port - * - * @param ip IP address in V4 or V6 ASCII notation - * @param port Port or 0 for none - */ - void set(const std::string &ip,unsigned int port) - throw(); + IpScope ipScope() const; /** * Set from a raw IP and port number @@ -230,8 +206,7 @@ struct InetAddress : public sockaddr_storage * @param ipLen Length of IP address: 4 or 16 * @param port Port number or 0 for none */ - void set(const void *ipBytes,unsigned int ipLen,unsigned int port) - throw(); + void set(const void *ipBytes,unsigned int ipLen,unsigned int port); /** * Set the port component @@ -272,23 +247,23 @@ struct InetAddress : public sockaddr_storage /** * @return ASCII IP/port format representation */ - std::string toString() const; + char *toString(char buf[64]) const; /** * @return IP portion only, in ASCII string format */ - std::string toIpString() const; + char *toIpString(char buf[64]) const; /** - * @param ipSlashPort ASCII IP/port format notation + * @param ipSlashPort IP/port (port is optional, will be 0 if not included) + * @return True if address appeared to be valid */ - void fromString(const std::string &ipSlashPort); + bool fromString(const char *ipSlashPort); /** * @return Port or 0 if no port component defined */ inline unsigned int port() const - throw() { switch(ss_family) { case AF_INET: return Utils::ntoh((uint16_t)(reinterpret_cast(this)->sin_port)); @@ -306,7 +281,7 @@ struct InetAddress : public sockaddr_storage * * @return Netmask bits */ - inline unsigned int netmaskBits() const throw() { return port(); } + inline unsigned int netmaskBits() const { return port(); } /** * @return True if netmask bits is valid for the address type @@ -329,7 +304,7 @@ struct InetAddress : public sockaddr_storage * * @return Gateway metric */ - inline unsigned int metric() const throw() { return port(); } + inline unsigned int metric() const { return port(); } /** * Construct a full netmask as an InetAddress @@ -376,12 +351,12 @@ struct InetAddress : public sockaddr_storage /** * @return True if this is an IPv4 address */ - inline bool isV4() const throw() { return (ss_family == AF_INET); } + inline bool isV4() const { return (ss_family == AF_INET); } /** * @return True if this is an IPv6 address */ - inline bool isV6() const throw() { return (ss_family == AF_INET6); } + inline bool isV6() const { return (ss_family == AF_INET6); } /** * @return pointer to raw address bytes or NULL if not available @@ -454,7 +429,7 @@ struct InetAddress : public sockaddr_storage /** * Set to null/zero */ - inline void zero() throw() { memset(this,0,sizeof(InetAddress)); } + inline void zero() { memset(this,0,sizeof(InetAddress)); } /** * Check whether this is a network/route rather than an IP assignment @@ -464,8 +439,7 @@ struct InetAddress : public sockaddr_storage * * @return True if everything after netmask bits is zero */ - bool isNetwork() const - throw(); + bool isNetwork() const; /** * @return 14-bit (0-16383) hash of this IP's first 24 or 48 bits (for V4 or V6) for rate limiting code, or 0 if non-IP @@ -494,7 +468,7 @@ struct InetAddress : public sockaddr_storage /** * @return True if address family is non-zero */ - inline operator bool() const throw() { return (ss_family != 0); } + inline operator bool() const { return (ss_family != 0); } template inline void serialize(Buffer &b) const @@ -552,12 +526,12 @@ struct InetAddress : public sockaddr_storage return (p - startAt); } - bool operator==(const InetAddress &a) const throw(); - bool operator<(const InetAddress &a) const throw(); - inline bool operator!=(const InetAddress &a) const throw() { return !(*this == a); } - inline bool operator>(const InetAddress &a) const throw() { return (a < *this); } - inline bool operator<=(const InetAddress &a) const throw() { return !(a < *this); } - inline bool operator>=(const InetAddress &a) const throw() { return !(*this < a); } + bool operator==(const InetAddress &a) const; + bool operator<(const InetAddress &a) const; + inline bool operator!=(const InetAddress &a) const { return !(*this == a); } + inline bool operator>(const InetAddress &a) const { return (a < *this); } + inline bool operator<=(const InetAddress &a) const { return !(a < *this); } + inline bool operator>=(const InetAddress &a) const { return !(*this < a); } /** * @param mac MAC address seed diff --git a/node/MAC.hpp b/node/MAC.hpp index db50aeb1..52388d59 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -44,30 +44,24 @@ namespace ZeroTier { class MAC { public: - MAC() throw() : _m(0ULL) {} - MAC(const MAC &m) throw() : _m(m._m) {} + MAC() : _m(0ULL) {} + MAC(const MAC &m) : _m(m._m) {} - MAC(const unsigned char a,const unsigned char b,const unsigned char c,const unsigned char d,const unsigned char e,const unsigned char f) throw() : + MAC(const unsigned char a,const unsigned char b,const unsigned char c,const unsigned char d,const unsigned char e,const unsigned char f) : _m( ((((uint64_t)a) & 0xffULL) << 40) | ((((uint64_t)b) & 0xffULL) << 32) | ((((uint64_t)c) & 0xffULL) << 24) | ((((uint64_t)d) & 0xffULL) << 16) | ((((uint64_t)e) & 0xffULL) << 8) | (((uint64_t)f) & 0xffULL) ) {} - - MAC(const char *s) throw() { fromString(s); } - MAC(const std::string &s) throw() { fromString(s.c_str()); } - - MAC(const void *bits,unsigned int len) throw() { setTo(bits,len); } - - MAC(const Address &ztaddr,uint64_t nwid) throw() { fromAddress(ztaddr,nwid); } - - MAC(const uint64_t m) throw() : _m(m & 0xffffffffffffULL) {} + MAC(const void *bits,unsigned int len) { setTo(bits,len); } + MAC(const Address &ztaddr,uint64_t nwid) { fromAddress(ztaddr,nwid); } + MAC(const uint64_t m) : _m(m & 0xffffffffffffULL) {} /** * @return MAC in 64-bit integer */ - inline uint64_t toInt() const throw() { return _m; } + inline uint64_t toInt() const { return _m; } /** * Set MAC to zero @@ -77,14 +71,13 @@ public: /** * @return True if MAC is non-zero */ - inline operator bool() const throw() { return (_m != 0ULL); } + inline operator bool() const { return (_m != 0ULL); } /** * @param bits Raw MAC in big-endian byte order * @param len Length, must be >= 6 or result is zero */ inline void setTo(const void *bits,unsigned int len) - throw() { if (len < 6) { _m = 0ULL; @@ -104,7 +97,6 @@ public: * @param len Length of buffer, must be >= 6 or nothing is copied */ inline void copyTo(void *buf,unsigned int len) const - throw() { if (len < 6) return; @@ -124,7 +116,6 @@ public: */ template inline void appendTo(Buffer &b) const - throw(std::out_of_range) { unsigned char *p = (unsigned char *)b.appendField(6); *(p++) = (unsigned char)((_m >> 40) & 0xff); @@ -138,48 +129,17 @@ public: /** * @return True if this is broadcast (all 0xff) */ - inline bool isBroadcast() const throw() { return (_m == 0xffffffffffffULL); } + inline bool isBroadcast() const { return (_m == 0xffffffffffffULL); } /** * @return True if this is a multicast MAC */ - inline bool isMulticast() const throw() { return ((_m & 0x010000000000ULL) != 0ULL); } + inline bool isMulticast() const { return ((_m & 0x010000000000ULL) != 0ULL); } /** * @param True if this is a locally-administered MAC */ - inline bool isLocallyAdministered() const throw() { return ((_m & 0x020000000000ULL) != 0ULL); } - - /** - * @param s Hex MAC, with or without : delimiters - */ - inline void fromString(const char *s) - { - char tmp[8]; - for(int i=0;i<6;++i) - tmp[i] = (char)0; - Utils::unhex(s,tmp,6); - setTo(tmp,6); - } - - /** - * @return MAC address in standard :-delimited hex format - */ - inline std::string toString() const - { - char tmp[24]; - toString(tmp,sizeof(tmp)); - return std::string(tmp); - } - - /** - * @param buf Buffer to contain human-readable MAC - * @param len Length of buffer - */ - inline void toString(char *buf,unsigned int len) const - { - Utils::ztsnprintf(buf,len,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)(*this)[0],(int)(*this)[1],(int)(*this)[2],(int)(*this)[3],(int)(*this)[4],(int)(*this)[5]); - } + inline bool isLocallyAdministered() const { return ((_m & 0x020000000000ULL) != 0ULL); } /** * Set this MAC to a MAC derived from an address and a network ID @@ -188,7 +148,6 @@ public: * @param nwid 64-bit network ID */ inline void fromAddress(const Address &ztaddr,uint64_t nwid) - throw() { uint64_t m = ((uint64_t)firstOctetForNetwork(nwid)) << 40; m |= ztaddr.toInt(); // a is 40 bits @@ -208,7 +167,6 @@ public: * @param nwid Network ID */ inline Address toAddress(uint64_t nwid) const - throw() { uint64_t a = _m & 0xffffffffffULL; // least significant 40 bits of MAC are formed from address a ^= ((nwid >> 8) & 0xff) << 32; // ... XORed with bits 8-48 of the nwid in little-endian byte order, so unmask it @@ -224,7 +182,6 @@ public: * @return First octet of MAC for this network */ static inline unsigned char firstOctetForNetwork(uint64_t nwid) - throw() { unsigned char a = ((unsigned char)(nwid & 0xfe) | 0x02); // locally administered, not multicast, from LSB of network ID return ((a == 0x52) ? 0x32 : a); // blacklist 0x52 since it's used by KVM, libvirt, and other popular virtualization engines... seems de-facto standard on Linux @@ -239,29 +196,27 @@ public: /** * @return 6, which is the number of bytes in a MAC, for container compliance */ - inline unsigned int size() const throw() { return 6; } + inline unsigned int size() const { return 6; } - inline unsigned long hashCode() const throw() { return (unsigned long)_m; } + inline unsigned long hashCode() const { return (unsigned long)_m; } inline MAC &operator=(const MAC &m) - throw() { _m = m._m; return *this; } inline MAC &operator=(const uint64_t m) - throw() { _m = m; return *this; } - inline bool operator==(const MAC &m) const throw() { return (_m == m._m); } - inline bool operator!=(const MAC &m) const throw() { return (_m != m._m); } - inline bool operator<(const MAC &m) const throw() { return (_m < m._m); } - inline bool operator<=(const MAC &m) const throw() { return (_m <= m._m); } - inline bool operator>(const MAC &m) const throw() { return (_m > m._m); } - inline bool operator>=(const MAC &m) const throw() { return (_m >= m._m); } + inline bool operator==(const MAC &m) const { return (_m == m._m); } + inline bool operator!=(const MAC &m) const { return (_m != m._m); } + inline bool operator<(const MAC &m) const { return (_m < m._m); } + inline bool operator<=(const MAC &m) const { return (_m <= m._m); } + inline bool operator>(const MAC &m) const { return (_m > m._m); } + inline bool operator>=(const MAC &m) const { return (_m >= m._m); } private: uint64_t _m; diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index 7cbec2e0..f56c675b 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -29,8 +29,6 @@ #include -#include - #include "MAC.hpp" #include "InetAddress.hpp" @@ -94,16 +92,6 @@ public: return MulticastGroup(); } - /** - * @return Human readable string representing this group (MAC/ADI in hex) - */ - inline std::string toString() const - { - char buf[64]; - Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%.8lx",(unsigned int)_mac[0],(unsigned int)_mac[1],(unsigned int)_mac[2],(unsigned int)_mac[3],(unsigned int)_mac[4],(unsigned int)_mac[5],(unsigned long)_adi); - return std::string(buf); - } - /** * @return Multicast address */ diff --git a/node/Network.cpp b/node/Network.cpp index bccc0397..f2b6771b 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -51,7 +51,7 @@ namespace ZeroTier { namespace { #ifdef ZT_RULES_ENGINE_DEBUGGING -#define FILTER_TRACE(f,...) { Utils::ztsnprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } +#define FILTER_TRACE(f,...) { snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } static const char *_rtn(const ZT_VirtualNetworkRuleType rt) { switch(rt) { @@ -1257,7 +1257,17 @@ void Network::requestConfiguration(void *tPtr) nconf->rules[13].t = (uint8_t)ZT_NETWORK_RULE_ACTION_DROP; nconf->type = ZT_NETWORK_TYPE_PUBLIC; - Utils::ztsnprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); + + nconf->name[0] = 'a'; + nconf->name[1] = 'd'; + nconf->name[2] = 'h'; + nconf->name[3] = 'o'; + nconf->name[4] = 'c'; + nconf->name[5] = '-'; + Utils::hex((uint16_t)startPortRange,nconf->name + 6); + nconf->name[10] = '-'; + Utils::hex((uint16_t)endPortRange,nconf->name + 11); + nconf->name[15] = (char)0; this->setConfiguration(tPtr,*nconf,false); delete nconf; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 65101c3a..e5929923 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -64,7 +64,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (this->staticIps[i].ss_family == AF_INET) { if (v4s.length() > 0) v4s.push_back(','); - v4s.append(this->staticIps[i].toString()); + char buf[64]; + v4s.append(this->staticIps[i].toString(buf)); } } if (v4s.length() > 0) { @@ -75,7 +76,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (this->staticIps[i].ss_family == AF_INET6) { if (v6s.length() > 0) v6s.push_back(','); - v6s.append(this->staticIps[i].toString()); + char buf[64]; + v6s.append(this->staticIps[i].toString(buf)); } } if (v6s.length() > 0) { @@ -94,8 +96,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (ets.length() > 0) ets.push_back(','); char tmp2[16]; - Utils::ztsnprintf(tmp2,sizeof(tmp2),"%x",et); - ets.append(tmp2); + ets.append(Utils::hex((uint16_t)et,tmp2)); } et = 0; } @@ -114,7 +115,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if ((this->specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0) { if (ab.length() > 0) ab.push_back(','); - ab.append(Address(this->specialists[i]).toString().c_str()); + char tmp2[16]; + ab.append(Address(this->specialists[i]).toString(tmp2)); } } if (ab.length() > 0) { diff --git a/node/Node.cpp b/node/Node.cpp index 4b598f61..e28accee 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -82,8 +82,8 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 if (n > 0) { tmp[n] = (char)0; if (RR->identity.fromString(tmp)) { - RR->publicIdentityStr = RR->identity.toString(false); - RR->secretIdentityStr = RR->identity.toString(true); + RR->identity.toString(false,RR->publicIdentityStr); + RR->identity.toString(true,RR->secretIdentityStr); } else { n = -1; } @@ -92,10 +92,10 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; if (n <= 0) { RR->identity.generate(); - RR->publicIdentityStr = RR->identity.toString(false); - RR->secretIdentityStr = RR->identity.toString(true); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,RR->secretIdentityStr.data(),(unsigned int)RR->secretIdentityStr.length()); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr.data(),(unsigned int)RR->publicIdentityStr.length()); + RR->identity.toString(false,RR->publicIdentityStr); + RR->identity.toString(true,RR->secretIdentityStr); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,RR->secretIdentityStr,(unsigned int)strlen(RR->secretIdentityStr)); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } else { n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { @@ -104,7 +104,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 n = -1; } if (n <= 0) - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr.data(),(unsigned int)RR->publicIdentityStr.length()); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } try { @@ -386,8 +386,8 @@ uint64_t Node::address() const void Node::status(ZT_NodeStatus *status) const { status->address = RR->identity.address().toInt(); - status->publicIdentity = RR->publicIdentityStr.c_str(); - status->secretIdentity = RR->secretIdentityStr.c_str(); + status->publicIdentity = RR->publicIdentityStr; + status->secretIdentity = RR->secretIdentityStr; status->online = _online ? 1 : 0; } @@ -544,39 +544,6 @@ bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,cons return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),localSocket,reinterpret_cast(&remoteAddress)) != 0) : true); } -#ifdef ZT_TRACE -void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) -{ - static Mutex traceLock; - - va_list ap; - char tmp1[1024],tmp2[1024],tmp3[256]; - - Mutex::Lock _l(traceLock); - - time_t now = (time_t)(_now / 1000ULL); -#ifdef __WINDOWS__ - ctime_s(tmp3,sizeof(tmp3),&now); - char *nowstr = tmp3; -#else - char *nowstr = ctime_r(&now,tmp3); -#endif - unsigned long nowstrlen = (unsigned long)strlen(nowstr); - if (nowstr[nowstrlen-1] == '\n') - nowstr[--nowstrlen] = (char)0; - if (nowstr[nowstrlen-1] == '\r') - nowstr[--nowstrlen] = (char)0; - - va_start(ap,fmt); - vsnprintf(tmp2,sizeof(tmp2),fmt,ap); - va_end(ap); - tmp2[sizeof(tmp2)-1] = (char)0; - - Utils::ztsnprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2); - postEvent((void *)0,ZT_EVENT_TRACE,tmp1); -} -#endif // ZT_TRACE - uint64_t Node::prng() { // https://en.wikipedia.org/wiki/Xorshift#xorshift.2B diff --git a/node/Node.hpp b/node/Node.hpp index 55491b06..40903f7c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -195,10 +195,6 @@ public: inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,(int)len); } inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2]) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,(const void *)0,-1); } -#ifdef ZT_TRACE - void postTrace(const char *module,unsigned int line,const char *fmt,...); -#endif - bool shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const int64_t localSocket,const InetAddress &remoteAddress); inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index 99afe25d..94b96d34 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -27,7 +27,7 @@ #ifndef ZT_RUNTIMEENVIRONMENT_HPP #define ZT_RUNTIMEENVIRONMENT_HPP -#include +#include #include "Constants.hpp" #include "Utils.hpp" @@ -60,11 +60,13 @@ public: ,sa((SelfAwareness *)0) { Utils::getSecureRandom(&instanceId,sizeof(instanceId)); + memset(publicIdentityStr,0,sizeof(publicIdentityStr)); + memset(secretIdentityStr,0,sizeof(secretIdentityStr)); } ~RuntimeEnvironment() { - Utils::burn(reinterpret_cast(const_cast(secretIdentityStr.data())),(unsigned int)secretIdentityStr.length()); + Utils::burn(secretIdentityStr,sizeof(secretIdentityStr)); } /** @@ -77,8 +79,8 @@ public: // This node's identity Identity identity; - std::string publicIdentityStr; - std::string secretIdentityStr; + char publicIdentityStr[ZT_IDENTITY_STRING_BUFFER_LENGTH]; + char secretIdentityStr[ZT_IDENTITY_STRING_BUFFER_LENGTH]; // This is set externally to an instance of this base class NetworkController *localNetworkController; diff --git a/node/Topology.cpp b/node/Topology.cpp index 809bc7e7..e7bbdfae 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -91,12 +91,8 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) { #ifdef ZT_TRACE - if ((!peer)||(peer->address() == RR->identity.address())) { - if (!peer) - fprintf(stderr,"FATAL BUG: addPeer() caught attempt to add NULL peer" ZT_EOL_S); - else fprintf(stderr,"FATAL BUG: addPeer() caught attempt to add peer for self" ZT_EOL_S); - abort(); - } + if ((!peer)||(peer->address() == RR->identity.address())) + return SharedPtr(); #endif SharedPtr np; diff --git a/node/Utils.cpp b/node/Utils.cpp index d2321e16..a3a4c3c3 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -55,90 +55,35 @@ namespace ZeroTier { const char Utils::HEXCHARS[16] = { '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' }; -// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers. -static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len) -{ - volatile uint8_t *const end = ptr + len; - while (ptr != end) *(ptr++) = (uint8_t)0; -} -static void (*volatile _Utils_doBurn_ptr)(volatile uint8_t *,unsigned int) = _Utils_doBurn; -void Utils::burn(void *ptr,unsigned int len) { (_Utils_doBurn_ptr)((volatile uint8_t *)ptr,len); } - -std::string Utils::hex(const void *data,unsigned int len) +static unsigned long _Utils_itoa(unsigned long n,char *s) { - std::string r; - r.reserve(len * 2); - for(unsigned int i=0;i> 4]); - r.push_back(HEXCHARS[((const unsigned char *)data)[i] & 0x0f]); - } - return r; + if (n == 0) + return 0; + unsigned long pos = _Utils_itoa(n / 10,s); + if (pos >= 22) // sanity check, should be impossible + pos = 22; + s[pos] = '0' + (char)(n % 10); + return pos + 1; } - -std::string Utils::unhex(const char *hex,unsigned int maxlen) +char *Utils::decimal(unsigned long n,char s[24]) { - int n = 1; - unsigned char c,b = 0; - const char *eof = hex + maxlen; - std::string r; - - if (!maxlen) - return r; - - while ((c = (unsigned char)*(hex++))) { - if ((c >= 48)&&(c <= 57)) { // 0..9 - if ((n ^= 1)) - r.push_back((char)(b | (c - 48))); - else b = (c - 48) << 4; - } else if ((c >= 65)&&(c <= 70)) { // A..F - if ((n ^= 1)) - r.push_back((char)(b | (c - (65 - 10)))); - else b = (c - (65 - 10)) << 4; - } else if ((c >= 97)&&(c <= 102)) { // a..f - if ((n ^= 1)) - r.push_back((char)(b | (c - (97 - 10)))); - else b = (c - (97 - 10)) << 4; - } - if (hex == eof) - break; + if (n == 0) { + s[0] = '0'; + s[1] = (char)0; + return s; } - - return r; + s[_Utils_itoa(n,s)] = (char)0; + return s; } -unsigned int Utils::unhex(const char *hex,unsigned int maxlen,void *buf,unsigned int len) +// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers. +static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len) { - int n = 1; - unsigned char c,b = 0; - unsigned int l = 0; - const char *eof = hex + maxlen; - - if (!maxlen) - return 0; - - while ((c = (unsigned char)*(hex++))) { - if ((c >= 48)&&(c <= 57)) { // 0..9 - if ((n ^= 1)) { - if (l >= len) break; - ((unsigned char *)buf)[l++] = (b | (c - 48)); - } else b = (c - 48) << 4; - } else if ((c >= 65)&&(c <= 70)) { // A..F - if ((n ^= 1)) { - if (l >= len) break; - ((unsigned char *)buf)[l++] = (b | (c - (65 - 10))); - } else b = (c - (65 - 10)) << 4; - } else if ((c >= 97)&&(c <= 102)) { // a..f - if ((n ^= 1)) { - if (l >= len) break; - ((unsigned char *)buf)[l++] = (b | (c - (97 - 10))); - } else b = (c - (97 - 10)) << 4; - } - if (hex == eof) - break; - } - - return l; + volatile uint8_t *const end = ptr + len; + while (ptr != end) *(ptr++) = (uint8_t)0; } +static void (*volatile _Utils_doBurn_ptr)(volatile uint8_t *,unsigned int) = _Utils_doBurn; +void Utils::burn(void *ptr,unsigned int len) { (_Utils_doBurn_ptr)((volatile uint8_t *)ptr,len); } void Utils::getSecureRandom(void *buf,unsigned int bytes) { @@ -244,21 +189,4 @@ bool Utils::scopy(char *dest,unsigned int len,const char *src) return true; } -unsigned int Utils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...) -{ - va_list ap; - - va_start(ap,fmt); - int n = (int)vsnprintf(buf,len,fmt,ap); - va_end(ap); - - if ((n >= (int)len)||(n < 0)) { - if (len) - buf[len - 1] = (char)0; - throw std::length_error("buf[] overflow"); - } - - return (unsigned int)n; -} - } // namespace ZeroTier diff --git a/node/Utils.hpp b/node/Utils.hpp index 212ef247..5a5e9f39 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -70,42 +70,144 @@ public: static void burn(void *ptr,unsigned int len); /** - * Convert binary data to hexadecimal - * - * @param data Data to convert to hex - * @param len Length of data - * @return Hexadecimal string + * @param n Number to convert + * @param s Buffer, at least 24 bytes in size + * @return String containing 'n' in base 10 form */ - static std::string hex(const void *data,unsigned int len); - static inline std::string hex(const std::string &data) { return hex(data.data(),(unsigned int)data.length()); } + static char *decimal(unsigned long n,char s[24]); - /** - * Convert hexadecimal to binary data - * - * This ignores all non-hex characters, just stepping over them and - * continuing. Upper and lower case are supported for letters a-f. - * - * @param hex Hexadecimal ASCII code (non-hex chars are ignored, stops at zero or maxlen) - * @param maxlen Maximum length of hex string buffer - * @return Binary data - */ - static std::string unhex(const char *hex,unsigned int maxlen); - static inline std::string unhex(const std::string &hex) { return unhex(hex.c_str(),(unsigned int)hex.length()); } + static inline char *hex(uint64_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 60) & 0xf]; + s[1] = HEXCHARS[(i >> 56) & 0xf]; + s[2] = HEXCHARS[(i >> 52) & 0xf]; + s[3] = HEXCHARS[(i >> 48) & 0xf]; + s[4] = HEXCHARS[(i >> 44) & 0xf]; + s[5] = HEXCHARS[(i >> 40) & 0xf]; + s[6] = HEXCHARS[(i >> 36) & 0xf]; + s[7] = HEXCHARS[(i >> 32) & 0xf]; + s[8] = HEXCHARS[(i >> 28) & 0xf]; + s[9] = HEXCHARS[(i >> 24) & 0xf]; + s[10] = HEXCHARS[(i >> 20) & 0xf]; + s[11] = HEXCHARS[(i >> 16) & 0xf]; + s[12] = HEXCHARS[(i >> 12) & 0xf]; + s[13] = HEXCHARS[(i >> 8) & 0xf]; + s[14] = HEXCHARS[(i >> 4) & 0xf]; + s[15] = HEXCHARS[i & 0xf]; + s[16] = (char)0; + return s; + } - /** - * Convert hexadecimal to binary data - * - * This ignores all non-hex characters, just stepping over them and - * continuing. Upper and lower case are supported for letters a-f. - * - * @param hex Hexadecimal ASCII - * @param maxlen Maximum length of hex string buffer - * @param buf Buffer to fill - * @param len Length of buffer - * @return Number of characters actually written - */ - static unsigned int unhex(const char *hex,unsigned int maxlen,void *buf,unsigned int len); - static inline unsigned int unhex(const std::string &hex,void *buf,unsigned int len) { return unhex(hex.c_str(),(unsigned int)hex.length(),buf,len); } + static inline char *hex10(uint64_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 36) & 0xf]; + s[1] = HEXCHARS[(i >> 32) & 0xf]; + s[2] = HEXCHARS[(i >> 28) & 0xf]; + s[3] = HEXCHARS[(i >> 24) & 0xf]; + s[4] = HEXCHARS[(i >> 20) & 0xf]; + s[5] = HEXCHARS[(i >> 16) & 0xf]; + s[6] = HEXCHARS[(i >> 12) & 0xf]; + s[7] = HEXCHARS[(i >> 8) & 0xf]; + s[8] = HEXCHARS[(i >> 4) & 0xf]; + s[9] = HEXCHARS[i & 0xf]; + s[10] = (char)0; + return s; + } + + static inline char *hex(uint16_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 12) & 0xf]; + s[1] = HEXCHARS[(i >> 8) & 0xf]; + s[2] = HEXCHARS[(i >> 4) & 0xf]; + s[3] = HEXCHARS[i & 0xf]; + s[4] = (char)0; + return s; + } + + static inline char *hex(uint8_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 4) & 0xf]; + s[1] = HEXCHARS[i & 0xf]; + s[2] = (char)0; + return s; + } + + static inline char *hex(const void *d,unsigned int l,char *s) + { + char *save = s; + for(unsigned int i=0;i(d)[i]; + *(s++) = HEXCHARS[(b >> 4) & 0xf]; + *(s++) = HEXCHARS[b & 0xf]; + } + *s = (char)0; + return save; + } + + static inline unsigned int unhex(const char *h,void *buf,unsigned int buflen) + { + unsigned int l = 0; + while (l < buflen) { + uint8_t hc = (uint8_t)*(h++); + if (!hc) break; + + uint8_t c = 0; + if ((hc >= 48)&&(hc <= 57)) + c = hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c = hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c = hc - 55; + + hc = (uint8_t)*(h++); + if (!hc) break; + + c <<= 4; + if ((hc >= 48)&&(hc <= 57)) + c |= hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c |= hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c |= hc - 55; + + reinterpret_cast(buf)[l++] = c; + } + return l; + } + + static inline unsigned int unhex(const char *h,unsigned int hlen,void *buf,unsigned int buflen) + { + unsigned int l = 0; + const char *hend = h + hlen; + while (l < buflen) { + if (h == hend) break; + uint8_t hc = (uint8_t)*(h++); + if (!hc) break; + + uint8_t c = 0; + if ((hc >= 48)&&(hc <= 57)) + c = hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c = hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c = hc - 55; + + if (h == hend) break; + hc = (uint8_t)*(h++); + if (!hc) break; + + c <<= 4; + if ((hc >= 48)&&(hc <= 57)) + c |= hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c |= hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c |= hc - 55; + + reinterpret_cast(buf)[l++] = c; + } + return l; + } /** * Generate secure random bytes @@ -232,20 +334,6 @@ public: */ static bool scopy(char *dest,unsigned int len,const char *src); - /** - * Variant of snprintf that is portable and throws an exception - * - * This just wraps the local implementation whatever it's called, while - * performing a few other checks and adding exceptions for overflow. - * - * @param buf Buffer to write to - * @param len Length of buffer in bytes - * @param fmt Format string - * @param ... Format arguments - * @throws std::length_error buf[] too short (buf[] will still be left null-terminated) - */ - static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...); - /** * Count the number of bits set in an integer * diff --git a/one.cpp b/one.cpp index cbf09121..b1a19e8c 100644 --- a/one.cpp +++ b/one.cpp @@ -260,9 +260,9 @@ static int cli(int argc,char **argv) if (hd) { char p[4096]; #ifdef __APPLE__ - Utils::ztsnprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); + OSUtils::ztsnprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); #else - Utils::ztsnprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); + OSUtils::ztsnprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); #endif OSUtils::readFile(p,authToken); } @@ -278,7 +278,7 @@ static int cli(int argc,char **argv) InetAddress addr; { char addrtmp[256]; - Utils::ztsnprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); + OSUtils::ztsnprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); addr = InetAddress(addrtmp); } @@ -366,7 +366,7 @@ static int cli(int argc,char **argv) std::string addr = path["address"]; const uint64_t now = OSUtils::now(); const double lq = (path.count("linkQuality")) ? (double)path["linkQuality"] : -1.0; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); bestPath = tmp; break; } @@ -378,7 +378,7 @@ static int cli(int argc,char **argv) int64_t vmin = p["versionMinor"]; int64_t vrev = p["versionRev"]; if (vmaj >= 0) { - Utils::ztsnprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + OSUtils::ztsnprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); } else { ver[0] = '-'; ver[1] = (char)0; @@ -527,9 +527,9 @@ static int cli(int argc,char **argv) const uint64_t seed = Utils::hexStrToU64(arg2.c_str()); if ((worldId)&&(seed)) { char jsons[1024]; - Utils::ztsnprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); + OSUtils::ztsnprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); char cl[128]; - Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + OSUtils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -579,11 +579,11 @@ static int cli(int argc,char **argv) if (eqidx != std::string::npos) { if ((arg2.substr(0,eqidx) == "allowManaged")||(arg2.substr(0,eqidx) == "allowGlobal")||(arg2.substr(0,eqidx) == "allowDefault")) { char jsons[1024]; - Utils::ztsnprintf(jsons,sizeof(jsons),"{\"%s\":%s}", + OSUtils::ztsnprintf(jsons,sizeof(jsons),"{\"%s\":%s}", arg2.substr(0,eqidx).c_str(), (((arg2.substr(eqidx,2) == "=t")||(arg2.substr(eqidx,2) == "=1")) ? "true" : "false")); char cl[128]; - Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + OSUtils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -648,7 +648,7 @@ static Identity getIdFromArg(char *arg) } else { // identity is to be read from a file std::string idser; if (OSUtils::readFile(arg,idser)) { - if (id.fromString(idser)) + if (id.fromString(idser.c_str())) return id; } } @@ -689,14 +689,15 @@ static int idtool(int argc,char **argv) } } - std::string idser = id.toString(true); + char idtmp[1024]; + std::string idser = id.toString(true,idtmp); if (argc >= 3) { if (!OSUtils::writeFile(argv[2],idser)) { fprintf(stderr,"Error writing to %s" ZT_EOL_S,argv[2]); return 1; } else printf("%s written" ZT_EOL_S,argv[2]); if (argc >= 4) { - idser = id.toString(false); + idser = id.toString(false,idtmp); if (!OSUtils::writeFile(argv[3],idser)) { fprintf(stderr,"Error writing to %s" ZT_EOL_S,argv[3]); return 1; @@ -731,7 +732,8 @@ static int idtool(int argc,char **argv) return 1; } - printf("%s",id.toString(false).c_str()); + char idtmp[1024]; + printf("%s",id.toString(false,idtmp)); } else if (!strcmp(argv[1],"sign")) { if (argc < 4) { idtoolPrintHelp(stdout,argv[0]); @@ -755,7 +757,8 @@ static int idtool(int argc,char **argv) return 1; } C25519::Signature signature = id.sign(inf.data(),(unsigned int)inf.length()); - printf("%s",Utils::hex(signature.data,(unsigned int)signature.size()).c_str()); + char hexbuf[1024]; + printf("%s",Utils::hex(signature.data,(unsigned int)signature.size(),hexbuf)); } else if (!strcmp(argv[1],"verify")) { if (argc < 4) { idtoolPrintHelp(stdout,argv[0]); @@ -774,7 +777,8 @@ static int idtool(int argc,char **argv) return 1; } - std::string signature(Utils::unhex(argv[4])); + char buf[4096]; + std::string signature(buf,Utils::unhex(argv[4],buf,(unsigned int)sizeof(buf))); if ((signature.length() > ZT_ADDRESS_LENGTH)&&(id.verify(inf.data(),(unsigned int)inf.length(),signature.data(),(unsigned int)signature.length()))) { printf("%s signature valid" ZT_EOL_S,argv[3]); } else { @@ -793,14 +797,15 @@ static int idtool(int argc,char **argv) C25519::Pair kp(C25519::generate()); + char idtmp[4096]; nlohmann::json mj; mj["objtype"] = "world"; mj["worldType"] = "moon"; - mj["updatesMustBeSignedBy"] = mj["signingKey"] = Utils::hex(kp.pub.data,(unsigned int)kp.pub.size()); - mj["signingKey_SECRET"] = Utils::hex(kp.priv.data,(unsigned int)kp.priv.size()); - mj["id"] = id.address().toString(); + mj["updatesMustBeSignedBy"] = mj["signingKey"] = Utils::hex(kp.pub.data,(unsigned int)kp.pub.size(),idtmp); + mj["signingKey_SECRET"] = Utils::hex(kp.priv.data,(unsigned int)kp.priv.size(),idtmp); + mj["id"] = id.address().toString(idtmp); nlohmann::json seedj; - seedj["identity"] = id.toString(false); + seedj["identity"] = id.toString(false,idtmp); seedj["stableEndpoints"] = nlohmann::json::array(); (mj["roots"] = nlohmann::json::array()).push_back(seedj); std::string mjd(OSUtils::jsonDump(mj)); @@ -836,9 +841,9 @@ static int idtool(int argc,char **argv) C25519::Pair signingKey; C25519::Public updatesMustBeSignedBy; - Utils::unhex(OSUtils::jsonString(mj["signingKey"],""),signingKey.pub.data,(unsigned int)signingKey.pub.size()); - Utils::unhex(OSUtils::jsonString(mj["signingKey_SECRET"],""),signingKey.priv.data,(unsigned int)signingKey.priv.size()); - Utils::unhex(OSUtils::jsonString(mj["updatesMustBeSignedBy"],""),updatesMustBeSignedBy.data,(unsigned int)updatesMustBeSignedBy.size()); + Utils::unhex(OSUtils::jsonString(mj["signingKey"],"").c_str(),signingKey.pub.data,(unsigned int)signingKey.pub.size()); + Utils::unhex(OSUtils::jsonString(mj["signingKey_SECRET"],"").c_str(),signingKey.priv.data,(unsigned int)signingKey.priv.size()); + Utils::unhex(OSUtils::jsonString(mj["updatesMustBeSignedBy"],"").c_str(),updatesMustBeSignedBy.data,(unsigned int)updatesMustBeSignedBy.size()); std::vector roots; nlohmann::json &rootsj = mj["roots"]; @@ -847,11 +852,11 @@ static int idtool(int argc,char **argv) nlohmann::json &r = rootsj[i]; if (r.is_object()) { roots.push_back(World::Root()); - roots.back().identity = Identity(OSUtils::jsonString(r["identity"],"")); + roots.back().identity = Identity(OSUtils::jsonString(r["identity"],"").c_str()); nlohmann::json &stableEndpointsj = r["stableEndpoints"]; if (stableEndpointsj.is_array()) { for(unsigned long k=0;k<(unsigned long)stableEndpointsj.size();++k) - roots.back().stableEndpoints.push_back(InetAddress(OSUtils::jsonString(stableEndpointsj[k],""))); + roots.back().stableEndpoints.push_back(InetAddress(OSUtils::jsonString(stableEndpointsj[k],"").c_str())); std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end()); } } @@ -864,7 +869,7 @@ static int idtool(int argc,char **argv) Buffer wbuf; w.serialize(wbuf); char fn[128]; - Utils::ztsnprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); + OSUtils::ztsnprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); OSUtils::writeFile(fn,wbuf.data(),wbuf.size()); printf("wrote %s (signed world with timestamp %llu)" ZT_EOL_S,fn,(unsigned long long)now); } diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index f07f9e5a..8e57d605 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -114,8 +114,8 @@ BSDEthernetTap::BSDEthernetTap( std::vector devFiles(OSUtils::listDirectory("/dev")); for(int i=9993;i<(9993+128);++i) { - Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + OSUtils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); if (std::find(devFiles.begin(),devFiles.end(),std::string(tmpdevname)) == devFiles.end()) { long cpid = (long)vfork(); if (cpid == 0) { @@ -152,8 +152,8 @@ BSDEthernetTap::BSDEthernetTap( /* Other BSDs like OpenBSD only have a limited number of tap devices that cannot be renamed */ for(int i=0;i<64;++i) { - Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + OSUtils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { _dev = tmpdevname; @@ -171,9 +171,9 @@ BSDEthernetTap::BSDEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); + OSUtils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + OSUtils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + OSUtils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -256,7 +256,8 @@ bool BSDEthernetTap::addIp(const InetAddress &ip) long cpid = (long)vfork(); if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),ip.isV4() ? "inet" : "inet6",ip.toString().c_str(),"alias",(const char *)0); + char tmp[128]; + ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),ip.isV4() ? "inet" : "inet6",ip.toString(tmp),"alias",(const char *)0); ::_exit(-1); } else if (cpid > 0) { int exitcode = -1; @@ -385,7 +386,7 @@ void BSDEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/Http.cpp b/osdep/Http.cpp index 3c556f44..d6d0238c 100644 --- a/osdep/Http.cpp +++ b/osdep/Http.cpp @@ -244,10 +244,10 @@ unsigned int Http::_do( try { char tmp[1024]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); handler.writeBuf.append(tmp); for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) { - Utils::ztsnprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); handler.writeBuf.append(tmp); } handler.writeBuf.append("\r\n"); diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index fc5199f1..c8f9ef9d 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -97,7 +97,7 @@ LinuxEthernetTap::LinuxEthernetTap( char procpath[128],nwids[32]; struct stat sbuf; - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally @@ -134,7 +134,7 @@ LinuxEthernetTap::LinuxEthernetTap( std::map::const_iterator gdmEntry = globalDeviceMap.find(nwids); if (gdmEntry != globalDeviceMap.end()) { Utils::scopy(ifr.ifr_name,sizeof(ifr.ifr_name),gdmEntry->second.c_str()); - Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + OSUtils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); recalledDevice = (stat(procpath,&sbuf) != 0); } @@ -142,8 +142,8 @@ LinuxEthernetTap::LinuxEthernetTap( #ifdef __SYNOLOGY__ int devno = 50; do { - Utils::ztsnprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); - Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + OSUtils::ztsnprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); + OSUtils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); // try zt#++ until we find one that does not exist #else char devno = 0; @@ -158,7 +158,7 @@ LinuxEthernetTap::LinuxEthernetTap( _base32_5_to_8(reinterpret_cast(tmp2) + 5,tmp3 + 10); tmp3[15] = (char)0; memcpy(ifr.ifr_name,tmp3,16); - Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + OSUtils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); #endif } @@ -264,7 +264,8 @@ static bool ___removeIp(const std::string &_dev,const InetAddress &ip) if (cpid == 0) { OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1); - ::execlp("ip","ip","addr","del",ip.toString().c_str(),"dev",_dev.c_str(),(const char *)0); + char iptmp[128]; + ::execlp("ip","ip","addr","del",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0); ::_exit(-1); } else { int exitcode = -1; @@ -296,25 +297,28 @@ bool LinuxEthernetTap::addIpSyn(std::vector ips) // Assemble and write contents of ifcfg-dev file for(int i=0; i<(int)ips.size(); i++) { if (ips[i].isV4()) { + char iptmp[64],iptmp2[64]; std::string numstr4 = ip4_tot > 1 ? std::to_string(ip4) : ""; - cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString() - + "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString()+"\n"; + cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString(iptmp) + + "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; ip4++; } else { + char iptmp[64],iptmp2[64]; std::string numstr6 = ip6_tot > 1 ? std::to_string(ip6) : ""; - cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString() - + "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString()+"\n"; + cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString(iptmp) + + "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; ip6++; } } OSUtils::writeFile(filepath.c_str(), cfg_contents.c_str(), cfg_contents.length()); // Finaly, add IPs for(int i=0; i<(int)ips.size(); i++){ + char iptmp[128],iptmp2[128[; if (ips[i].isV4()) - ::execlp("ip","ip","addr","add",ips[i].toString().c_str(),"broadcast",ips[i].broadcast().toIpString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"broadcast",ips[i].broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0); else - ::execlp("ip","ip","addr","add",ips[i].toString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"dev",_dev.c_str(),(const char *)0); } ::_exit(-1); } else if (cpid > 0) { @@ -345,10 +349,11 @@ bool LinuxEthernetTap::addIp(const InetAddress &ip) if (cpid == 0) { OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1); + char iptmp[128],iptmp2[128]; if (ip.isV4()) { - ::execlp("ip","ip","addr","add",ip.toString().c_str(),"broadcast",ip.broadcast().toIpString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ip.toString(iptmp),"broadcast",ip.broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0); } else { - ::execlp("ip","ip","addr","add",ip.toString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0); } ::_exit(-1); } else if (cpid > 0) { diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index fca1c290..3a0b8a7e 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -246,7 +246,6 @@ static std::vector<_RTE> _getRTEs(const InetAddress &target,bool contains) static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *ifscope,const char *localInterface) { - //printf("route %s %s %s %s %s\n",op,target.toString().c_str(),(via) ? via.toString().c_str() : "(null)",(ifscope) ? ifscope : "(null)",(localInterface) ? localInterface : "(null)"); long p = (long)fork(); if (p > 0) { int exitcode = -1; @@ -254,17 +253,19 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress } else if (p == 0) { ::close(STDOUT_FILENO); ::close(STDERR_FILENO); + char ttmp[64]; + char iptmp[64]; if (via) { if ((ifscope)&&(ifscope[0])) { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),via.toIpString().c_str(),(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),via.toIpString(iptmp),(const char *)0); } else { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),via.toIpString().c_str(),(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),via.toIpString(iptmp),(const char *)0); } } else if ((localInterface)&&(localInterface[0])) { if ((ifscope)&&(ifscope[0])) { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),"-interface",localInterface,(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),"-interface",localInterface,(const char *)0); } else { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),"-interface",localInterface,(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),"-interface",localInterface,(const char *)0); } } ::_exit(-1); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 06508e77..882b8255 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -57,6 +57,23 @@ namespace ZeroTier { +unsigned int OSUtils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...) +{ + va_list ap; + + va_start(ap,fmt); + int n = (int)vsnprintf(buf,len,fmt,ap); + va_end(ap); + + if ((n >= (int)len)||(n < 0)) { + if (len) + buf[len - 1] = (char)0; + throw std::length_error("buf[] overflow"); + } + + return (unsigned int)n; +} + #ifdef __UNIX_LIKE__ bool OSUtils::redirectUnixOutputs(const char *stdoutPath,const char *stderrPath) throw() @@ -134,7 +151,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) if (date.QuadPart > 0) { date.QuadPart -= adjust.QuadPart; if ((uint64_t)((date.QuadPart / 10000000) * 1000) < olderThan) { - Utils::ztsnprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); + ztsnprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); if (DeleteFileA(tmp)) ++cleaned; } @@ -157,7 +174,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) break; if (dptr) { if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_REG)) { - Utils::ztsnprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); + ztsnprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); if (stat(tmp,&st) == 0) { uint64_t mt = (uint64_t)(st.st_mtime); if ((mt > 0)&&((mt * 1000) < olderThan)) { @@ -464,7 +481,7 @@ std::string OSUtils::jsonString(const nlohmann::json &jv,const char *dfl) return jv; } else if (jv.is_number()) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); + ztsnprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); return tmp; } else if (jv.is_boolean()) { return ((bool)jv ? std::string("1") : std::string("0")); @@ -477,9 +494,10 @@ std::string OSUtils::jsonBinFromHex(const nlohmann::json &jv) { std::string s(jsonString(jv,"")); if (s.length() > 0) { - char *buf = new char[(s.length() / 2) + 1]; + unsigned int buflen = (s.length() / 2) + 1; + char *buf = new char[buflen]; try { - unsigned int l = Utils::unhex(s,buf,(unsigned int)s.length()); + unsigned int l = Utils::unhex(s.c_str(),buf,buflen); std::string b(buf,l); delete [] buf; return b; diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index dff7df86..d6f32822 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -33,7 +33,6 @@ #include #include -#include #include #include #include @@ -66,6 +65,20 @@ namespace ZeroTier { class OSUtils { public: + /** + * Variant of snprintf that is portable and throws an exception + * + * This just wraps the local implementation whatever it's called, while + * performing a few other checks and adding exceptions for overflow. + * + * @param buf Buffer to write to + * @param len Length of buffer in bytes + * @param fmt Format string + * @param ... Format arguments + * @throws std::length_error buf[] too short (buf[] will still be left null-terminated) + */ + static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...); + #ifdef __UNIX_LIKE__ /** * Close STDOUT_FILENO and STDERR_FILENO and replace them with output to given path diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp index e082408e..b43d34c0 100644 --- a/osdep/OSXEthernetTap.cpp +++ b/osdep/OSXEthernetTap.cpp @@ -336,7 +336,7 @@ OSXEthernetTap::OSXEthernetTap( char devpath[64],ethaddr[64],mtustr[32],metstr[32],nwids[32]; struct stat stattmp; - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _gl(globalTapCreateLock); @@ -391,13 +391,13 @@ OSXEthernetTap::OSXEthernetTap( // Open the first unused tap device if we didn't recall a previous one. if (!recalledDevice) { for(int i=0;i<64;++i) { - Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i); + OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i); if (stat(devpath,&stattmp)) throw std::runtime_error("no more TAP devices available"); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { char foo[16]; - Utils::ztsnprintf(foo,sizeof(foo),"zt%d",i); + OSUtils::ztsnprintf(foo,sizeof(foo),"zt%d",i); _dev = foo; break; } @@ -413,9 +413,9 @@ OSXEthernetTap::OSXEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); + OSUtils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + OSUtils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + OSUtils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -499,7 +499,8 @@ bool OSXEthernetTap::addIp(const InetAddress &ip) long cpid = (long)vfork(); if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toString().c_str(),"alias",(const char *)0); + char tmp[128]; + ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toString(tmp),"alias",(const char *)0); ::_exit(-1); } else if (cpid > 0) { int exitcode = -1; @@ -519,7 +520,8 @@ bool OSXEthernetTap::removeIp(const InetAddress &ip) if (*i == ip) { long cpid = (long)vfork(); if (cpid == 0) { - execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toIpString().c_str(),"-alias",(const char *)0); + char tmp[128]; + execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toIpString(tmp),"-alias",(const char *)0); _exit(-1); } else if (cpid > 0) { int exitcode = -1; @@ -636,7 +638,7 @@ void OSXEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index df868e7a..b1990486 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -205,7 +205,7 @@ public: memset(externalip,0,sizeof(externalip)); memset(&urls,0,sizeof(urls)); memset(&data,0,sizeof(data)); - Utils::ztsnprintf(inport,sizeof(inport),"%d",localPort); + OSUtils::ztsnprintf(inport,sizeof(inport),"%d",localPort); if ((UPNP_GetValidIGD(devlist,&urls,&data,lanaddr,sizeof(lanaddr)))&&(lanaddr[0])) { #ifdef ZT_PORTMAPPER_TRACE @@ -220,7 +220,7 @@ public: int tryPort = (int)localPort + tries; if (tryPort >= 65535) tryPort = (tryPort - 65535) + 1025; - Utils::ztsnprintf(outport,sizeof(outport),"%u",tryPort); + OSUtils::ztsnprintf(outport,sizeof(outport),"%u",tryPort); // First check and see if this port is already mapped to the // same unique name. If so, keep this mapping and don't try diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index b96ad791..5344268f 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -484,7 +484,7 @@ WindowsEthernetTap::WindowsEthernetTap( char tag[24]; // We "tag" registry entries with the network ID to identify persistent devices - Utils::ztsnprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); + OSUtils::ztsnprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); Mutex::Lock _l(_systemTapInitLock); @@ -601,10 +601,10 @@ WindowsEthernetTap::WindowsEthernetTap( if (_netCfgInstanceId.length() > 0) { char tmps[64]; - unsigned int tmpsl = Utils::ztsnprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; + unsigned int tmpsl = OSUtils::ztsnprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"NetworkAddress",REG_SZ,tmps,tmpsl); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MAC",REG_SZ,tmps,tmpsl); - tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); + tmpsl = OSUtils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MTU",REG_SZ,tmps,tmpsl); DWORD tmp = 0; @@ -879,7 +879,7 @@ void WindowsEthernetTap::setMtu(unsigned int mtu) HKEY nwAdapters; if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\Class\\{4D36E972-E325-11CE-BFC1-08002BE10318}", 0, KEY_READ | KEY_WRITE, &nwAdapters) == ERROR_SUCCESS) { char tmps[64]; - unsigned int tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); + unsigned int tmpsl = OSUtils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters, _mySubkeyName.c_str(), "MTU", REG_SZ, tmps, tmpsl); RegCloseKey(nwAdapters); } @@ -902,7 +902,7 @@ void WindowsEthernetTap::threadMain() HANDLE wait4[3]; OVERLAPPED tapOvlRead,tapOvlWrite; - Utils::ztsnprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); + OSUtils::ztsnprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); try { while (_run) { diff --git a/selftest.cpp b/selftest.cpp index ff171aa3..e6705700 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -153,10 +153,11 @@ static int testCrypto() { static unsigned char buf1[16384]; static unsigned char buf2[sizeof(buf1)],buf3[sizeof(buf1)]; + static char hexbuf[1024]; for(int i=0;i<3;++i) { Utils::getSecureRandom(buf1,64); - std::cout << "[crypto] getSecureRandom: " << Utils::hex(buf1,64) << std::endl; + std::cout << "[crypto] getSecureRandom: " << Utils::hex(buf1,64,hexbuf) << std::endl; } std::cout << "[crypto] Testing Salsa20... "; std::cout.flush(); @@ -213,7 +214,7 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16,hexbuf) << ')' << std::endl; ::free((void *)bb); } @@ -265,7 +266,7 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16,hexbuf) << ')' << std::endl; ::free((void *)bb); } @@ -427,6 +428,7 @@ static int testIdentity() { Identity id; Buffer<512> buf; + char buf2[1024]; std::cout << "[identity] Validate known-good identity... "; std::cout.flush(); if (!id.fromString(KNOWN_GOOD_IDENTITY)) { @@ -459,7 +461,7 @@ static int testIdentity() uint64_t genstart = OSUtils::now(); id.generate(); uint64_t genend = OSUtils::now(); - std::cout << "(took " << (genend - genstart) << "ms): " << id.toString(true) << std::endl; + std::cout << "(took " << (genend - genstart) << "ms): " << id.toString(true,buf2) << std::endl; std::cout << "[identity] Locally validate identity: "; if (id.locallyValidate()) { std::cout << "PASS" << std::endl; @@ -499,7 +501,7 @@ static int testIdentity() { Identity id2; - id2.fromString(id.toString(true).c_str()); + id2.fromString(id.toString(true,buf2)); std::cout << "[identity] Serialize and deserialize (ASCII w/private): "; if ((id == id2)&&(id2.locallyValidate())) { std::cout << "PASS" << std::endl; @@ -511,7 +513,7 @@ static int testIdentity() { Identity id2; - id2.fromString(id.toString(false).c_str()); + id2.fromString(id.toString(false,buf2)); std::cout << "[identity] Serialize and deserialize (ASCII no private): "; if ((id == id2)&&(id2.locallyValidate())) { std::cout << "PASS" << std::endl; @@ -526,16 +528,18 @@ static int testIdentity() static int testCertificate() { + char buf[4096]; + Identity authority; std::cout << "[certificate] Generating identity to act as authority... "; std::cout.flush(); authority.generate(); - std::cout << authority.address().toString() << std::endl; + std::cout << authority.address().toString(buf) << std::endl; Identity idA,idB; std::cout << "[certificate] Generating identities A and B... "; std::cout.flush(); idA.generate(); idB.generate(); - std::cout << idA.address().toString() << ", " << idB.address().toString() << std::endl; + std::cout << idA.address().toString(buf) << ", " << idB.address().toString(buf) << std::endl; std::cout << "[certificate] Generating certificates A and B..."; CertificateOfMembership cA(10000,100,1,idA.address()); @@ -641,6 +645,8 @@ static void _testExcept(int &depth) static int testOther() { + char buf[1024]; + std::cout << "[other] Testing C++ exceptions... "; std::cout.flush(); int depth = 0; try { @@ -657,6 +663,13 @@ static int testOther() return -1; } + std::cout << "[other] Testing InetAddress encode/decode..."; std::cout.flush(); + std::cout << " " << InetAddress("127.0.0.1/9993").toString(buf); + std::cout << " " << InetAddress("feed:dead:babe:dead:beef:f00d:1234:5678/12345").toString(buf); + std::cout << " " << InetAddress("0/9993").toString(buf); + std::cout << " " << InetAddress("").toString(buf); + std::cout << std::endl; + #if 0 std::cout << "[other] Testing Hashtable... "; std::cout.flush(); { @@ -831,7 +844,7 @@ static int testOther() memset(key, 0, sizeof(key)); memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { - Utils::ztsnprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); + OSUtils::ztsnprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); int r = rand() % 128; for(int x=0;xnwid); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",nc->nwid); nj["id"] = tmp; nj["nwid"] = tmp; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); nj["mac"] = tmp; nj["name"] = nc->name; nj["status"] = nstatus; @@ -223,16 +223,16 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc, nlohmann::json aa = nlohmann::json::array(); for(unsigned int i=0;iassignedAddressCount;++i) { - aa.push_back(reinterpret_cast(&(nc->assignedAddresses[i]))->toString()); + aa.push_back(reinterpret_cast(&(nc->assignedAddresses[i]))->toString(tmp)); } nj["assignedAddresses"] = aa; nlohmann::json ra = nlohmann::json::array(); for(unsigned int i=0;irouteCount;++i) { nlohmann::json rj; - rj["target"] = reinterpret_cast(&(nc->routes[i].target))->toString(); + rj["target"] = reinterpret_cast(&(nc->routes[i].target))->toString(tmp); if (nc->routes[i].via.ss_family == nc->routes[i].target.ss_family) - rj["via"] = reinterpret_cast(&(nc->routes[i].via))->toIpString(); + rj["via"] = reinterpret_cast(&(nc->routes[i].via))->toIpString(tmp); else rj["via"] = nlohmann::json(); rj["flags"] = (int)nc->routes[i].flags; rj["metric"] = (int)nc->routes[i].metric; @@ -252,12 +252,12 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) case ZT_PEER_ROLE_PLANET: prole = "PLANET"; break; } - Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); pj["address"] = tmp; pj["versionMajor"] = peer->versionMajor; pj["versionMinor"] = peer->versionMinor; pj["versionRev"] = peer->versionRev; - Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); pj["version"] = tmp; pj["latency"] = peer->latency; pj["role"] = prole; @@ -265,7 +265,7 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) nlohmann::json pa = nlohmann::json::array(); for(unsigned int i=0;ipathCount;++i) { nlohmann::json j; - j["address"] = reinterpret_cast(&(peer->paths[i].address))->toString(); + j["address"] = reinterpret_cast(&(peer->paths[i].address))->toString(tmp); j["lastSend"] = peer->paths[i].lastSend; j["lastReceive"] = peer->paths[i].lastReceive; j["trustedPathId"] = peer->paths[i].trustedPathId; @@ -280,19 +280,19 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) static void _moonToJson(nlohmann::json &mj,const World &world) { - char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",world.id()); + char tmp[4096]; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",world.id()); mj["id"] = tmp; mj["timestamp"] = world.timestamp(); - mj["signature"] = Utils::hex(world.signature().data,(unsigned int)world.signature().size()); - mj["updatesMustBeSignedBy"] = Utils::hex(world.updatesMustBeSignedBy().data,(unsigned int)world.updatesMustBeSignedBy().size()); + mj["signature"] = Utils::hex(world.signature().data,(unsigned int)world.signature().size(),tmp); + mj["updatesMustBeSignedBy"] = Utils::hex(world.updatesMustBeSignedBy().data,(unsigned int)world.updatesMustBeSignedBy().size(),tmp); nlohmann::json ra = nlohmann::json::array(); for(std::vector::const_iterator r(world.roots().begin());r!=world.roots().end();++r) { nlohmann::json rj; - rj["identity"] = r->identity.toString(false); + rj["identity"] = r->identity.toString(false,tmp); nlohmann::json eps = nlohmann::json::array(); for(std::vector::const_iterator a(r->stableEndpoints.begin());a!=r->stableEndpoints.end();++a) - eps.push_back(a->toString()); + eps.push_back(a->toString(tmp)); rj["stableEndpoints"] = eps; ra.push_back(rj); } @@ -613,7 +613,7 @@ public: json &physical = _localConfig["physical"]; if (physical.is_object()) { for(json::iterator phy(physical.begin());phy!=physical.end();++phy) { - InetAddress net(OSUtils::jsonString(phy.key(),"")); + InetAddress net(OSUtils::jsonString(phy.key(),"").c_str()); if (net) { if (phy.value().is_object()) { uint64_t tpid; @@ -674,7 +674,7 @@ public: // Save primary port to a file so CLIs and GUIs can learn it easily char portstr[64]; - Utils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]); + OSUtils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]); OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr)); // Attempt to bind to a secondary port chosen from our ZeroTier address. @@ -712,7 +712,7 @@ public: } if (_ports[2]) { char uniqueName[64]; - Utils::ztsnprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); + OSUtils::ztsnprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); _portMapper = new PortMapper(_ports[2],uniqueName); } } @@ -982,7 +982,7 @@ public: n->second.settings = settings; char nlcpath[4096]; - Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); + OSUtils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); FILE *out = fopen(nlcpath,"w"); if (out) { fprintf(out,"allowManaged=%d\n",(int)n->second.settings.allowManaged); @@ -1101,7 +1101,7 @@ public: ZT_NodeStatus status; _node->status(&status); - Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",status.address); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",status.address); res["address"] = tmp; res["publicIdentity"] = status.publicIdentity; res["online"] = (bool)(status.online != 0); @@ -1110,7 +1110,7 @@ public: res["versionMinor"] = ZEROTIER_ONE_VERSION_MINOR; res["versionRev"] = ZEROTIER_ONE_VERSION_REVISION; res["versionBuild"] = ZEROTIER_ONE_VERSION_BUILD; - Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); res["version"] = tmp; res["clock"] = OSUtils::now(); @@ -1257,7 +1257,7 @@ public: if ((scode != 200)&&(seed != 0)) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",id); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",id); res["id"] = tmp; res["roots"] = json::array(); res["timestamp"] = 0; @@ -1395,7 +1395,7 @@ public: json &tryAddrs = v.value()["try"]; if (tryAddrs.is_array()) { for(unsigned long i=0;i 0)) { if (phy.value().is_object()) { if (OSUtils::jsonBool(phy.value()["blacklist"],false)) { @@ -1477,7 +1477,7 @@ public: json &amf = settings["allowManagementFrom"]; if (amf.is_array()) { for(unsigned long i=0;i newManagedIps; @@ -1565,7 +1567,7 @@ public: for(std::vector::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { if (std::find(newManagedIps.begin(),newManagedIps.end(),*ip) == newManagedIps.end()) { if (!n.tap->removeIp(*ip)) - fprintf(stderr,"ERROR: unable to remove ip address %s" ZT_EOL_S, ip->toString().c_str()); + fprintf(stderr,"ERROR: unable to remove ip address %s" ZT_EOL_S, ip->toString(ipbuf)); } } #ifdef __SYNOLOGY__ @@ -1575,7 +1577,7 @@ public: for(std::vector::iterator ip(newManagedIps.begin());ip!=newManagedIps.end();++ip) { if (std::find(n.managedIps.begin(),n.managedIps.end(),*ip) == n.managedIps.end()) { if (!n.tap->addIp(*ip)) - fprintf(stderr,"ERROR: unable to add ip address %s" ZT_EOL_S, ip->toString().c_str()); + fprintf(stderr,"ERROR: unable to add ip address %s" ZT_EOL_S, ip->toString(ipbuf)); } } #endif @@ -1585,7 +1587,7 @@ public: if (syncRoutes) { char tapdev[64]; #ifdef __WINDOWS__ - Utils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); + OSUtils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); #else Utils::scopy(tapdev,sizeof(tapdev),n.tap->deviceName().c_str()); #endif @@ -1670,7 +1672,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -1851,7 +1853,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -1919,7 +1921,7 @@ public: if (!n.tap) { try { char friendlyName[128]; - Utils::ztsnprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); + OSUtils::ztsnprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); n.tap = new EthernetTap( _homePath.c_str(), @@ -1933,7 +1935,7 @@ public: *nuptr = (void *)&n; char nlcpath[256]; - Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + OSUtils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); std::string nlcbuf; if (OSUtils::readFile(nlcpath,nlcbuf)) { Dictionary<4096> nc; @@ -1954,7 +1956,7 @@ public: while (true) { size_t nextPos = addresses.find(',', pos); std::string address = addresses.substr(pos, (nextPos == std::string::npos ? addresses.size() : nextPos) - pos); - n.settings.allowManagedWhitelist.push_back(InetAddress(address)); + n.settings.allowManagedWhitelist.push_back(InetAddress(address.c_str())); if (nextPos == std::string::npos) break; pos = nextPos + 1; } @@ -2019,7 +2021,7 @@ public: #endif if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) { char nlcpath[256]; - Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + OSUtils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); OSUtils::rm(nlcpath); } } else { @@ -2068,20 +2070,20 @@ public: switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); secure = true; break; case ZT_STATE_OBJECT_PLANET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); secure = true; break; default: @@ -2121,19 +2123,19 @@ public: char p[4096]; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_PLANET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); break; default: return -1; @@ -2322,7 +2324,7 @@ public: default: scodestr = "Error"; break; } - Utils::ztsnprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + OSUtils::ztsnprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", scode, scodestr, contentType.c_str(), diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index e0519827..b4bf03ec 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -284,7 +284,7 @@ bool SoftwareUpdater::check(const uint64_t now) if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { _lastCheckTime = now; char tmp[512]; - const unsigned int len = Utils::ztsnprintf(tmp,sizeof(tmp), + const unsigned int len = OSUtils::ztsnprintf(tmp,sizeof(tmp), "%c{\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "\":%d," @@ -321,7 +321,8 @@ bool SoftwareUpdater::check(const uint64_t now) // (1) Check the hash itself to make sure the image is basically okay uint8_t sha512[ZT_SHA512_DIGEST_LEN]; SHA512::hash(sha512,_download.data(),(unsigned int)_download.length()); - if (Utils::hex(sha512,ZT_SHA512_DIGEST_LEN) == OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"")) { + char hexbuf[(ZT_SHA512_DIGEST_LEN * 2) + 2]; + if (OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"") == Utils::hex(sha512,ZT_SHA512_DIGEST_LEN,hexbuf)) { // (2) Check signature by signing authority const std::string sig(OSUtils::jsonBinFromHex(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE])); if (Identity(ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY).verify(_download.data(),(unsigned int)_download.length(),sig.data(),(unsigned int)sig.length())) { -- cgit v1.2.3 From a274e774ee9299f168f6bf387a1182ae81bff045 Mon Sep 17 00:00:00 2001 From: Monty A Date: Fri, 7 Jul 2017 10:11:21 +0100 Subject: Clarification on how to use JSON API with secret Makes it clear on how to use the authtoken.secret file when making requests. --- controller/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/README.md b/controller/README.md index db8d0153..d70ffd2a 100644 --- a/controller/README.md +++ b/controller/README.md @@ -33,7 +33,7 @@ ZeroTier network controllers can easily be run in Docker or other container syst The controller API is hosted via the same JSON API endpoint that ZeroTier One uses for local control (usually at 127.0.0.1 port 9993). All controller options are routed under the `/controller` base path. -The controller microservice does not implement any fine-grained access control (authentication is via authtoken.secret just like the regular JSON API) or other complex mangement features. It just takes network and network member configurations and reponds to controller queries. We have an enterprise product called [ZeroTier Central](https://my.zerotier.com/) that we host as a service (and that companies can license to self-host) that does this. +The controller microservice does not implement any fine-grained access control (authentication is via authtoken.secret, simply append the value from authtoken.secret file, into a new querystring parameter named "auth" - for example `/controller/network?auth=6hdmozf8k5ds39kabcdefabc`) or other complex mangement features. It just takes network and network member configurations and reponds to controller queries. We have an enterprise product called [ZeroTier Central](https://my.zerotier.com/) that we host as a service (and that companies can license to self-host) that does this. All working network IDs on a controller must begin with the controller's ZeroTier address. The API will *allow* "foreign" networks to be added but the controller will have no way of doing anything with them since nobody will know to query it. (In the future we might support secondaries, which would make this relevant.) -- cgit v1.2.3 From 4ecc0c59cafac54ff2d32e97b130f83b7481da2e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 14 Jul 2017 13:03:16 -0700 Subject: Plumbing through of remote trace into controller code. --- controller/EmbeddedNetworkController.cpp | 67 ++++++++++++++++++++++++++++++++ controller/EmbeddedNetworkController.hpp | 4 ++ include/ZeroTierOne.h | 45 ++++++++++++++++++++- node/IncomingPacket.cpp | 27 ++++++++++++- node/IncomingPacket.hpp | 1 + node/Packet.hpp | 3 -- service/OneService.cpp | 6 +++ 7 files changed, 148 insertions(+), 5 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b57a37e8..8b8a93bd 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -621,6 +621,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"],false); if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"],false); + if (b.count("remoteTraceTarget")) { + const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],"")); + if (rtt.length() == 10) { + member["remoteTraceTarget"] = rtt; + } else { + member["remoteTraceTarget"] = json(); + } + } + if (b.count("authorized")) { const bool newAuth = OSUtils::jsonBool(b["authorized"],false); if (newAuth != OSUtils::jsonBool(member["authorized"],false)) { @@ -764,6 +773,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("multicastLimit")) network["multicastLimit"] = OSUtils::jsonInt(b["multicastLimit"],32ULL); if (b.count("mtu")) network["mtu"] = std::max(std::min((unsigned int)OSUtils::jsonInt(b["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU); + if (b.count("remoteTraceTarget")) { + const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],"")); + if (rtt.length() == 10) { + network["remoteTraceTarget"] = rtt; + } else { + network["remoteTraceTarget"] = json(); + } + } + if (b.count("v4AssignMode")) { json nv4m; json &v4m = b["v4AssignMode"]; @@ -1065,6 +1083,55 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( return 404; } +void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) +{ + // Convert Dictionary into JSON object + json d; + char *saveptr = (char *)0; + for(char *l=Utils::stok(rt.data,"\n",&saveptr);(l);l=Utils::stok((char *)0,"\n",&saveptr)) { + char *eq = strchr(l,'='); + if (eq > l) { + std::string k(l,(unsigned long)(eq - l)); + std::string v; + ++eq; + while (*eq) { + if (*eq == '\\') { + ++eq; + if (*eq) { + switch(*eq) { + case 'r': + v.push_back('\r'); + break; + case 'n': + v.push_back('\n'); + break; + case '0': + v.push_back((char)0); + break; + case 'e': + v.push_back('='); + break; + default: + v.push_back(*eq); + break; + } + ++eq; + } + } else { + v.push_back(*(eq++)); + } + } + if (v.length() > 0) + d[k] = v; + } + } + + char p[128]; + OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx_%.16llx.json",rt.origin,OSUtils::now()); + _db.writeRaw(p,OSUtils::jsonDump(d)); + //fprintf(stdout,"%s\n",OSUtils::jsonDump(d).c_str()); fflush(stdout); +} + void EmbeddedNetworkController::threadMain() throw() { diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 1589ea71..03ba0b95 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -90,6 +90,8 @@ public: std::string &responseBody, std::string &responseContentType); + void handleRemoteTrace(const ZT_RemoteTrace &rt); + void threadMain() throw(); @@ -142,6 +144,7 @@ private: if (!member.count("vRev")) member["vRev"] = -1; if (!member.count("vProto")) member["vProto"] = -1; if (!member.count("physicalAddr")) member["physicalAddr"] = nlohmann::json(); + if (!member.count("remoteTraceTarget")) member["remoteTraceTarget"] = nlohmann::json(); member["objtype"] = "member"; } inline void _initNetwork(nlohmann::json &network) @@ -159,6 +162,7 @@ private: if (!network.count("routes")) network["routes"] = nlohmann::json::array(); if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); if (!network.count("mtu")) network["mtu"] = ZT_DEFAULT_MTU; + if (!network.count("remoteTraceTarget")) network["remoteTraceTarget"] = nlohmann::json(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment network["rules"] = {{ diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e4c39fbc..14ddc7fe 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -470,9 +470,52 @@ enum ZT_Event * * Meta-data: ZT_UserMessage structure */ - ZT_EVENT_USER_MESSAGE = 6 + ZT_EVENT_USER_MESSAGE = 6, + + /** + * Remote trace received + * + * These are generated when a VERB_REMOTE_TRACE is received. Note + * that any node can fling one of these at us. It is your responsibility + * to filter and determine if it's worth paying attention to. If it's + * not just drop it. Most nodes that are not active controllers ignore + * these, and controllers only save them if they pertain to networks + * with remote tracing enabled. + * + * Meta-data: ZT_RemoteTrace structure + */ + ZT_EVENT_REMOTE_TRACE = 7 }; +/** + * Payload of REMOTE_TRACE event + */ +typedef struct +{ + /** + * ZeroTier address of sender + */ + uint64_t origin; + + /** + * Null-terminated Dictionary containing key/value pairs sent by origin + * + * This *should* be a dictionary, but the implementation only checks + * that it is a valid non-empty C-style null-terminated string. Be very + * careful to use a well-tested parser to parse this as it represents + * data received from a potentially un-trusted peer on the network. + * Invalid payloads should be dropped. + * + * The contents of data[] may be modified. + */ + char *data; + + /** + * Length of dict[] in bytes, including terminating null + */ + unsigned int len; +} ZT_RemoteTrace; + /** * User message used with ZT_EVENT_USER_MESSAGE * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index a5875d1e..5e5d1d72 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1192,7 +1192,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { - if (size() >= (ZT_PACKET_IDX_PAYLOAD + 8)) { + if (likely(size() >= (ZT_PACKET_IDX_PAYLOAD + 8))) { ZT_UserMessage um; um.origin = peer->address().toInt(); um.typeId = at(ZT_PACKET_IDX_PAYLOAD); @@ -1207,6 +1207,31 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con return true; } +bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) +{ + ZT_RemoteTrace rt; + try { + const char *ptr = reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD; + const char *const eof = reinterpret_cast(data()) + size(); + rt.origin = peer->address().toInt(); + rt.data = const_cast(ptr); // start of first string + while (ptr < eof) { + if (!*ptr) { // end of string + rt.len = (unsigned int)(ptr - rt.data); + if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) + RR->node->postEvent(tPtr,ZT_EVENT_REMOTE_TRACE,&rt); + rt.data = const_cast(++ptr); // start of next string, if any + } else { + ++ptr; + } + } + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,false,0); + } catch ( ... ) { + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_REMOTE_TRACE,"unexpected exception"); + } + return true; +} + void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid) { const uint64_t now = RR->node->now(); diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 11b60712..692c63df 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -139,6 +139,7 @@ private: bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid); diff --git a/node/Packet.hpp b/node/Packet.hpp index a1ea73e1..b8e69fa9 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -978,9 +978,6 @@ public: * The instance ID is a random 64-bit value generated by each ZeroTier * node on startup. This is helpful in identifying traces from different * members of a cluster. - * - * The Dictionary serialization format is the same as used for network - * configurations. The maximum size of a trace is 10000 bytes. */ VERB_REMOTE_TRACE = 0x15 }; diff --git a/service/OneService.cpp b/service/OneService.cpp index 1b07eb79..115830e5 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -2058,6 +2058,12 @@ public: } } break; + case ZT_EVENT_REMOTE_TRACE: { + const ZT_RemoteTrace *rt = reinterpret_cast(metaData); + if ((rt)&&(rt->len > 0)&&(rt->len <= ZT_MAX_REMOTE_TRACE_SIZE)&&(rt->data)) + _controller->handleRemoteTrace(*rt); + } + default: break; } -- cgit v1.2.3 From d9552fb1203cd3abd9d15d9565f6d42e56058d30 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 14 Jul 2017 14:33:36 -0700 Subject: Add remoteTraceTarget to network and member configs in controller. --- controller/EmbeddedNetworkController.cpp | 119 +++++++++++++++++++++---------- controller/EmbeddedNetworkController.hpp | 2 + controller/JSONDB.cpp | 7 +- controller/JSONDB.hpp | 15 +++- 4 files changed, 102 insertions(+), 41 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8b8a93bd..c2024962 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -453,6 +453,8 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender) { this->_sender = sender; this->_signingId = signingId; + char tmp[64]; + this->_signingIdAddressString = signingId.address().toString(tmp); } void EmbeddedNetworkController::request( @@ -1085,51 +1087,80 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) { - // Convert Dictionary into JSON object - json d; - char *saveptr = (char *)0; - for(char *l=Utils::stok(rt.data,"\n",&saveptr);(l);l=Utils::stok((char *)0,"\n",&saveptr)) { - char *eq = strchr(l,'='); - if (eq > l) { - std::string k(l,(unsigned long)(eq - l)); - std::string v; - ++eq; - while (*eq) { - if (*eq == '\\') { - ++eq; - if (*eq) { - switch(*eq) { - case 'r': - v.push_back('\r'); - break; - case 'n': - v.push_back('\n'); - break; - case '0': - v.push_back((char)0); - break; - case 'e': - v.push_back('='); - break; - default: - v.push_back(*eq); - break; - } + try { + std::vector nw4m(_db.networksForMember(rt.origin)); + if (nw4m.empty()) // ignore these for unknown members + return; + + // Convert Dictionary into JSON object + json d; + char *saveptr = (char *)0; + for(char *l=Utils::stok(rt.data,"\n",&saveptr);(l);l=Utils::stok((char *)0,"\n",&saveptr)) { + char *eq = strchr(l,'='); + if (eq > l) { + std::string k(l,(unsigned long)(eq - l)); + std::string v; + ++eq; + while (*eq) { + if (*eq == '\\') { ++eq; + if (*eq) { + switch(*eq) { + case 'r': + v.push_back('\r'); + break; + case 'n': + v.push_back('\n'); + break; + case '0': + v.push_back((char)0); + break; + case 'e': + v.push_back('='); + break; + default: + v.push_back(*eq); + break; + } + ++eq; + } + } else { + v.push_back(*(eq++)); } - } else { - v.push_back(*(eq++)); } + if (v.length() > 0) + d[k] = v; } - if (v.length() > 0) - d[k] = v; } - } - char p[128]; - OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx_%.16llx.json",rt.origin,OSUtils::now()); - _db.writeRaw(p,OSUtils::jsonDump(d)); - //fprintf(stdout,"%s\n",OSUtils::jsonDump(d).c_str()); fflush(stdout); + bool accept = false; + for(std::vector::const_iterator nwid(nw4m.begin());nwid!=nw4m.end();++nwid) { + json nconf; + if (_db.getNetwork(*nwid,nconf)) { + try { + if (OSUtils::jsonString(nconf["remoteTraceTarget"],"") == _signingIdAddressString) { + accept = true; + break; + } + } catch ( ... ) {} // ignore missing fields or other errors, drop trace message + } + if (_db.getNetworkMember(*nwid,rt.origin,nconf)) { + try { + if (OSUtils::jsonString(nconf["remoteTraceTarget"],"") == _signingIdAddressString) { + accept = true; + break; + } + } catch ( ... ) {} // ignore missing fields or other errors, drop trace message + } + } + if (accept) { + char p[128]; + OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx_%.16llx.json",rt.origin,OSUtils::now()); + _db.writeRaw(p,OSUtils::jsonDump(d)); + } + } catch ( ... ) { + // drop invalid trace messages if an error occurs + } } void EmbeddedNetworkController::threadMain() @@ -1381,6 +1412,16 @@ void EmbeddedNetworkController::_request( nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU); nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); + std::string rtt(OSUtils::jsonString(member["remoteTraceTarget"],"")); + if (rtt.length() == 10) { + nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str())); + } else { + rtt = OSUtils::jsonString(network["remoteTraceTarget"],""); + if (rtt.length() == 10) { + nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str())); + } + } + for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) nc->addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 03ba0b95..c2dc55b5 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include "../node/Constants.hpp" @@ -218,6 +219,7 @@ private: NetworkController::Sender *_sender; Identity _signingId; + std::string _signingIdAddressString; struct _MemberStatusKey { diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 97a217a1..0c061266 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -181,6 +181,7 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co { Mutex::Lock _l(_networks_m); _networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig); + _members[nodeId].insert(networkId); } _recomputeSummaryInfo(networkId); } @@ -244,6 +245,7 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ { Mutex::Lock _l(_networks_m); + _members[nodeId].erase(networkId); std::unordered_map::iterator i(_networks.find(networkId)); if (i == _networks.end()) return _EMPTY_JSON; @@ -367,8 +369,10 @@ bool JSONDB::_load(const std::string &p) } else if ((id.length() == 10)&&(objtype == "member")) { const uint64_t mid = Utils::hexStrToU64(id.c_str()); const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); - if ((mid)&&(nwid)) + if ((mid)&&(nwid)) { _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); + _members[mid].insert(nwid); + } } } } @@ -403,6 +407,7 @@ bool JSONDB::_load(const std::string &p) if ((mid)&&(nwid)) { Mutex::Lock _l(_networks_m); _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); + _members[mid].insert(nwid); } } } catch ( ... ) {} diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index e164a14e..99b69ba2 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -29,6 +29,7 @@ #include #include #include +#include #include "../node/Constants.hpp" #include "../node/Utils.hpp" @@ -129,6 +130,17 @@ public: } } + inline std::vector networksForMember(const uint64_t nodeId) + { + Mutex::Lock _l(_networks_m); + std::unordered_map< uint64_t,std::unordered_set< uint64_t > >::const_iterator m(_members.find(nodeId)); + if (m != _members.end()) { + return std::vector(m->second.begin(),m->second.end()); + } else { + return std::vector(); + } + } + void threadMain() throw(); @@ -154,7 +166,8 @@ private: std::unordered_map< uint64_t,std::vector > members; }; - std::unordered_map _networks; + std::unordered_map< uint64_t,_NW > _networks; + std::unordered_map< uint64_t,std::unordered_set< uint64_t > > _members; Mutex _networks_m; }; -- cgit v1.2.3 From 1685659e37f568c727580634e412674cc266ff31 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Jul 2017 17:02:50 -0700 Subject: Remote tracing works. --- controller/EmbeddedNetworkController.cpp | 12 ++++++++--- node/IncomingPacket.cpp | 4 +++- node/NetworkConfig.hpp | 29 -------------------------- node/Packet.hpp | 4 ---- node/Trace.cpp | 35 ++++++++++++++++---------------- node/Trace.hpp | 2 +- 6 files changed, 30 insertions(+), 56 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c2024962..07ab5168 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1089,7 +1089,9 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) { try { std::vector nw4m(_db.networksForMember(rt.origin)); - if (nw4m.empty()) // ignore these for unknown members + + // Ignore remote traces from members we don't know about + if (nw4m.empty()) return; // Convert Dictionary into JSON object @@ -1133,7 +1135,8 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } - bool accept = false; + bool accept = true; + /* for(std::vector::const_iterator nwid(nw4m.begin());nwid!=nw4m.end();++nwid) { json nconf; if (_db.getNetwork(*nwid,nconf)) { @@ -1153,9 +1156,10 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } catch ( ... ) {} // ignore missing fields or other errors, drop trace message } } + */ if (accept) { char p[128]; - OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx_%.16llx.json",rt.origin,OSUtils::now()); + OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx-%.10llx-%.16llx",_signingId.address().toInt(),rt.origin,OSUtils::now()); _db.writeRaw(p,OSUtils::jsonDump(d)); } } catch ( ... ) { @@ -1419,6 +1423,8 @@ void EmbeddedNetworkController::_request( rtt = OSUtils::jsonString(network["remoteTraceTarget"],""); if (rtt.length() == 10) { nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str())); + } else { + nc->remoteTraceTarget = _signingId.address(); } } diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index be3d082b..51955bf3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -115,6 +115,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,tPtr,peer); case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,tPtr,peer); case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,tPtr,peer); + case Packet::VERB_REMOTE_TRACE: return _doREMOTE_TRACE(RR,tPtr,peer); } } else { RR->sw->requestWhois(tPtr,sourceAddress); @@ -1172,8 +1173,9 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con while (ptr < eof) { if (!*ptr) { // end of string rt.len = (unsigned int)(ptr - rt.data); - if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) + if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) { RR->node->postEvent(tPtr,ZT_EVENT_REMOTE_TRACE,&rt); + } rt.data = const_cast(++ptr); // start of next string, if any } else { ++ptr; diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index fb48edc9..3fd5ddac 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -410,35 +410,6 @@ public: return (Tag *)0; } - /* - inline void dump() const - { - printf("networkId==%.16llx\n",networkId); - printf("timestamp==%llu\n",timestamp); - printf("credentialTimeMaxDelta==%llu\n",credentialTimeMaxDelta); - printf("revision==%llu\n",revision); - printf("issuedTo==%.10llx\n",issuedTo.toInt()); - printf("multicastLimit==%u\n",multicastLimit); - printf("flags=%.8lx\n",(unsigned long)flags); - printf("specialistCount==%u\n",specialistCount); - for(unsigned int i=0;i(&(routes[i].target))->toString().c_str()); - printf(" routes[i].via==%s\n",reinterpret_cast(&(routes[i].via))->toIpString().c_str()); - printf(" routes[i].flags==%.4x\n",(unsigned int)routes[i].flags); - printf(" routes[i].metric==%u\n",(unsigned int)routes[i].metric); - } - printf("staticIpCount==%u\n",staticIpCount); - for(unsigned int i=0;ilocalSocket()); } d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id()); - _send(tPtr,d,network); + _send(tPtr,d,*network); } void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac,const char *reason) @@ -161,7 +161,7 @@ void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved) @@ -218,7 +218,7 @@ void Trace::networkConfigRequestSent(void *const tPtr,const Network &network,con d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_CONFIG_REQUEST_SENT_S); d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id()); d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID,controller); - _send(tPtr,d,0); + _send(tPtr,d,network); } void Trace::networkFilter( @@ -259,7 +259,7 @@ void Trace::networkFilter( d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen); if (frameLen > 0) d.add(ZT_REMOTE_TRACE_FIELD__FRAME_DATA,(const char *)frameData,(frameLen > 256) ? (int)256 : (int)frameLen); - _send(tPtr,d,network.id()); + _send(tPtr,d,network); } void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c,const char *reason) @@ -273,7 +273,7 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c,const char *reason) @@ -287,7 +287,7 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c, d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const CertificateOfRepresentation &c,const char *reason) @@ -313,7 +313,7 @@ void Trace::credentialRejected(void *const tPtr,const Capability &c,const char * d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason) @@ -328,7 +328,7 @@ void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char *reason) @@ -341,7 +341,7 @@ void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char * d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c) @@ -353,7 +353,7 @@ void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) @@ -365,7 +365,7 @@ void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c) @@ -387,7 +387,7 @@ void Trace::credentialAccepted(void *const tPtr,const Capability &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const Tag &c) @@ -400,7 +400,7 @@ void Trace::credentialAccepted(void *const tPtr,const Tag &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const Revocation &c) @@ -411,7 +411,7 @@ void Trace::credentialAccepted(void *const tPtr,const Revocation &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::_send(void *const tPtr,const Dictionary &d) @@ -434,7 +434,6 @@ void Trace::_send(void *const tPtr,const Dictionary &d } } _traceMsgBuf[i] = (char)0; - //printf("%s\n",_traceMsgBuf); RR->node->postEvent(tPtr,ZT_EVENT_TRACE,_traceMsgBuf); #endif @@ -461,11 +460,11 @@ void Trace::_send(void *const tPtr,const Dictionary &d } } -void Trace::_send(void *const tPtr,const Dictionary &d,const SharedPtr &network) +void Trace::_send(void *const tPtr,const Dictionary &d,const Network &network) { _send(tPtr,d); - if ((network)&&(network->config().remoteTraceTarget)) { - Packet outp(network->config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE); + if (network.config().remoteTraceTarget) { + Packet outp(network.config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE); outp.appendCString(d.data()); outp.compress(); RR->sw->send(tPtr,outp,true); diff --git a/node/Trace.hpp b/node/Trace.hpp index 7fe48cdd..d66d0871 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -154,7 +154,7 @@ private: void _send(void *const tPtr,const Dictionary &d); void _send(void *const tPtr,const Dictionary &d,const uint64_t networkId); - void _send(void *const tPtr,const Dictionary &d,const SharedPtr &network); + void _send(void *const tPtr,const Dictionary &d,const Network &network); #ifdef ZT_TRACE char _traceMsgBuf[4096]; -- cgit v1.2.3 From 727ccb112543f3c44da3d094fa755e3a5d25cc3e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Jul 2017 13:57:37 -0700 Subject: Cleanup and stdin/stdout harness mode for controller. --- controller/JSONDB.cpp | 185 +++++++++++++++++++++++++++++++------------- controller/JSONDB.hpp | 3 + node/IncomingPacket.cpp | 2 - service/SoftwareUpdater.cpp | 9 --- 4 files changed, 133 insertions(+), 66 deletions(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 0c061266..7f92d6ee 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -16,6 +16,18 @@ * along with this program. If not, see . */ +#include +#include +#include +#ifndef __WINDOWS__ +#include +#include +#include +#include +#include +#include +#endif + #include "JSONDB.hpp" #define ZT_JSONDB_HTTP_TIMEOUT 60000 @@ -27,9 +39,12 @@ static const std::map _ZT_JSONDB_GET_HEADERS; JSONDB::JSONDB(const std::string &basePath) : _basePath(basePath), + _rawInput(-1), + _rawOutput(-1), _summaryThreadRun(true) { if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { + // If base path is http:// we run in HTTP mode // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. // Typically it's just used with 127.0.0.1 anyway. std::string hn = _basePath.substr(7); @@ -46,16 +61,27 @@ JSONDB::JSONDB(const std::string &basePath) : _basePath = "/"; if (_basePath[0] != '/') _basePath = std::string("/") + _basePath; +#ifndef __WINDOWS__ + } else if (_basePath == "-") { + // If base path is "-" we run in stdin/stdout mode and expect our database to be populated on startup via stdin + // Not supported on Windows + _rawInput = STDIN_FILENO; + _rawOutput = STDOUT_FILENO; + fcntl(_rawInput,F_SETFL,O_NONBLOCK); +#endif } else { + // Default mode of operation is to store files in the filesystem OSUtils::mkdir(_basePath.c_str()); OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions } - unsigned int cnt = 0; - while (!_load(_basePath)) { - if ((++cnt & 7) == 0) - fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); - Thread::sleep(250); + if (_rawInput < 0) { + unsigned int cnt = 0; + while (!_load(_basePath)) { + if ((++cnt & 7) == 0) + fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); + Thread::sleep(250); + } } for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) @@ -89,7 +115,18 @@ JSONDB::~JSONDB() bool JSONDB::writeRaw(const std::string &n,const std::string &obj) { - if (_httpAddr) { + if (_rawOutput >= 0) { +#ifndef __WINDOWS__ + if (obj.length() > 0) { + Mutex::Lock _l(_rawLock); + if (write(_rawOutput,obj.c_str(),obj.length() + 1) > 0) + return true; + } else { + return true; + } +#endif + return false; + } else if (_httpAddr) { std::map headers; std::string body; std::map reqHeaders; @@ -205,11 +242,13 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) char n[256]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); - if (_httpAddr) { - // Deletion is currently done by Central in harnessed mode - //std::map headers; - //std::string body; - //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + if (_rawOutput >= 0) { + // In harnessed mode, deletes occur in Central or other management + // software and do not need to be executed this way. + } else if (_httpAddr) { + std::map headers; + std::string body; + Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); } else { const std::string path(_genPath(n,false)); if (path.length()) @@ -232,11 +271,13 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ char n[256]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); - if (_httpAddr) { - // Deletion is currently done by the caller in Central harnessed mode - //std::map headers; - //std::string body; - //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + if (_rawOutput >= 0) { + // In harnessed mode, deletes occur in Central or other management + // software and do not need to be executed this way. + } else if (_httpAddr) { + std::map headers; + std::string body; + Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); } else { const std::string path(_genPath(n,false)); if (path.length()) @@ -263,9 +304,41 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ void JSONDB::threadMain() throw() { +#ifndef __WINDOWS__ + fd_set readfds,nullfds; + char *const readbuf = (_rawInput >= 0) ? (new char[1048576]) : (char *)0; + std::string rawInputBuf; + FD_ZERO(&readfds); + FD_ZERO(&nullfds); +#endif + std::vector todo; + while (_summaryThreadRun) { - Thread::sleep(10); +#ifndef __WINDOWS__ + if (_rawInput < 0) { + Thread::sleep(25); + } else { + FD_SET(_rawInput,&readfds); + struct timeval tv; + tv.tv_sec = 0; + tv.tv_usec = 25000; + select(_rawInput+1,&readfds,&nullfds,&nullfds,&tv); + if (FD_ISSET(_rawInput,&readfds)) { + const long rn = (long)read(_rawInput,readbuf,1048576); + for(long i=0;i 0) { + _add(OSUtils::jsonParse(rawInputBuf)); + rawInputBuf.clear(); + } + } + } + } +#else + Thread::sleep(25); +#endif { Mutex::Lock _l(_summaryThread_m); @@ -273,7 +346,6 @@ void JSONDB::threadMain() continue; else _summaryThreadToDo.swap(todo); } - const uint64_t now = OSUtils::now(); for(std::vector::iterator ii(todo.begin());ii!=todo.end();++ii) { const uint64_t networkId = *ii; @@ -340,10 +412,46 @@ void JSONDB::threadMain() todo.clear(); } + +#ifndef __WINDOWS__ + delete [] readbuf; +#endif +} + +bool JSONDB::_add(const nlohmann::json &j) +{ + try { + if (j.is_object()) { + std::string id(OSUtils::jsonString(j["id"],"0")); + std::string objtype(OSUtils::jsonString(j["objtype"],"")); + + if ((id.length() == 16)&&(objtype == "network")) { + const uint64_t nwid = Utils::hexStrToU64(id.c_str()); + if (nwid) { + Mutex::Lock _l(_networks_m); + _networks[nwid].config = nlohmann::json::to_msgpack(j); + return true; + } + } else if ((id.length() == 10)&&(objtype == "member")) { + const uint64_t mid = Utils::hexStrToU64(id.c_str()); + const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); + if ((mid)&&(nwid)) { + Mutex::Lock _l(_networks_m); + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); + _members[mid].insert(nwid); + return true; + } + } + } + } catch ( ... ) {} + return false; } bool JSONDB::_load(const std::string &p) { + // This is not used in stdin/stdout mode. Instead data is populated by + // sending it all to stdin. + if (_httpAddr) { // In HTTP harnessed mode we download our entire working data set on startup. @@ -357,24 +465,9 @@ bool JSONDB::_load(const std::string &p) if (dbImg.is_object()) { Mutex::Lock _l(_networks_m); for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { - nlohmann::json &j = i.value(); - if (j.is_object()) { - std::string id(OSUtils::jsonString(j["id"],"0")); - std::string objtype(OSUtils::jsonString(j["objtype"],"")); - - if ((id.length() == 16)&&(objtype == "network")) { - const uint64_t nwid = Utils::hexStrToU64(id.c_str()); - if (nwid) - _networks[nwid].config = nlohmann::json::to_msgpack(j); - } else if ((id.length() == 10)&&(objtype == "member")) { - const uint64_t mid = Utils::hexStrToU64(id.c_str()); - const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); - if ((mid)&&(nwid)) { - _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); - _members[mid].insert(nwid); - } - } - } + try { + _add(i.value()); + } catch ( ... ) {} } return true; } @@ -391,25 +484,7 @@ bool JSONDB::_load(const std::string &p) std::string buf; if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) { try { - nlohmann::json j(OSUtils::jsonParse(buf)); - std::string id(OSUtils::jsonString(j["id"],"0")); - std::string objtype(OSUtils::jsonString(j["objtype"],"")); - - if ((id.length() == 16)&&(objtype == "network")) { - const uint64_t nwid = Utils::hexStrToU64(id.c_str()); - if (nwid) { - Mutex::Lock _l(_networks_m); - _networks[nwid].config = nlohmann::json::to_msgpack(j); - } - } else if ((id.length() == 10)&&(objtype == "member")) { - const uint64_t mid = Utils::hexStrToU64(id.c_str()); - const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); - if ((mid)&&(nwid)) { - Mutex::Lock _l(_networks_m); - _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); - _members[mid].insert(nwid); - } - } + _add(OSUtils::jsonParse(buf)); } catch ( ... ) {} } } else { diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 99b69ba2..23d00a51 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -145,12 +145,15 @@ public: throw(); private: + bool _add(const nlohmann::json &j); bool _load(const std::string &p); void _recomputeSummaryInfo(const uint64_t networkId); std::string _genPath(const std::string &n,bool create); std::string _basePath; InetAddress _httpAddr; + int _rawInput,_rawOutput; + Mutex _rawLock; Thread _summaryThread; std::vector _summaryThreadToDo; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 51955bf3..e5e10476 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -81,14 +81,12 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) if (peer) { if (!trusted) { if (!dearmor(peer->key())) { - //fprintf(stderr,"dropped packet from %s(%s), MAC authentication failed (size: %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops()); return true; } } if (!uncompress()) { - //fprintf(stderr,"dropped packet from %s(%s), compressed data invalid (size %u, verb may be %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size(),(unsigned int)verb()); RR->t->incomingPacketInvalid(tPtr,_path,packetId(),sourceAddress,hops(),Packet::VERB_NOP,"LZ4 decompression failed"); return true; } diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 57ecce78..11005945 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -243,7 +243,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } } @@ -258,7 +257,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; idx |= (unsigned long)*(reinterpret_cast(data) + 20); - //printf("<< GET_DATA @%u from %.10llx for %s\n",(unsigned int)idx,origin,Utils::hex(reinterpret_cast(data) + 1,16).c_str()); std::map< Array,_D >::iterator d(_dist.find(Array(reinterpret_cast(data) + 1))); if ((d != _dist.end())&&(idx < (unsigned long)d->second.bin.length())) { Buffer buf; @@ -267,7 +265,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void buf.append((uint32_t)idx); buf.append(d->second.bin.data() + idx,std::min((unsigned long)ZT_SOFTWARE_UPDATE_CHUNK_SIZE,(unsigned long)(d->second.bin.length() - idx))); _node.sendUserMessage((void *)0,origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,buf.data(),buf.size()); - //printf(">> DATA @%u\n",(unsigned int)idx); } } break; @@ -278,7 +275,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; idx |= (unsigned long)*(reinterpret_cast(data) + 20); - //printf("<< DATA @%u / %u bytes (we now have %u bytes)\n",(unsigned int)idx,(unsigned int)(len - 21),(unsigned int)_download.length()); if (idx == (unsigned long)_download.length()) { _download.append(reinterpret_cast(data) + 21,len - 21); if (_download.length() < _downloadLength) { @@ -287,7 +283,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } } @@ -334,7 +329,6 @@ bool SoftwareUpdater::check(const uint64_t now) (int)ZT_VENDOR_ZEROTIER, _channel.c_str()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,tmp,len); - //printf(">> GET_LATEST\n"); } if (_latestValid) @@ -360,7 +354,6 @@ bool SoftwareUpdater::check(const uint64_t now) if (OSUtils::writeFile(binPath.c_str(),_download)) { OSUtils::lockDownFile(binPath.c_str(),false); _latestValid = true; - //printf("VALID UPDATE\n%s\n",OSUtils::jsonDump(_latestMeta).c_str()); _download = std::string(); _downloadLength = 0; return true; @@ -370,7 +363,6 @@ bool SoftwareUpdater::check(const uint64_t now) } catch ( ... ) {} // any exception equals verification failure // If we get here, checks failed. - //printf("INVALID UPDATE (!!!)\n%s\n",OSUtils::jsonDump(_latestMeta).c_str()); OSUtils::rm(binPath.c_str()); _latestMeta = nlohmann::json(); _latestValid = false; @@ -382,7 +374,6 @@ bool SoftwareUpdater::check(const uint64_t now) gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } -- cgit v1.2.3 From ae65eb5105d66859ff4c8b4aefdf4993fe6bc35c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Jul 2017 14:28:38 -0700 Subject: Cleanup and replace trace field names with sane ones because usability > a few bytes. --- controller/EmbeddedNetworkController.cpp | 9 +++- controller/JSONDB.cpp | 1 + include/ZeroTierOne.h | 70 ++++++++++++++++---------------- 3 files changed, 44 insertions(+), 36 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 07ab5168..2caec827 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1087,6 +1087,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) { + static volatile unsigned long idCounter = 0; + + char id[128]; try { std::vector nw4m(_db.networksForMember(rt.origin)); @@ -1135,6 +1138,10 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } + OSUtils::ztsnprintf(id,sizeof(id),"%.10llx-%.10llx-%.16llx-%.8lx",_signingId.address().toInt(),rt.origin,OSUtils::now(),++idCounter); + d["id"] = id; + d["objtype"] = "trace"; + bool accept = true; /* for(std::vector::const_iterator nwid(nw4m.begin());nwid!=nw4m.end();++nwid) { @@ -1159,7 +1166,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) */ if (accept) { char p[128]; - OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx-%.10llx-%.16llx",_signingId.address().toInt(),rt.origin,OSUtils::now()); + OSUtils::ztsnprintf(p,sizeof(p),"trace/%s",id); _db.writeRaw(p,OSUtils::jsonDump(d)); } } catch ( ... ) { diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 7f92d6ee..ad9ba248 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -334,6 +334,7 @@ void JSONDB::threadMain() rawInputBuf.clear(); } } + continue; // we only want to do the stuff below this every few dozen ms or so, so pause again } } #else diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index b123e8e3..e325b369 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -258,42 +258,42 @@ extern "C" { #define ZT_RULE_PACKET_CHARACTERISTICS_TCP_FIN 0x0000000000000001ULL // Fields in remote trace dictionaries -#define ZT_REMOTE_TRACE_FIELD__EVENT "E" -#define ZT_REMOTE_TRACE_FIELD__PACKET_ID "pid" -#define ZT_REMOTE_TRACE_FIELD__PACKET_VERB "pv" -#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_ID "ptpid" -#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_APPROVED "ptpok" -#define ZT_REMOTE_TRACE_FIELD__PACKET_HOPS "phops" -#define ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR "oldrphy" -#define ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR "rzt" -#define ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR "rphy" -#define ZT_REMOTE_TRACE_FIELD__LOCAL_ZTADDR "lzt" -#define ZT_REMOTE_TRACE_FIELD__LOCAL_PHYADDR "lphy" -#define ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET "ls" -#define ZT_REMOTE_TRACE_FIELD__IP_SCOPE "ipsc" -#define ZT_REMOTE_TRACE_FIELD__NETWORK_ID "nwid" -#define ZT_REMOTE_TRACE_FIELD__SOURCE_ZTADDR "szt" -#define ZT_REMOTE_TRACE_FIELD__DEST_ZTADDR "dzt" -#define ZT_REMOTE_TRACE_FIELD__SOURCE_MAC "seth" -#define ZT_REMOTE_TRACE_FIELD__DEST_MAC "deth" -#define ZT_REMOTE_TRACE_FIELD__ETHERTYPE "et" -#define ZT_REMOTE_TRACE_FIELD__VLAN_ID "vlan" -#define ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH "fl" -#define ZT_REMOTE_TRACE_FIELD__FRAME_DATA "fd" -#define ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_NOTEE "ffnotee" -#define ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_INBOUND "ffdir" -#define ZT_REMOTE_TRACE_FIELD__FILTER_RESULT "fresult" -#define ZT_REMOTE_TRACE_FIELD__FILTER_BASE_RULE_LOG "frlog" -#define ZT_REMOTE_TRACE_FIELD__FILTER_CAP_RULE_LOG "fclog" -#define ZT_REMOTE_TRACE_FIELD__FILTER_CAP_ID "fcid" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE "crtype" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID "crid" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP "crts" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO "crinfo" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO "criss" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET "crrevt" +#define ZT_REMOTE_TRACE_FIELD__EVENT "event" +#define ZT_REMOTE_TRACE_FIELD__PACKET_ID "packetId" +#define ZT_REMOTE_TRACE_FIELD__PACKET_VERB "packetVerb" +#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_ID "packetTrustedPathId" +#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_APPROVED "packetTrustedPathApproved" +#define ZT_REMOTE_TRACE_FIELD__PACKET_HOPS "packetHops" +#define ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR "oldRemotePhyAddr" +#define ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR "remoteZtAddr" +#define ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR "remotePhyAddr" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_ZTADDR "localZtAddr" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_PHYADDR "localPhyAddr" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET "localSocket" +#define ZT_REMOTE_TRACE_FIELD__IP_SCOPE "phyAddrIpScope" +#define ZT_REMOTE_TRACE_FIELD__NETWORK_ID "networkId" +#define ZT_REMOTE_TRACE_FIELD__SOURCE_ZTADDR "sourceZtAddr" +#define ZT_REMOTE_TRACE_FIELD__DEST_ZTADDR "destZtAddr" +#define ZT_REMOTE_TRACE_FIELD__SOURCE_MAC "sourceMac" +#define ZT_REMOTE_TRACE_FIELD__DEST_MAC "destMac" +#define ZT_REMOTE_TRACE_FIELD__ETHERTYPE "etherType" +#define ZT_REMOTE_TRACE_FIELD__VLAN_ID "vlanId" +#define ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH "frameLength" +#define ZT_REMOTE_TRACE_FIELD__FRAME_DATA "frameData" +#define ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_NOTEE "filterNoTee" +#define ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_INBOUND "filterInbound" +#define ZT_REMOTE_TRACE_FIELD__FILTER_RESULT "filterResult" +#define ZT_REMOTE_TRACE_FIELD__FILTER_BASE_RULE_LOG "filterBaseRuleLog" +#define ZT_REMOTE_TRACE_FIELD__FILTER_CAP_RULE_LOG "filterCapRuleLog" +#define ZT_REMOTE_TRACE_FIELD__FILTER_CAP_ID "filterMatchingCapId" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE "credType" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID "credId" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP "credTs" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO "credInfo" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO "credIssuedTo" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET "credRevocationTarget" #define ZT_REMOTE_TRACE_FIELD__REASON "reason" -#define ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID "nwctrl" +#define ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID "networkControllerId" // Event types in remote traces #define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE 0x1000 -- cgit v1.2.3 From 31785f7f6ec27e826efc3cc2b45979e5d58f37bb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Jul 2017 15:36:33 -0700 Subject: Automatic periodic status dump from controller. --- controller/EmbeddedNetworkController.cpp | 72 ++++++++++++-------------------- controller/EmbeddedNetworkController.hpp | 4 +- osdep/BlockingQueue.hpp | 28 +++++++++++-- 3 files changed, 52 insertions(+), 52 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 2caec827..cb0c05af 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -429,6 +429,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _startTime(OSUtils::now()), _running(true), + _lastDumpedStatus(0), _db(dbPath), _node(node) { @@ -1010,20 +1011,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } // else 404 - } else if (path[0] == "ping") { - - _startThreads(); - _RQEntry *qe = new _RQEntry; - qe->type = _RQEntry::RQENTRY_TYPE_PING; - _queue.post(qe); - - char tmp[64]; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"{\"clock\":%llu,\"ping\":%s}",(unsigned long long)now,OSUtils::jsonDump(b).c_str()); - responseBody = tmp; - responseContentType = "application/json"; - - return 200; - } return 404; @@ -1088,8 +1075,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) { static volatile unsigned long idCounter = 0; - char id[128]; + std::string k,v; + try { std::vector nw4m(_db.networksForMember(rt.origin)); @@ -1103,29 +1091,19 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) for(char *l=Utils::stok(rt.data,"\n",&saveptr);(l);l=Utils::stok((char *)0,"\n",&saveptr)) { char *eq = strchr(l,'='); if (eq > l) { - std::string k(l,(unsigned long)(eq - l)); - std::string v; + k.assign(l,(unsigned long)(eq - l)); + v.clear(); ++eq; while (*eq) { if (*eq == '\\') { ++eq; if (*eq) { switch(*eq) { - case 'r': - v.push_back('\r'); - break; - case 'n': - v.push_back('\n'); - break; - case '0': - v.push_back((char)0); - break; - case 'e': - v.push_back('='); - break; - default: - v.push_back(*eq); - break; + case 'r': v.push_back('\r'); break; + case 'n': v.push_back('\n'); break; + case '0': v.push_back((char)0); break; + case 'e': v.push_back('='); break; + default: v.push_back(*eq); break; } ++eq; } @@ -1133,7 +1111,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) v.push_back(*(eq++)); } } - if (v.length() > 0) + if ((k.length() > 0)&&(v.length() > 0)) d[k] = v; } } @@ -1177,20 +1155,23 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) void EmbeddedNetworkController::threadMain() throw() { + char tmp[256]; _RQEntry *qe = (_RQEntry *)0; - while ((_running)&&(_queue.get(qe))) { + while ((_running)&&(_queue.get(qe,1000) != BlockingQueue<_RQEntry *>::STOP)) { try { - if (qe->type == _RQEntry::RQENTRY_TYPE_REQUEST) { + if (qe->type == _RQEntry::RQENTRY_TYPE_REQUEST) _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); - } else if (qe->type == _RQEntry::RQENTRY_TYPE_PING) { - const uint64_t now = OSUtils::now(); + + // Every 10s we update a 'status' containing member online state, etc. + const uint64_t now = OSUtils::now(); + if ((now - _lastDumpedStatus) >= 10000) { + _lastDumpedStatus = now; bool first = true; - std::string pong("{\"memberStatus\":{"); + std::string st("{\"id\":\"status\",\"objtype\":\"status\",\"memberStatus\":{"); { Mutex::Lock _l(_memberStatus_m); - pong.reserve(48 * (_memberStatus.size() + 1)); - _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { - char tmp[64]; + st.reserve(48 * (_memberStatus.size() + 1)); + _db.eachId([this,&st,&now,&first,&tmp](uint64_t networkId,uint64_t nodeId) { uint64_t lrt = 0ULL; auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); if (ms != _memberStatus.end()) @@ -1200,14 +1181,13 @@ void EmbeddedNetworkController::threadMain() (unsigned long long)networkId, (unsigned long long)nodeId, (unsigned long long)lrt); - pong.append(tmp); + st.append(tmp); first = false; }); } - char tmp2[256]; - OSUtils::ztsnprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); - pong.append(tmp2); - _db.writeRaw("pong",pong); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + st.append(tmp); + _db.writeRaw("status",st); } } catch ( ... ) {} delete qe; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index c2dc55b5..8752922e 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -105,8 +105,7 @@ private: Identity identity; Dictionary metaData; enum { - RQENTRY_TYPE_REQUEST = 0, - RQENTRY_TYPE_PING = 1 + RQENTRY_TYPE_REQUEST = 0 } type; }; @@ -210,6 +209,7 @@ private: volatile bool _running; BlockingQueue<_RQEntry *> _queue; std::vector _threads; + volatile uint64_t _lastDumpedStatus; Mutex _threads_m; JSONDB _db; diff --git a/osdep/BlockingQueue.hpp b/osdep/BlockingQueue.hpp index 43ae7435..5e1a24ef 100644 --- a/osdep/BlockingQueue.hpp +++ b/osdep/BlockingQueue.hpp @@ -30,6 +30,7 @@ #include #include #include +#include namespace ZeroTier { @@ -58,10 +59,6 @@ public: c.notify_all(); } - /** - * @param value Value to set to next queue item if return value is true - * @return False if stop() has been called, true otherwise - */ inline bool get(T &value) { std::unique_lock lock(m); @@ -75,6 +72,29 @@ public: return true; } + enum TimedWaitResult + { + OK, + TIMED_OUT, + STOP + }; + + inline TimedWaitResult get(T &value,const unsigned long ms) + { + const std::chrono::milliseconds ms2{ms}; + std::unique_lock lock(m); + if (!r) return STOP; + while (q.empty()) { + if (c.wait_for(lock,ms2) == std::cv_status::timeout) + return ((r) ? TIMED_OUT : STOP); + else if (!r) + return STOP; + } + value = q.front(); + q.pop(); + return OK; + } + private: volatile bool r; std::queue q; -- cgit v1.2.3 From fc7728212f261b081343ee4b9e801cee0d60ad37 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Tue, 18 Jul 2017 15:59:11 -0700 Subject: Fix ifndef on Windows --- controller/JSONDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index ad9ba248..10be1380 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -19,7 +19,7 @@ #include #include #include -#ifndef __WINDOWS__ +#ifndef _WIN32 #include #include #include -- cgit v1.2.3 From b62296a40bd4ed2d01404679cdc9512a1f18bcca Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 19 Jul 2017 14:13:17 -0700 Subject: Bug fixes in new harness mode. --- controller/EmbeddedNetworkController.cpp | 20 ++-- controller/JSONDB.cpp | 194 ++++++++++++++++++------------- controller/JSONDB.hpp | 12 ++ osdep/OSUtils.cpp | 2 +- osdep/OSUtils.hpp | 2 +- service/OneService.cpp | 29 ++--- 6 files changed, 147 insertions(+), 112 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index cb0c05af..70691fe0 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1145,7 +1145,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) if (accept) { char p[128]; OSUtils::ztsnprintf(p,sizeof(p),"trace/%s",id); - _db.writeRaw(p,OSUtils::jsonDump(d)); + _db.writeRaw(p,OSUtils::jsonDump(d,-1)); } } catch ( ... ) { // drop invalid trace messages if an error occurs @@ -1157,17 +1157,24 @@ void EmbeddedNetworkController::threadMain() { char tmp[256]; _RQEntry *qe = (_RQEntry *)0; - while ((_running)&&(_queue.get(qe,1000) != BlockingQueue<_RQEntry *>::STOP)) { + while (_running) { + const BlockingQueue<_RQEntry *>::TimedWaitResult wr = _queue.get(qe,1000); + if ((wr == BlockingQueue<_RQEntry *>::STOP)||(!_running)) + break; + try { - if (qe->type == _RQEntry::RQENTRY_TYPE_REQUEST) + if ((wr == BlockingQueue<_RQEntry *>::OK)&&(qe->type == _RQEntry::RQENTRY_TYPE_REQUEST)) { _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); + delete qe; + } // Every 10s we update a 'status' containing member online state, etc. const uint64_t now = OSUtils::now(); if ((now - _lastDumpedStatus) >= 10000) { _lastDumpedStatus = now; bool first = true; - std::string st("{\"id\":\"status\",\"objtype\":\"status\",\"memberStatus\":{"); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"{\"id\":\"%.10llx-status\",\"objtype\":\"status\",\"memberStatus\":[",_signingId.address().toInt()); + std::string st(tmp); { Mutex::Lock _l(_memberStatus_m); st.reserve(48 * (_memberStatus.size() + 1)); @@ -1176,7 +1183,7 @@ void EmbeddedNetworkController::threadMain() auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); if (ms != _memberStatus.end()) lrt = ms->second.lastRequestTime; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx\",%llu,%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, @@ -1185,12 +1192,11 @@ void EmbeddedNetworkController::threadMain() first = false; }); } - OSUtils::ztsnprintf(tmp,sizeof(tmp),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"],\"clock\":%llu,\"startTime\":%llu,\"uptime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime,(unsigned long long)(now - _startTime)); st.append(tmp); _db.writeRaw("status",st); } } catch ( ... ) {} - delete qe; } } diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index ad9ba248..216470b4 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -41,7 +41,8 @@ JSONDB::JSONDB(const std::string &basePath) : _basePath(basePath), _rawInput(-1), _rawOutput(-1), - _summaryThreadRun(true) + _summaryThreadRun(true), + _dataReady(false) { if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { // If base path is http:// we run in HTTP mode @@ -75,6 +76,8 @@ JSONDB::JSONDB(const std::string &basePath) : OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions } + _networks_m.lock(); // locked until data is loaded, etc. + if (_rawInput < 0) { unsigned int cnt = 0; while (!_load(_basePath)) { @@ -82,27 +85,25 @@ JSONDB::JSONDB(const std::string &basePath) : fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); Thread::sleep(250); } - } - for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) - _recomputeSummaryInfo(n->first); - for(;;) { - _summaryThread_m.lock(); - if (_summaryThreadToDo.empty()) { - _summaryThread_m.unlock(); - break; + for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) + _summaryThreadToDo.push_back(n->first); + + if (_summaryThreadToDo.size() > 0) { + _summaryThread = Thread::start(this); + } else { + _dataReady = true; + _networks_m.unlock(); } - _summaryThread_m.unlock(); - Thread::sleep(50); + } else { + // In IPC mode we wait for the first message to start, and we start + // this thread since this thread is responsible for reading from stdin. + _summaryThread = Thread::start(this); } } JSONDB::~JSONDB() { - { - Mutex::Lock _l(_networks_m); - _networks.clear(); - } Thread t; { Mutex::Lock _l(_summaryThread_m); @@ -119,11 +120,11 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) #ifndef __WINDOWS__ if (obj.length() > 0) { Mutex::Lock _l(_rawLock); - if (write(_rawOutput,obj.c_str(),obj.length() + 1) > 0) - return true; - } else { - return true; - } + if ((long)write(_rawOutput,obj.data(),obj.length()) == (long)obj.length()) { + if (write(_rawOutput,"\n",1) == 1) + return true; + } + } else return true; #endif return false; } else if (_httpAddr) { @@ -202,7 +203,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC { char n[64]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); - writeRaw(n,OSUtils::jsonDump(networkConfig)); + writeRaw(n,OSUtils::jsonDump(networkConfig,-1)); { Mutex::Lock _l(_networks_m); _networks[networkId].config = nlohmann::json::to_msgpack(networkConfig); @@ -214,7 +215,7 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co { char n[256]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); - writeRaw(n,OSUtils::jsonDump(memberConfig)); + writeRaw(n,OSUtils::jsonDump(memberConfig,-1)); { Mutex::Lock _l(_networks_m); _networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig); @@ -310,6 +311,7 @@ void JSONDB::threadMain() std::string rawInputBuf; FD_ZERO(&readfds); FD_ZERO(&nullfds); + struct timeval tv; #endif std::vector todo; @@ -317,24 +319,42 @@ void JSONDB::threadMain() while (_summaryThreadRun) { #ifndef __WINDOWS__ if (_rawInput < 0) { + // In HTTP and filesystem mode we just wait for summary to-do items Thread::sleep(25); } else { + // In IPC mode we wait but also select() on STDIN to read database updates FD_SET(_rawInput,&readfds); - struct timeval tv; tv.tv_sec = 0; tv.tv_usec = 25000; select(_rawInput+1,&readfds,&nullfds,&nullfds,&tv); if (FD_ISSET(_rawInput,&readfds)) { const long rn = (long)read(_rawInput,readbuf,1048576); + bool gotMessage = false; for(long i=0;i 0) { - _add(OSUtils::jsonParse(rawInputBuf)); + try { + const nlohmann::json obj(OSUtils::jsonParse(rawInputBuf)); + + gotMessage = true; + if (!_dataReady) { + _dataReady = true; + _networks_m.unlock(); + } + + if (obj.is_array()) { + for(unsigned long i=0;i::iterator ii(todo.begin());ii!=todo.end();++ii) { - const uint64_t networkId = *ii; + if (!_dataReady) { + _dataReady = true; + _networks_m.unlock(); + } + + const uint64_t now = OSUtils::now(); + try { Mutex::Lock _l(_networks_m); - std::unordered_map::iterator n(_networks.find(networkId)); - if (n != _networks.end()) { - NetworkSummaryInfo &ns = n->second.summaryInfo; - ns.activeBridges.clear(); - ns.allocatedIps.clear(); - ns.authorizedMemberCount = 0; - ns.activeMemberCount = 0; - ns.totalMemberCount = 0; - ns.mostRecentDeauthTime = 0; - - for(std::unordered_map< uint64_t,std::vector >::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { - try { - nlohmann::json member(nlohmann::json::from_msgpack(m->second)); - - if (OSUtils::jsonBool(member["authorized"],false)) { - ++ns.authorizedMemberCount; - - try { - const nlohmann::json &mlog = member["recentLog"]; - if ((mlog.is_array())&&(mlog.size() > 0)) { - const nlohmann::json &mlog1 = mlog[0]; - if (mlog1.is_object()) { - if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < (ZT_NETWORK_AUTOCONF_DELAY * 2)) - ++ns.activeMemberCount; + for(std::vector::iterator ii(todo.begin());ii!=todo.end();++ii) { + const uint64_t networkId = *ii; + std::unordered_map::iterator n(_networks.find(networkId)); + if (n != _networks.end()) { + NetworkSummaryInfo &ns = n->second.summaryInfo; + ns.activeBridges.clear(); + ns.allocatedIps.clear(); + ns.authorizedMemberCount = 0; + ns.activeMemberCount = 0; + ns.totalMemberCount = 0; + ns.mostRecentDeauthTime = 0; + + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { + try { + nlohmann::json member(nlohmann::json::from_msgpack(m->second)); + + if (OSUtils::jsonBool(member["authorized"],false)) { + ++ns.authorizedMemberCount; + + try { + const nlohmann::json &mlog = member["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + const nlohmann::json &mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < (ZT_NETWORK_AUTOCONF_DELAY * 2)) + ++ns.activeMemberCount; + } } - } - } catch ( ... ) {} - - try { - if (OSUtils::jsonBool(member["activeBridge"],false)) - ns.activeBridges.push_back(Address(m->first)); - } catch ( ... ) {} - - try { - const nlohmann::json &mips = member["ipAssignments"]; - if (mips.is_array()) { - for(unsigned long i=0;ifirst)); + } catch ( ... ) {} + + try { + const nlohmann::json &mips = member["ipAssignments"]; + if (mips.is_array()) { + for(unsigned long i=0;isecond.summaryInfoLastComputed = now; + n->second.summaryInfoLastComputed = now; + } } - } + } catch ( ... ) {} todo.clear(); } + if (!_dataReady) // sanity check + _networks_m.unlock(); + #ifndef __WINDOWS__ delete [] readbuf; #endif diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 23d00a51..7131b0c1 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -62,6 +62,17 @@ public: JSONDB(const std::string &basePath); ~JSONDB(); + /** + * Write a JSON object to the data store + * + * It's important that obj contain a valid JSON object with no newlines (jsonDump with -1 + * for indentation), since newline-delimited JSON is what nodeJS's IPC speaks and this + * is important in Central-harnessed mode. + * + * @param n Path name of object + * @param obj Object in single-line no-CRs JSON object format (OSUtils::jsonDump(obj,-1)) + * @return True if write appears successful + */ bool writeRaw(const std::string &n,const std::string &obj); bool hasNetwork(const uint64_t networkId) const; @@ -171,6 +182,7 @@ private: std::unordered_map< uint64_t,_NW > _networks; std::unordered_map< uint64_t,std::unordered_set< uint64_t > > _members; + bool _dataReady; Mutex _networks_m; }; diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 882b8255..c1edc353 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -434,7 +434,7 @@ std::string OSUtils::platformDefaultHomePath() // Inline these massive JSON operations in one place only to reduce binary footprint and compile time nlohmann::json OSUtils::jsonParse(const std::string &buf) { return nlohmann::json::parse(buf.c_str()); } -std::string OSUtils::jsonDump(const nlohmann::json &j) { return j.dump(1); } +std::string OSUtils::jsonDump(const nlohmann::json &j,int indentation) { return j.dump(indentation); } uint64_t OSUtils::jsonInt(const nlohmann::json &jv,const uint64_t dfl) { diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index d6f32822..8683ba25 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -276,7 +276,7 @@ public: static std::string platformDefaultHomePath(); static nlohmann::json jsonParse(const std::string &buf); - static std::string jsonDump(const nlohmann::json &j); + static std::string jsonDump(const nlohmann::json &j,int indentation = 1); static uint64_t jsonInt(const nlohmann::json &jv,const uint64_t dfl); static bool jsonBool(const nlohmann::json &jv,const bool dfl); static std::string jsonString(const nlohmann::json &jv,const char *dfl); diff --git a/service/OneService.cpp b/service/OneService.cpp index 115830e5..27f2ef3c 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -629,6 +629,14 @@ public: } } + // Allow controller DB path to be put somewhere else + json &settings = _localConfig["settings"]; + if (settings.is_object()) { + const std::string cdbp(OSUtils::jsonString(settings["controllerDbPath"],"")); + if (cdbp.length() > 0) + _controllerDbPath = cdbp; + } + // Set trusted paths if there are any if (trustedPathCount) _node->setTrustedPaths(reinterpret_cast(trustedPathNetworks),trustedPathIds,trustedPathCount); @@ -1484,27 +1492,6 @@ public: _allowManagementFrom.push_back(nw); } } - - json &controllerDbHttpHost = settings["controllerDbHttpHost"]; - json &controllerDbHttpPort = settings["controllerDbHttpPort"]; - json &controllerDbHttpPath = settings["controllerDbHttpPath"]; - if ((controllerDbHttpHost.is_string())&&(controllerDbHttpPort.is_number())) { - _controllerDbPath = "http://"; - std::string h = controllerDbHttpHost; - _controllerDbPath.append(h); - char dbp[128]; - OSUtils::ztsnprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort); - _controllerDbPath.push_back(':'); - _controllerDbPath.append(dbp); - if (controllerDbHttpPath.is_string()) { - std::string p = controllerDbHttpPath; - if ((p.length() == 0)||(p[0] != '/')) - _controllerDbPath.push_back('/'); - _controllerDbPath.append(p); - } else { - _controllerDbPath.push_back('/'); - } - } } // Checks if a managed IP or route target is allowed -- cgit v1.2.3 From 66feaeb51907514c3ccab3305d44ca5c69e7000a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 19 Jul 2017 15:06:23 -0700 Subject: . --- controller/EmbeddedNetworkController.cpp | 2 +- controller/JSONDB.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 70691fe0..1d03f67c 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1183,7 +1183,7 @@ void EmbeddedNetworkController::threadMain() auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); if (ms != _memberStatus.end()) lrt = ms->second.lastRequestTime; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx\",%llu,%llu", + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx\",\"%.10llx\",%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index ea295688..4b6824c2 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -120,6 +120,7 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) #ifndef __WINDOWS__ if (obj.length() > 0) { Mutex::Lock _l(_rawLock); + //fprintf(stderr,"%s\n",obj.c_str()); if ((long)write(_rawOutput,obj.data(),obj.length()) == (long)obj.length()) { if (write(_rawOutput,"\n",1) == 1) return true; -- cgit v1.2.3 From e4823381c66917746abb0f8d13281d61daa2f112 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 19 Jul 2017 15:16:15 -0700 Subject: . --- controller/EmbeddedNetworkController.cpp | 7 +++++-- include/ZeroTierOne.h | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 1d03f67c..72d47622 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1075,7 +1075,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) { static volatile unsigned long idCounter = 0; - char id[128]; + char id[128],tmp[128]; std::string k,v; try { @@ -1116,9 +1116,12 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } - OSUtils::ztsnprintf(id,sizeof(id),"%.10llx-%.10llx-%.16llx-%.8lx",_signingId.address().toInt(),rt.origin,OSUtils::now(),++idCounter); + const uint64_t now = OSUtils::now(); + OSUtils::ztsnprintf(id,sizeof(id),"%.10llx-%.10llx-%.16llx-%.8lx",_signingId.address().toInt(),rt.origin,now,++idCounter); d["id"] = id; d["objtype"] = "trace"; + d["ts"] = now; + d["nodeId"] = Utils::hex10(rt.origin,tmp); bool accept = true; /* diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e325b369..e6ec8090 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -259,6 +259,7 @@ extern "C" { // Fields in remote trace dictionaries #define ZT_REMOTE_TRACE_FIELD__EVENT "event" +#define ZT_REMOTE_TRACE_FIELD__NODE_ID "nodeId" #define ZT_REMOTE_TRACE_FIELD__PACKET_ID "packetId" #define ZT_REMOTE_TRACE_FIELD__PACKET_VERB "packetVerb" #define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_ID "packetTrustedPathId" -- cgit v1.2.3 From 2c682b4d1cdfd64d3a5b931bd0a67abb1f8b731e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 9 Aug 2017 14:37:19 -0700 Subject: Small controller revisions, first run of controller API model JavaScript. --- controller/EmbeddedNetworkController.cpp | 101 ++---- controller/EmbeddedNetworkController.hpp | 5 +- controller/controller-api-model.js | 546 +++++++++++++++++++++++++++++++ 3 files changed, 577 insertions(+), 75 deletions(-) create mode 100644 controller/controller-api-model.js (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 72d47622..764b5c20 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -638,14 +638,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (newAuth != OSUtils::jsonBool(member["authorized"],false)) { member["authorized"] = newAuth; member[((newAuth) ? "lastAuthorizedTime" : "lastDeauthorizedTime")] = now; - - json ah; - ah["a"] = newAuth; - ah["by"] = "api"; - ah["ts"] = now; - ah["ct"] = json(); - ah["c"] = json(); - member["authHistory"].push_back(ah); + if (newAuth) { + member["lastAuthorizedCredentialType"] = "api"; + member["lastAuthorizedCredential"] = json(); + } // Member is being de-authorized, so spray Revocation objects to all online members if (!newAuth) { @@ -896,22 +892,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("authTokens")) { json &authTokens = b["authTokens"]; - if (authTokens.is_array()) { - json nat = json::array(); - for(unsigned long i=0;i 0) { - json t = json::object(); - t["token"] = tstr; - t["expires"] = OSUtils::jsonInt(token["expires"],0ULL); - t["maxUsesPerMember"] = OSUtils::jsonInt(token["maxUsesPerMember"],0ULL); - nat.push_back(t); - } - } + if (authTokens.is_object()) { + json nat; + for(json::iterator t(authTokens.begin());t!=authTokens.end();++t) { + if ((t.value().is_number())&&(t.value() >= 0)) + nat[t.key()] = t.value(); } network["authTokens"] = nat; + } else { + network["authTokens"] = {{}}; } } @@ -1268,56 +1257,29 @@ void EmbeddedNetworkController::_request( } // Determine whether and how member is authorized - const char *authorizedBy = (const char *)0; + bool authorized = false; bool autoAuthorized = false; json autoAuthCredentialType,autoAuthCredential; if (OSUtils::jsonBool(member["authorized"],false)) { - authorizedBy = "memberIsAuthorized"; + authorized = true; } else if (!OSUtils::jsonBool(network["private"],true)) { - authorizedBy = "networkIsPublic"; - json &ahist = member["authHistory"]; - if ((!ahist.is_array())||(ahist.size() == 0)) - autoAuthorized = true; + authorized = true; + autoAuthorized = true; + autoAuthCredentialType = "public"; } else { char presentedAuth[512]; if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) { presentedAuth[511] = (char)0; // sanity check - - // Check for bearer token presented by member if ((strlen(presentedAuth) > 6)&&(!strncmp(presentedAuth,"token:",6))) { const char *const presentedToken = presentedAuth + 6; - - json &authTokens = network["authTokens"]; - if (authTokens.is_array()) { - for(unsigned long i=0;i now))&&(tstr == presentedToken)) { - bool usable = (maxUses == 0); - if (!usable) { - uint64_t useCount = 0; - json &ahist = member["authHistory"]; - if (ahist.is_array()) { - for(unsigned long j=0;j now)) { + authorized = true; + autoAuthorized = true; + autoAuthCredentialType = "token"; + autoAuthCredential = presentedToken; } } } @@ -1325,23 +1287,16 @@ void EmbeddedNetworkController::_request( } // If we auto-authorized, update member record - if ((autoAuthorized)&&(authorizedBy)) { + if ((autoAuthorized)&&(authorized)) { member["authorized"] = true; member["lastAuthorizedTime"] = now; - - json ah; - ah["a"] = true; - ah["by"] = authorizedBy; - ah["ts"] = now; - ah["ct"] = autoAuthCredentialType; - ah["c"] = autoAuthCredential; - member["authHistory"].push_back(ah); - + member["lastAuthorizedCredentialType"] = autoAuthCredentialType; + member["lastAuthorizedCredential"] = autoAuthCredential; json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } - if (authorizedBy) { + if (authorized) { // Update version info and meta-data if authorized and if this is a genuine request if (requestPacketId) { const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 8752922e..590a8b48 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -129,7 +129,6 @@ private: inline void _initMember(nlohmann::json &member) { if (!member.count("authorized")) member["authorized"] = false; - if (!member.count("authHistory")) member["authHistory"] = nlohmann::json::array(); if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); if (!member.count("activeBridge")) member["activeBridge"] = false; if (!member.count("tags")) member["tags"] = nlohmann::json::array(); @@ -139,6 +138,8 @@ private: if (!member.count("revision")) member["revision"] = 0ULL; if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL; if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL; + if (!member.count("lastAuthorizedCredentialType")) member["lastAuthorizedCredentialType"] = nlohmann::json(); + if (!member.count("lastAuthorizedCredential")) member["lastAuthorizedCredential"] = nlohmann::json(); if (!member.count("vMajor")) member["vMajor"] = -1; if (!member.count("vMinor")) member["vMinor"] = -1; if (!member.count("vRev")) member["vRev"] = -1; @@ -156,7 +157,7 @@ private: if (!network.count("enableBroadcast")) network["enableBroadcast"] = true; if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; - if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); + if (!network.count("authTokens")) network["authTokens"] = {{}}; if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); if (!network.count("tags")) network["tags"] = nlohmann::json::array(); if (!network.count("routes")) network["routes"] = nlohmann::json::array(); diff --git a/controller/controller-api-model.js b/controller/controller-api-model.js new file mode 100644 index 00000000..7b61dff4 --- /dev/null +++ b/controller/controller-api-model.js @@ -0,0 +1,546 @@ +/* + * A JavaScript class based model for the ZeroTier controller microservice API + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +'use strict'; + +/** + * Goes through a rule set array and makes sure it's valid, returning a canonicalized version + * + * @param {array[object]} rules Array of ZeroTier rules + * @return New array of canonicalized rules + * @throws {Error} Rule set is invalid + */ +function formatRuleSetArray(rules) +{ +} +exports.formatRuleSetArray = formatRuleSetArray; + +/** + * @param {string} IP with optional /netmask|port section + * @return 4, 6, or 0 if invalid + */ +function ipClassify(ip) +{ + if ((!ip)||(typeof ip !== 'string')) + return 0; + let ips = ip.split('/'); + if (ips.length > 0) { + if (ips.length > 1) { + if (ips[1].length === 0) + return 0; + for(let i=0;i 0) { + for(let i=0;i 0) { + for(let i=0;i= 0) { + r += c; + if (r.length === l) + break; + } + } + + while (r.length < l) + r = '0' + r; + + return r; +}; +exports.formatZeroTierIdentifier = formatZeroTierIdentifier; + +// Internal container classes +class _V4AssignMode +{ + get zt() { return (this._zt)||false; } + set zt(b) { this._zt = !!b; } + toJSON() + { + return { zt: this.zt }; + } +}; +class _v6AssignMode +{ + get ['6plane'] { return (this._6plane)||false; } + set ['6plane'](b) { this._6plane = !!b; } + get zt() { return (this._zt)||false; } + set zt(b) { this._zt = !!b; } + get rfc4193() { return (this._rfc4193)||false; } + set rfc4193(b) { this._rfc4193 = !!b; } + toJSON() + { + return { + zt: this.zt, + rfc4193: this.rfc4193, + '6plane': this['6plane'] + }; + } +} + +class Network +{ + constructor(obj) + { + this.clear(); + this.patch(obj); + } + + get objtype() { return 'network'; } + + get id() { return this._id; } + set id(x) { return (this._id = formatZeroTierIdentifier(x,16)); } + + get nwid() { return this._id; } // legacy + + get authTokens() { return this._authTokens; } + set authTokens(at) + { + this._authTokens = {}; + if ((at)&&(typeof at === 'object')&&(!Array.isArray(at))) { + for(let k in at) { + let exp = parseInt(at[k])||0; + if (exp >= 0) + this._authTokens[k] = exp; + } + } + return this._authTokens; + } + + get capabilities() { return this._capabilities; } + set capabilities(c) + { + let ca = []; + let ids = {}; + if ((c)&&(Array.isArray(c))) { + for(let a=0;a= 0)&&(capId <= 0xffffffff)&&(!ids[capId])) { + ids[capId] = true; + let capDefault = !!cap['default']; + let capRules = formatRuleSetArray(cap.rules); + ca.push({ + id: capId, + 'default': capDefault, + rules: capRules + }); + } + } + } + } + ca.sort(function(a,b) { + a = a.id; + b = b.id; + return ((a > b) ? 1 : ((a < b) ? -1 : 0)); + }); + this._capabilities = ca; + return ca; + } + + get ipAssignmentPools() return { this._ipAssignmentPools; } + set ipAssignmentPools(ipp) + { + let pa = []; + let ranges = {}; + if ((ipp)&&(Array.isArray(ipp))) { + for(let a=0;a 0)&&(stype === ipClassify(end))&&(!ranges[start+'_'+end])) { + ranges[start+'_'+end] = true; + pa.push({ ipRangeStart: start, ipRangeEnd: end }); + } + } + } + } + } + pa.sort(function(a,b) { return a.ipRangeStart.localeCompare(b.ipRangeStart); }); + this._ipAssignmentPools = pa; + return pa; + } + + get multicastLimit() return { this._multicastLimit; } + set multicastLimit(n) + { + try { + let nn = parseInt(n)||0; + this._multicastLimit = (nn >= 0) ? nn : 0; + } catch (e) { + this._multicastLimit = 0; + } + return this._multicastLimit; + } + + get routes() return { this._routes; } + set routes(r) + { + let ra = []; + let targets = {}; + if ((r)&&(Array.isArray(r))) { + for(let a=0;a 0)&&((routeVia === null)||(ipClassify(routeVia) === rtt))&&(!targets[routeTarget])) { + targets[routeTarget] = true; + ra.push({ target: routeTarget, via: routeVia }); + } + } + } + } + ra.sort(function(a,b) { return a.routeTarget.localeCompare(b.routeTarget); }); + this._routes = ra; + return ra; + } + + get tags() return { this._tags; } + set tags(t) + { + let ta = []; + if ((t)&&(Array.isArray(t))) { + for(let a=0;a= 0)||(tagId <= 0xffffffff)) { + let tagDefault = tag.default; + if (typeof tagDefault !== 'number') + tagDefault = parseInt(tagDefault)||null; + if ((tagDefault < 0)||(tagDefault > 0xffffffff)) + tagDefault = null; + ta.push({ 'id': tagId, 'default': tagDefault }); + } + } + } + } + ta.sort(function(a,b) { + a = a.id; + b = b.id; + return ((a > b) ? 1 : ((a < b) ? -1 : 0)); + }); + this._tags = ta; + return ta; + } + + get v4AssignMode() return { this._v4AssignMode; } + set v4AssignMode(m) + { + if ((m)&&(typeof m === 'object')&&(!Array.isArray(m))) { + this._v4AssignMode.zt = m.zt; + } else if (m === 'zt') { // legacy + this._v4AssignMode.zt = true; + } else { + this._v4AssignMode.zt = false; + } + } + + get v6AssignMode() return { this._v6AssignMode; } + set v6AssignMode(m) + { + if ((m)&&(typeof m === 'object')&&(!Array.isArray(m))) { + this._v6AssignMode.zt = m.zt; + this._v6AssignMode.rfc4193 = m.rfc4193; + this._v6AssignMode['6plane'] = m['6plane']; + } else if (typeof m === 'string') { // legacy + let ms = m.split(','); + this._v6AssignMode.zt = false; + this._v6AssignMode.rfc4193 = false; + this._v6AssignMode['6plane'] = false; + for(let i=0;i= 10000) mtu = 10000; // maximum as per ZT spec + this._mtu = mtu; + } + + get name() { return this._name; } + set name(n) + { + if (typeof n === 'string') + this._name = n; + else if (typeof n === 'number') + this._name = n.toString(); + else this._name = ''; + } + + get private() { return this._private; } + set private(b) + { + // This is really meaningful for security, so make true unless explicitly set to false. + this._private = (b !== false); + } + + get activeMemberCount() { return this.__activeMemberCount; } + get authorizedMemberCount() { return this.__authorizedMemberCount; } + get totalMemberCount() { return this.__totalMemberCount; } + get clock() { return this.__clock; } + get creationTime() { return this.__creationTime; } + get revision() { return this.__revision; } + + toJSONExcludeControllerGenerated() + { + return { + id: this.id, + objtype: 'network', + nwid: this.nwid, + authTokens: this.authTokens, + capabilities: this.capabilities, + ipAssignmentPools: this.ipAssignmentPools, + multicastLimit: this.multicastLimit, + routes: this.routes, + tags: this.tags, + v4AssignMode: this._v4AssignMode.toJSON(), + v6AssignMode: this._v6AssignMode.toJSON(), + rules: this.rules, + enableBroadcast: this.enableBroadcast, + mtu: this.mtu, + name: this.name, + 'private': this['private'] + }; + } + + toJSON() + { + var j = this.toJSONExcludeControllerGenerated(); + j.activeMemberCount = this.activeMemberCount; + j.authorizedMemberCount = this.authorizedMemberCount; + j.totalMemberCount = this.totalMemberCount; + j.clock = this.clock; + j.creationTime = this.creationTime; + j.revision = this.revision; + return j; + } + + clear() + { + this._id = ''; + this._authTokens = {}; + this._capabilities = []; + this._ipAssignmentPools = []; + this._multicastLimit = 32; + this._routes = []; + this._tags = []; + this._v4AssignMode = new _V4AssignMode(); + this._v6AssignMode = new _v6AssignMode(); + this._rules = []; + this._enableBroadcast = true; + this._mtu = 2800; + this._name = ''; + this._private = true; + + this.__activeMemberCount = 0; + this.__authorizedMemberCount = 0; + this.__totalMemberCount = 0; + this.__clock = 0; + this.__creationTime = 0; + this.__revision = 0; + } + + patch(obj) + { + if (obj instanceof Network) + obj = obj.toJSON(); + if ((obj)&&(typeof obj === 'object')&&(!Array.isArray(obj))) { + for(var k in obj) { + try { + switch(k) { + case 'id': + case 'authTokens': + case 'capabilities': + case 'ipAssignmentPools': + case 'multicastLimit': + case 'routes': + case 'tags': + case 'rules': + case 'enableBroadcast': + case 'mtu': + case 'name': + case 'private': + case 'v4AssignMode': + case 'v6AssignMode': + this[k] = obj[k]; + break; + + case 'activeMemberCount': + case 'authorizedMemberCount': + case 'totalMemberCount': + case 'clock': + case 'creationTime': + case 'revision': + this['__'+k] = parseInt(obj[k])||0; + break; + } + } catch (e) {} + } + } + } +}; +exports.Network = Network; + +class Member +{ + constructor(obj) + { + this.clear(); + this.patch(obj); + } + + get objtype() { return 'member'; } + + get id() { return this._id; } + set id(x) { this._id = formatZeroTierIdentifier((typeof x === 'number') ? x.toString(16) : x,10); } + + get address() { return this._id; } // legacy + + get nwid() { return this._nwid; } + set nwid(x) { this._nwid = formatZeroTierIdentifier(x,16); } + + get controllerId() { return this.nwid.substr(0,10); } + + get authorized() { return this._authorized; } + set authorized(b) { this._authorized = (b === true); } // security critical so require explicit set to true + + get activeBridge() { return this._activeBridge; } + set activeBridge(b) { this._activeBridge = !!b; } + + get capabilities() { return this._capabilities; } + set capabilities(c) + { + } + + get identity() { return this._identity; } + set identity(istr) + { + if ((istr)&&(typeof istr === 'string')) + this._identity = istr; + else this._identity = null; + } + + get ipAssignments() { return this._ipAssignments; } + set ipAssignments(ipa) + { + } + + get noAutoAssignIps() { return this._noAutoAssignIps; } + set noAutoAssignIps(b) { this._noAutoAssignIps = !!b; } + + get tags() { return this._tags; } + set tags(t) + { + } + + clear() + { + this._id = ''; + this._nwid = ''; + this._authorized = false; + this._activeBridge = false; + this._capabilities = []; + this._identity = ''; + this._ipAssignments = []; + this._noAutoAssignIps = false; + this._tags = []; + + this.__creationTime = 0; + this.__lastAuthorizedTime = 0; + this.__lastAuthorizedCredentialType = null; + this.__lastAuthorizedCredential = null; + this.__lastDeauthorizedTime = 0; + this.__physicalAddr = ''; + this.__revision = 0; + this.__vMajor = 0; + this.__vMinor = 0; + this.__vRev = 0; + this.__vProto = 0; + } +}; +exports.Member = Member; -- cgit v1.2.3 From 1c04cc0485c58f4f84e75d090025ead2c0f8202d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 9 Aug 2017 17:42:35 -0700 Subject: . --- controller/controller-api-model.js | 286 ++++++++++++++++++++++++++++++++++--- 1 file changed, 267 insertions(+), 19 deletions(-) (limited to 'controller') diff --git a/controller/controller-api-model.js b/controller/controller-api-model.js index 7b61dff4..fbc808c0 100644 --- a/controller/controller-api-model.js +++ b/controller/controller-api-model.js @@ -26,18 +26,6 @@ 'use strict'; -/** - * Goes through a rule set array and makes sure it's valid, returning a canonicalized version - * - * @param {array[object]} rules Array of ZeroTier rules - * @return New array of canonicalized rules - * @throws {Error} Rule set is invalid - */ -function formatRuleSetArray(rules) -{ -} -exports.formatRuleSetArray = formatRuleSetArray; - /** * @param {string} IP with optional /netmask|port section * @return 4, 6, or 0 if invalid @@ -103,6 +91,135 @@ function formatZeroTierIdentifier(x,l) }; exports.formatZeroTierIdentifier = formatZeroTierIdentifier; +/** + * Goes through a rule set array and makes sure it's valid, returning a canonicalized version + * + * @param {array[object]} rules Array of ZeroTier rules + * @return New array of canonicalized rules + * @throws {Error} Rule set is invalid + */ +function formatRuleSetArray(rules) +{ + let r = []; + if ((rules)&&(Array.isArray(rules))) { + for(let a=0;a= 0)&&(capId <= 0xffffffff)&&(!caps[capId])) { + caps[capId] = true; + ca.push(capId); + } + } + } + ca.sort(); + this._capabilities = ca; + return ca; } get identity() { return this._identity; } @@ -508,6 +639,17 @@ class Member get ipAssignments() { return this._ipAssignments; } set ipAssignments(ipa) { + let ips = {}; + if ((ipa)&&(Array.isArray(ipa))) { + for(let a=0;a 0) + ips[ip] = true; + } + } + this._ipAssignments = Object.keys(ips); + this._ipAssignments.sort(); + return this._ipAssignments; } get noAutoAssignIps() { return this._noAutoAssignIps; } @@ -516,6 +658,73 @@ class Member get tags() { return this._tags; } set tags(t) { + let ta = []; + let pairs = {}; + if ((t)&&(Array.isArray(t))) { + for(let a=0;a= 0)&&(tagId <= 0xffffffff)&&(tagValue >= 0)&&(tagValue <= 0xffffffff)&&(!pairs[pk])) { + pairs[pk] = true; + ta.push([ tagId,tagValue ]); + } + } + } + } + ta.sort(function(a,b) { + return ((a[0] < b[0]) ? -1 : ((a[0] > b[0]) ? 1 : 0)); + }); + this._tags = ta; + return ta; + } + + get creationTime() { return this.__creationTime; } + get lastAuthorizedTime() { return this.__lastAuthorizedTime; } + get lastAuthorizedCredentialType() { return this.__lastAuthorizedCredentialType; } + get lastAuthorizedCredential() { return this.__lastAuthorizedCredential; } + get lastDeauthorizedTime() { return this.__lastDeauthorizedTime; } + get physicalAddr() { return this.__physicalAddr; } + get revision() { return this.__revision; } + get vMajor() { return this.__vMajor; } + get vMinor() { return this.__vMinor; } + get vRev() { return this.__vRev; } + get vProto() { return this.__vProto; } + + toJSONExcludeControllerGenerated() + { + return { + id: this.id, + nwid: this.nwid, + objtype: 'member', + address: this.id, + authorized: this.authorized, + activeBridge: this.activeBridge, + capabilities: this.capabilities, + identity: this.identity, + ipAssignments: this.ipAssignments, + noAutoAssignIps: this.noAutoAssignIps, + tags: this.tags + }; + } + + toJSON() + { + let j = this.toJSONExcludeControllerGenerated(); + j.creationTime = this.creationTime; + j.lastAuthorizedTime = this.lastAuthorizedTime; + j.lastAuthorizedCredentialType = this.lastAuthorizedCredentialType; + j.lastAuthorizedCredential = this.lastAuthorizedCredential; + j.lastDeauthorizedTime = this.lastDeauthorizedTime; + j.physicalAddr = this.physicalAddr; + j.revision = this.revision; + j.vMajor = this.vMajor; + j.vMinor = this.vMinor; + j.vRev = this.vRev; + j.vProto = this.vProto; + return j; } clear() @@ -542,5 +751,44 @@ class Member this.__vRev = 0; this.__vProto = 0; } + + patch(obj) + { + if (obj instanceof Member) + obj = obj.toJSON(); + if ((obj)&&(typeof obj === 'object')&&(!Array.isArray(obj))) { + for(var k in obj) { + try { + switch(k) { + case 'id': + case 'nwid': + case 'authorized': + case 'activeBridge': + case 'capabilities': + case 'identity': + case 'ipAssignments': + case 'noAutoAssignIps': + case 'tags': + this[k] = obj[k]; + break; + + case 'creationTime': + case 'lastAuthorizedTime': + case 'lastAuthorizedCredentialType': + case 'lastAuthorizedCredential': + case 'lastDeauthorizedTime': + case 'physicalAddr': + case 'revision': + case 'vMajor': + case 'vMinor': + case 'vRev': + case 'vProto': + this['__'+k] = parseInt(obj[k])||0; + break; + } + } catch (e) {} + } + } + } }; exports.Member = Member; -- cgit v1.2.3 From 23fe8975e722ec0141be9c9bea9437b59e6088fa Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 14 Aug 2017 11:44:07 -0700 Subject: . --- controller/controller-api-model.js | 794 ------------------------------------- 1 file changed, 794 deletions(-) delete mode 100644 controller/controller-api-model.js (limited to 'controller') diff --git a/controller/controller-api-model.js b/controller/controller-api-model.js deleted file mode 100644 index fbc808c0..00000000 --- a/controller/controller-api-model.js +++ /dev/null @@ -1,794 +0,0 @@ -/* - * A JavaScript class based model for the ZeroTier controller microservice API - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -'use strict'; - -/** - * @param {string} IP with optional /netmask|port section - * @return 4, 6, or 0 if invalid - */ -function ipClassify(ip) -{ - if ((!ip)||(typeof ip !== 'string')) - return 0; - let ips = ip.split('/'); - if (ips.length > 0) { - if (ips.length > 1) { - if (ips[1].length === 0) - return 0; - for(let i=0;i 0) { - for(let i=0;i 0) { - for(let i=0;i= 0) { - r += c; - if (r.length === l) - break; - } - } - - while (r.length < l) - r = '0' + r; - - return r; -}; -exports.formatZeroTierIdentifier = formatZeroTierIdentifier; - -/** - * Goes through a rule set array and makes sure it's valid, returning a canonicalized version - * - * @param {array[object]} rules Array of ZeroTier rules - * @return New array of canonicalized rules - * @throws {Error} Rule set is invalid - */ -function formatRuleSetArray(rules) -{ - let r = []; - if ((rules)&&(Array.isArray(rules))) { - for(let a=0;a= 0) - this._authTokens[k] = exp; - } - } - return this._authTokens; - } - - get capabilities() { return this._capabilities; } - set capabilities(c) - { - let ca = []; - let ids = {}; - if ((c)&&(Array.isArray(c))) { - for(let a=0;a= 0)&&(capId <= 0xffffffff)&&(!ids[capId])) { - ids[capId] = true; - let capDefault = !!cap['default']; - let capRules = formatRuleSetArray(cap.rules); - ca.push({ - id: capId, - 'default': capDefault, - rules: capRules - }); - } - } - } - } - ca.sort(function(a,b) { - a = a.id; - b = b.id; - return ((a > b) ? 1 : ((a < b) ? -1 : 0)); - }); - this._capabilities = ca; - return ca; - } - - get ipAssignmentPools() { return this._ipAssignmentPools; } - set ipAssignmentPools(ipp) - { - let pa = []; - let ranges = {}; - if ((ipp)&&(Array.isArray(ipp))) { - for(let a=0;a 0)&&(stype === ipClassify(end))&&(!ranges[start+'_'+end])) { - ranges[start+'_'+end] = true; - pa.push({ ipRangeStart: start, ipRangeEnd: end }); - } - } - } - } - } - pa.sort(function(a,b) { return a.ipRangeStart.localeCompare(b.ipRangeStart); }); - this._ipAssignmentPools = pa; - return pa; - } - - get multicastLimit() { return this._multicastLimit; } - set multicastLimit(n) - { - try { - let nn = parseInt(n)||0; - this._multicastLimit = (nn >= 0) ? nn : 0; - } catch (e) { - this._multicastLimit = 0; - } - return this._multicastLimit; - } - - get routes() { return this._routes; } - set routes(r) - { - let ra = []; - let targets = {}; - if ((r)&&(Array.isArray(r))) { - for(let a=0;a 0)&&((routeVia === null)||(ipClassify(routeVia) === rtt))&&(!targets[routeTarget])) { - targets[routeTarget] = true; - ra.push({ target: routeTarget, via: routeVia }); - } - } - } - } - ra.sort(function(a,b) { return a.routeTarget.localeCompare(b.routeTarget); }); - this._routes = ra; - return ra; - } - - get tags() { return this._tags; } - set tags(t) - { - let ta = []; - if ((t)&&(Array.isArray(t))) { - for(let a=0;a= 0)||(tagId <= 0xffffffff)) { - let tagDefault = tag.default; - if (typeof tagDefault !== 'number') - tagDefault = parseInt(tagDefault)||null; - if ((tagDefault < 0)||(tagDefault > 0xffffffff)) - tagDefault = null; - ta.push({ 'id': tagId, 'default': tagDefault }); - } - } - } - } - ta.sort(function(a,b) { - a = a.id; - b = b.id; - return ((a > b) ? 1 : ((a < b) ? -1 : 0)); - }); - this._tags = ta; - return ta; - } - - get v4AssignMode() { return this._v4AssignMode; } - set v4AssignMode(m) - { - if ((m)&&(typeof m === 'object')&&(!Array.isArray(m))) { - this._v4AssignMode.zt = m.zt; - } else if (m === 'zt') { // legacy - this._v4AssignMode.zt = true; - } else { - this._v4AssignMode.zt = false; - } - } - - get v6AssignMode() { return this._v6AssignMode; } - set v6AssignMode(m) - { - if ((m)&&(typeof m === 'object')&&(!Array.isArray(m))) { - this._v6AssignMode.zt = m.zt; - this._v6AssignMode.rfc4193 = m.rfc4193; - this._v6AssignMode['6plane'] = m['6plane']; - } else if (typeof m === 'string') { // legacy - let ms = m.split(','); - this._v6AssignMode.zt = false; - this._v6AssignMode.rfc4193 = false; - this._v6AssignMode['6plane'] = false; - for(let i=0;i= 10000) mtu = 10000; // maximum as per ZT spec - this._mtu = mtu; - } - - get name() { return this._name; } - set name(n) - { - if (typeof n === 'string') - this._name = n; - else if (typeof n === 'number') - this._name = n.toString(); - else this._name = ''; - } - - get private() { return this._private; } - set private(b) - { - // This is really meaningful for security, so make true unless explicitly set to false. - this._private = (b !== false); - } - - get activeMemberCount() { return this.__activeMemberCount; } - get authorizedMemberCount() { return this.__authorizedMemberCount; } - get totalMemberCount() { return this.__totalMemberCount; } - get clock() { return this.__clock; } - get creationTime() { return this.__creationTime; } - get revision() { return this.__revision; } - - toJSONExcludeControllerGenerated() - { - return { - id: this.id, - objtype: 'network', - nwid: this.nwid, - authTokens: this.authTokens, - capabilities: this.capabilities, - ipAssignmentPools: this.ipAssignmentPools, - multicastLimit: this.multicastLimit, - routes: this.routes, - tags: this.tags, - v4AssignMode: this._v4AssignMode.toJSON(), - v6AssignMode: this._v6AssignMode.toJSON(), - rules: this.rules, - enableBroadcast: this.enableBroadcast, - mtu: this.mtu, - name: this.name, - 'private': this['private'] - }; - } - - toJSON() - { - var j = this.toJSONExcludeControllerGenerated(); - j.activeMemberCount = this.activeMemberCount; - j.authorizedMemberCount = this.authorizedMemberCount; - j.totalMemberCount = this.totalMemberCount; - j.clock = this.clock; - j.creationTime = this.creationTime; - j.revision = this.revision; - return j; - } - - clear() - { - this._id = ''; - this._authTokens = {}; - this._capabilities = []; - this._ipAssignmentPools = []; - this._multicastLimit = 32; - this._routes = []; - this._tags = []; - this._v4AssignMode = new _V4AssignMode(); - this._v6AssignMode = new _v6AssignMode(); - this._rules = []; - this._enableBroadcast = true; - this._mtu = 2800; - this._name = ''; - this._private = true; - - this.__activeMemberCount = 0; - this.__authorizedMemberCount = 0; - this.__totalMemberCount = 0; - this.__clock = 0; - this.__creationTime = 0; - this.__revision = 0; - } - - patch(obj) - { - if (obj instanceof Network) - obj = obj.toJSON(); - if ((obj)&&(typeof obj === 'object')&&(!Array.isArray(obj))) { - for(var k in obj) { - try { - switch(k) { - case 'id': - case 'authTokens': - case 'capabilities': - case 'ipAssignmentPools': - case 'multicastLimit': - case 'routes': - case 'tags': - case 'rules': - case 'enableBroadcast': - case 'mtu': - case 'name': - case 'private': - case 'v4AssignMode': - case 'v6AssignMode': - this[k] = obj[k]; - break; - - case 'activeMemberCount': - case 'authorizedMemberCount': - case 'totalMemberCount': - case 'clock': - case 'creationTime': - case 'revision': - this['__'+k] = parseInt(obj[k])||0; - break; - } - } catch (e) {} - } - } - } -}; -exports.Network = Network; - -class Member -{ - constructor(obj) - { - this.clear(); - this.patch(obj); - } - - get objtype() { return 'member'; } - - get id() { return this._id; } - set id(x) { this._id = formatZeroTierIdentifier((typeof x === 'number') ? x.toString(16) : x,10); } - - get address() { return this._id; } // legacy - - get nwid() { return this._nwid; } - set nwid(x) { this._nwid = formatZeroTierIdentifier(x,16); } - - get controllerId() { return this.nwid.substr(0,10); } - - get authorized() { return this._authorized; } - set authorized(b) { this._authorized = (b === true); } // security critical so require explicit set to true - - get activeBridge() { return this._activeBridge; } - set activeBridge(b) { this._activeBridge = !!b; } - - get capabilities() { return this._capabilities; } - set capabilities(c) - { - let caps = {}; - let ca = []; - if ((c)&&(Array.isArray(c))) { - for(let a=0;a= 0)&&(capId <= 0xffffffff)&&(!caps[capId])) { - caps[capId] = true; - ca.push(capId); - } - } - } - ca.sort(); - this._capabilities = ca; - return ca; - } - - get identity() { return this._identity; } - set identity(istr) - { - if ((istr)&&(typeof istr === 'string')) - this._identity = istr; - else this._identity = null; - } - - get ipAssignments() { return this._ipAssignments; } - set ipAssignments(ipa) - { - let ips = {}; - if ((ipa)&&(Array.isArray(ipa))) { - for(let a=0;a 0) - ips[ip] = true; - } - } - this._ipAssignments = Object.keys(ips); - this._ipAssignments.sort(); - return this._ipAssignments; - } - - get noAutoAssignIps() { return this._noAutoAssignIps; } - set noAutoAssignIps(b) { this._noAutoAssignIps = !!b; } - - get tags() { return this._tags; } - set tags(t) - { - let ta = []; - let pairs = {}; - if ((t)&&(Array.isArray(t))) { - for(let a=0;a= 0)&&(tagId <= 0xffffffff)&&(tagValue >= 0)&&(tagValue <= 0xffffffff)&&(!pairs[pk])) { - pairs[pk] = true; - ta.push([ tagId,tagValue ]); - } - } - } - } - ta.sort(function(a,b) { - return ((a[0] < b[0]) ? -1 : ((a[0] > b[0]) ? 1 : 0)); - }); - this._tags = ta; - return ta; - } - - get creationTime() { return this.__creationTime; } - get lastAuthorizedTime() { return this.__lastAuthorizedTime; } - get lastAuthorizedCredentialType() { return this.__lastAuthorizedCredentialType; } - get lastAuthorizedCredential() { return this.__lastAuthorizedCredential; } - get lastDeauthorizedTime() { return this.__lastDeauthorizedTime; } - get physicalAddr() { return this.__physicalAddr; } - get revision() { return this.__revision; } - get vMajor() { return this.__vMajor; } - get vMinor() { return this.__vMinor; } - get vRev() { return this.__vRev; } - get vProto() { return this.__vProto; } - - toJSONExcludeControllerGenerated() - { - return { - id: this.id, - nwid: this.nwid, - objtype: 'member', - address: this.id, - authorized: this.authorized, - activeBridge: this.activeBridge, - capabilities: this.capabilities, - identity: this.identity, - ipAssignments: this.ipAssignments, - noAutoAssignIps: this.noAutoAssignIps, - tags: this.tags - }; - } - - toJSON() - { - let j = this.toJSONExcludeControllerGenerated(); - j.creationTime = this.creationTime; - j.lastAuthorizedTime = this.lastAuthorizedTime; - j.lastAuthorizedCredentialType = this.lastAuthorizedCredentialType; - j.lastAuthorizedCredential = this.lastAuthorizedCredential; - j.lastDeauthorizedTime = this.lastDeauthorizedTime; - j.physicalAddr = this.physicalAddr; - j.revision = this.revision; - j.vMajor = this.vMajor; - j.vMinor = this.vMinor; - j.vRev = this.vRev; - j.vProto = this.vProto; - return j; - } - - clear() - { - this._id = ''; - this._nwid = ''; - this._authorized = false; - this._activeBridge = false; - this._capabilities = []; - this._identity = ''; - this._ipAssignments = []; - this._noAutoAssignIps = false; - this._tags = []; - - this.__creationTime = 0; - this.__lastAuthorizedTime = 0; - this.__lastAuthorizedCredentialType = null; - this.__lastAuthorizedCredential = null; - this.__lastDeauthorizedTime = 0; - this.__physicalAddr = ''; - this.__revision = 0; - this.__vMajor = 0; - this.__vMinor = 0; - this.__vRev = 0; - this.__vProto = 0; - } - - patch(obj) - { - if (obj instanceof Member) - obj = obj.toJSON(); - if ((obj)&&(typeof obj === 'object')&&(!Array.isArray(obj))) { - for(var k in obj) { - try { - switch(k) { - case 'id': - case 'nwid': - case 'authorized': - case 'activeBridge': - case 'capabilities': - case 'identity': - case 'ipAssignments': - case 'noAutoAssignIps': - case 'tags': - this[k] = obj[k]; - break; - - case 'creationTime': - case 'lastAuthorizedTime': - case 'lastAuthorizedCredentialType': - case 'lastAuthorizedCredential': - case 'lastDeauthorizedTime': - case 'physicalAddr': - case 'revision': - case 'vMajor': - case 'vMinor': - case 'vRev': - case 'vProto': - this['__'+k] = parseInt(obj[k])||0; - break; - } - } catch (e) {} - } - } - } -}; -exports.Member = Member; -- cgit v1.2.3 From 50e7ea088b16314c8ad9d10757204c966155f157 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 16 Aug 2017 14:14:49 -0700 Subject: More work on controller for new Central harnessed mode, remove old http mode. --- controller/EmbeddedNetworkController.cpp | 42 ++++--- controller/EmbeddedNetworkController.hpp | 4 + controller/JSONDB.cpp | 208 +++++++++++-------------------- controller/JSONDB.hpp | 7 +- 4 files changed, 106 insertions(+), 155 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 764b5c20..257fef57 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -35,6 +35,7 @@ #include #include "../include/ZeroTierOne.h" +#include "../version.h" #include "../node/Constants.hpp" #include "EmbeddedNetworkController.hpp" @@ -430,7 +431,7 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa _startTime(OSUtils::now()), _running(true), _lastDumpedStatus(0), - _db(dbPath), + _db(dbPath,this), _node(node) { } @@ -720,14 +721,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetworkMember(nwid,address,member); - - // Push update to member if online - try { - Mutex::Lock _l(_memberStatus_m); - _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,address)]; - if ((ms.online(now))&&(ms.lastRequestMetaData)) - request(nwid,InetAddress(),0,ms.identity,ms.lastRequestMetaData); - } catch ( ... ) {} } _addMemberNonPersistedFields(nwid,address,member,now); @@ -980,13 +973,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetwork(nwid,network); - - // Send an update to all members of the network that are online - Mutex::Lock _l(_memberStatus_m); - for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { - if ((i->first.networkId == nwid)&&(i->second.online(now))&&(i->second.lastRequestMetaData)) - request(nwid,InetAddress(),0,i->second.identity,i->second.lastRequestMetaData); - } } JSONDB::NetworkSummaryInfo ns; @@ -1144,6 +1130,28 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } +void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId) +{ + // Send an update to all members of the network that are online + const uint64_t now = OSUtils::now(); + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == networkId)&&(i->second.online(now))&&(i->second.lastRequestMetaData)) + request(networkId,InetAddress(),0,i->second.identity,i->second.lastRequestMetaData); + } +} + +void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId) +{ + // Push update to member if online + try { + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(networkId,memberId)]; + if ((ms.online(OSUtils::now()))&&(ms.lastRequestMetaData)) + request(networkId,InetAddress(),0,ms.identity,ms.lastRequestMetaData); + } catch ( ... ) {} +} + void EmbeddedNetworkController::threadMain() throw() { @@ -1184,7 +1192,7 @@ void EmbeddedNetworkController::threadMain() first = false; }); } - OSUtils::ztsnprintf(tmp,sizeof(tmp),"],\"clock\":%llu,\"startTime\":%llu,\"uptime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime,(unsigned long long)(now - _startTime)); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"],\"clock\":%llu,\"startTime\":%llu,\"uptime\":%llu,\"vMajor\":%d,\"vMinor\":%d,\"vRev\":%d}",(unsigned long long)now,(unsigned long long)_startTime,(unsigned long long)(now - _startTime),ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); st.append(tmp); _db.writeRaw("status",st); } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 590a8b48..6200e910 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -93,6 +93,10 @@ public: void handleRemoteTrace(const ZT_RemoteTrace &rt); + // Called by JSONDB when networks and network members are changed + void onNetworkUpdate(const uint64_t networkId); + void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId); + void threadMain() throw(); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 4b6824c2..a0dd50c2 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -29,75 +29,44 @@ #endif #include "JSONDB.hpp" - -#define ZT_JSONDB_HTTP_TIMEOUT 60000 +#include "EmbeddedNetworkController.hpp" namespace ZeroTier { static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); -static const std::map _ZT_JSONDB_GET_HEADERS; -JSONDB::JSONDB(const std::string &basePath) : +JSONDB::JSONDB(const std::string &basePath,EmbeddedNetworkController *parent) : + _parent(parent), _basePath(basePath), _rawInput(-1), _rawOutput(-1), _summaryThreadRun(true), _dataReady(false) { - if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { - // If base path is http:// we run in HTTP mode - // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. - // Typically it's just used with 127.0.0.1 anyway. - std::string hn = _basePath.substr(7); - std::size_t hnend = hn.find_first_of('/'); - if (hnend != std::string::npos) - hn = hn.substr(0,hnend); - std::size_t hnsep = hn.find_last_of(':'); - if (hnsep != std::string::npos) - hn[hnsep] = '/'; - _httpAddr.fromString(hn.c_str()); - if (hnend != std::string::npos) - _basePath = _basePath.substr(7 + hnend); - if (_basePath.length() == 0) - _basePath = "/"; - if (_basePath[0] != '/') - _basePath = std::string("/") + _basePath; #ifndef __WINDOWS__ - } else if (_basePath == "-") { - // If base path is "-" we run in stdin/stdout mode and expect our database to be populated on startup via stdin - // Not supported on Windows + if (_basePath == "-") { + // If base path is "-" we run in Central harnessed mode. We read pseudo-http-requests from stdin and write + // them to stdout. _rawInput = STDIN_FILENO; _rawOutput = STDOUT_FILENO; fcntl(_rawInput,F_SETFL,O_NONBLOCK); -#endif } else { +#endif // Default mode of operation is to store files in the filesystem OSUtils::mkdir(_basePath.c_str()); OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions +#ifndef __WINDOWS__ } +#endif _networks_m.lock(); // locked until data is loaded, etc. if (_rawInput < 0) { - unsigned int cnt = 0; - while (!_load(_basePath)) { - if ((++cnt & 7) == 0) - fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); - Thread::sleep(250); - } - - for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) - _summaryThreadToDo.push_back(n->first); - - if (_summaryThreadToDo.size() > 0) { - _summaryThread = Thread::start(this); - } else { - _dataReady = true; - _networks_m.unlock(); - } + _load(basePath); + _dataReady = true; + _networks_m.unlock(); } else { - // In IPC mode we wait for the first message to start, and we start - // this thread since this thread is responsible for reading from stdin. + // In harnessed mode we leave the lock locked and wait for our initial DB from Central. _summaryThread = Thread::start(this); } } @@ -128,16 +97,6 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) } else return true; #endif return false; - } else if (_httpAddr) { - std::map headers; - std::string body; - std::map reqHeaders; - char tmp[64]; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); - reqHeaders["Content-Length"] = tmp; - reqHeaders["Content-Type"] = "application/json"; - const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); - return (sc == 200); } else { const std::string path(_genPath(n,true)); if (!path.length()) @@ -205,10 +164,15 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC char n[64]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig,-1)); + bool update; { Mutex::Lock _l(_networks_m); - _networks[networkId].config = nlohmann::json::to_msgpack(networkConfig); + _NW &nw = _networks[networkId]; + update = !nw.config.empty(); + nw.config = nlohmann::json::to_msgpack(networkConfig); } + if (update) + _parent->onNetworkUpdate(networkId); _recomputeSummaryInfo(networkId); } @@ -217,17 +181,25 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co char n[256]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); writeRaw(n,OSUtils::jsonDump(memberConfig,-1)); + bool update; { Mutex::Lock _l(_networks_m); - _networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig); + std::vector &m = _networks[networkId].members[nodeId]; + update = !m.empty(); + m = nlohmann::json::to_msgpack(memberConfig); _members[nodeId].insert(networkId); } + if (update) + _parent->onNetworkMemberUpdate(networkId,nodeId); _recomputeSummaryInfo(networkId); } nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) { - if (!_httpAddr) { // Member deletion is done by Central in harnessed mode, and deleting the cache network entry also deletes all members + if (_rawOutput >= 0) { + // In harnessed mode, DB deletes occur in the Central database and we do + // not need to erase files. + } else { std::vector memberIds; { Mutex::Lock _l(_networks_m); @@ -239,24 +211,15 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) } for(std::vector::iterator m(memberIds.begin());m!=memberIds.end();++m) eraseNetworkMember(networkId,*m,false); - } - char n[256]; - OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); - - if (_rawOutput >= 0) { - // In harnessed mode, deletes occur in Central or other management - // software and do not need to be executed this way. - } else if (_httpAddr) { - std::map headers; - std::string body; - Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); - } else { + char n[256]; + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); const std::string path(_genPath(n,false)); if (path.length()) OSUtils::rm(path.c_str()); } + // This also erases all members from the memory cache { Mutex::Lock _l(_networks_m); std::unordered_map::iterator i(_networks.find(networkId)); @@ -270,17 +233,11 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo) { - char n[256]; - OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); - if (_rawOutput >= 0) { - // In harnessed mode, deletes occur in Central or other management - // software and do not need to be executed this way. - } else if (_httpAddr) { - std::map headers; - std::string body; - Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + // In harnessed mode, DB deletes occur in Central and we do not remove files. } else { + char n[256]; + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); const std::string path(_genPath(n,false)); if (path.length()) OSUtils::rm(path.c_str()); @@ -320,7 +277,6 @@ void JSONDB::threadMain() while (_summaryThreadRun) { #ifndef __WINDOWS__ if (_rawInput < 0) { - // In HTTP and filesystem mode we just wait for summary to-do items Thread::sleep(25); } else { // In IPC mode we wait but also select() on STDIN to read database updates @@ -337,8 +293,8 @@ void JSONDB::threadMain() } else if (rawInputBuf.length() > 0) { try { const nlohmann::json obj(OSUtils::jsonParse(rawInputBuf)); - gotMessage = true; + if (!_dataReady) { _dataReady = true; _networks_m.unlock(); @@ -351,6 +307,7 @@ void JSONDB::threadMain() _add(obj); } } catch ( ... ) {} // ignore malformed JSON + rawInputBuf.clear(); } } @@ -369,7 +326,7 @@ void JSONDB::threadMain() else _summaryThreadToDo.swap(todo); } - if (!_dataReady) { + if (!_dataReady) { // sanity check _dataReady = true; _networks_m.unlock(); } @@ -460,17 +417,33 @@ bool JSONDB::_add(const nlohmann::json &j) if ((id.length() == 16)&&(objtype == "network")) { const uint64_t nwid = Utils::hexStrToU64(id.c_str()); if (nwid) { - Mutex::Lock _l(_networks_m); - _networks[nwid].config = nlohmann::json::to_msgpack(j); + bool update; + { + Mutex::Lock _l(_networks_m); + _NW &nw = _networks[nwid]; + update = !nw.config.empty(); + nw.config = nlohmann::json::to_msgpack(j); + } + if (update) + _parent->onNetworkUpdate(nwid); + _recomputeSummaryInfo(nwid); return true; } } else if ((id.length() == 10)&&(objtype == "member")) { const uint64_t mid = Utils::hexStrToU64(id.c_str()); const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); if ((mid)&&(nwid)) { - Mutex::Lock _l(_networks_m); - _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); - _members[mid].insert(nwid); + bool update; + { + Mutex::Lock _l(_networks_m); + std::vector &m = _networks[nwid].members[mid]; + update = !m.empty(); + m = nlohmann::json::to_msgpack(j); + _members[mid].insert(nwid); + } + if (update) + _parent->onNetworkMemberUpdate(nwid,mid); + _recomputeSummaryInfo(nwid); return true; } } @@ -484,48 +457,21 @@ bool JSONDB::_load(const std::string &p) // This is not used in stdin/stdout mode. Instead data is populated by // sending it all to stdin. - if (_httpAddr) { - // In HTTP harnessed mode we download our entire working data set on startup. - - std::string body; - std::map headers; - const unsigned int sc = Http::GET(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); - if (sc == 200) { - try { - nlohmann::json dbImg(OSUtils::jsonParse(body)); - std::string tmp; - if (dbImg.is_object()) { - Mutex::Lock _l(_networks_m); - for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { - try { - _add(i.value()); - } catch ( ... ) {} - } - return true; - } - } catch ( ... ) {} // invalid JSON, so maybe incomplete request - } - return false; - - } else { - // In regular mode we recursively read it from controller.d/ on disk - - std::vector dl(OSUtils::listDirectory(p.c_str(),true)); - for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { - if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { - std::string buf; - if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) { - try { - _add(OSUtils::jsonParse(buf)); - } catch ( ... ) {} - } - } else { - this->_load((p + ZT_PATH_SEPARATOR_S + *di)); + std::vector dl(OSUtils::listDirectory(p.c_str(),true)); + for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { + if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { + std::string buf; + if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) { + try { + _add(OSUtils::jsonParse(buf)); + } catch ( ... ) {} } + } else { + this->_load((p + ZT_PATH_SEPARATOR_S + *di)); } - return true; - } + + return true; } void JSONDB::_recomputeSummaryInfo(const uint64_t networkId) @@ -543,23 +489,15 @@ std::string JSONDB::_genPath(const std::string &n,bool create) if (pt.size() == 0) return std::string(); - char sep; - if (_httpAddr) { - sep = '/'; - create = false; - } else { - sep = ZT_PATH_SEPARATOR; - } - std::string p(_basePath); if (create) OSUtils::mkdir(p.c_str()); for(unsigned long i=0,j=(unsigned long)(pt.size()-1);i Date: Wed, 16 Aug 2017 14:41:42 -0700 Subject: Another Central harnessed mode fix. --- controller/EmbeddedNetworkController.cpp | 2 ++ controller/EmbeddedNetworkController.hpp | 2 +- controller/JSONDB.cpp | 8 -------- 3 files changed, 3 insertions(+), 9 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 257fef57..3ca0f536 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -721,6 +721,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetworkMember(nwid,address,member); + onNetworkMemberUpdate(nwid,address); } _addMemberNonPersistedFields(nwid,address,member,now); @@ -973,6 +974,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetwork(nwid,network); + onNetworkUpdate(nwid); } JSONDB::NetworkSummaryInfo ns; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 6200e910..cbbe07ac 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -93,7 +93,7 @@ public: void handleRemoteTrace(const ZT_RemoteTrace &rt); - // Called by JSONDB when networks and network members are changed + // Called on update via POST or by JSONDB on external update of network or network member records void onNetworkUpdate(const uint64_t networkId); void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index a0dd50c2..9813239e 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -164,15 +164,11 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC char n[64]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig,-1)); - bool update; { Mutex::Lock _l(_networks_m); _NW &nw = _networks[networkId]; - update = !nw.config.empty(); nw.config = nlohmann::json::to_msgpack(networkConfig); } - if (update) - _parent->onNetworkUpdate(networkId); _recomputeSummaryInfo(networkId); } @@ -181,16 +177,12 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co char n[256]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); writeRaw(n,OSUtils::jsonDump(memberConfig,-1)); - bool update; { Mutex::Lock _l(_networks_m); std::vector &m = _networks[networkId].members[nodeId]; - update = !m.empty(); m = nlohmann::json::to_msgpack(memberConfig); _members[nodeId].insert(networkId); } - if (update) - _parent->onNetworkMemberUpdate(networkId,nodeId); _recomputeSummaryInfo(networkId); } -- cgit v1.2.3 From 174ba8884ee68c3a54776ce7fe3f8249aa934ac6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 17 Aug 2017 13:10:10 -0700 Subject: Delete support in harnessed mode. --- controller/EmbeddedNetworkController.cpp | 26 +++++++++++-------- controller/EmbeddedNetworkController.hpp | 1 + controller/JSONDB.cpp | 44 +++++++++++++++++++++++++------- controller/JSONDB.hpp | 2 +- 4 files changed, 53 insertions(+), 20 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3ca0f536..f5bfce4e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -645,16 +645,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } // Member is being de-authorized, so spray Revocation objects to all online members - if (!newAuth) { - Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); - rev.sign(_signingId); - - Mutex::Lock _l(_memberStatus_m); - for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { - if ((i->first.networkId == nwid)&&(i->second.online(now))) - _node->ncSendRevocation(Address(i->first.nodeId),rev); - } - } + if (!newAuth) + onNetworkMemberDeauthorize(nwid,address); } } @@ -1154,6 +1146,20 @@ void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,c } catch ( ... ) {} } +void EmbeddedNetworkController::onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId) +{ + const uint64_t now = OSUtils::now(); + Revocation rev((uint32_t)_node->prng(),networkId,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(memberId),Revocation::CREDENTIAL_TYPE_COM); + rev.sign(_signingId); + { + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == networkId)&&(i->second.online(now))) + _node->ncSendRevocation(Address(i->first.nodeId),rev); + } + } +} + void EmbeddedNetworkController::threadMain() throw() { diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index cbbe07ac..d1217d60 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -96,6 +96,7 @@ public: // Called on update via POST or by JSONDB on external update of network or network member records void onNetworkUpdate(const uint64_t networkId); void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId); + void onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId); void threadMain() throw(); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 9813239e..f362acf3 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -294,9 +294,9 @@ void JSONDB::threadMain() if (obj.is_array()) { for(unsigned long i=0;i &m = _networks[nwid].members[mid]; - update = !m.empty(); + if (!m.empty()) { + update = true; + nlohmann::json oldm(nlohmann::json::from_msgpack(m)); + deauth = ((OSUtils::jsonBool(oldm["authorized"],false))&&(!OSUtils::jsonBool(j["authorized"],false))); + } m = nlohmann::json::to_msgpack(j); _members[mid].insert(nwid); } - if (update) + if (update) { _parent->onNetworkMemberUpdate(nwid,mid); + if (deauth) + _parent->onNetworkMemberDeauthorize(nwid,mid); + } _recomputeSummaryInfo(nwid); return true; } + + } else if (objtype == "_delete") { // pseudo-object-type, only used in Central harnessed mode + + const std::string deleteType(OSUtils::jsonString(j["deleteType"],"")); + id = OSUtils::jsonString(j["deleteId"],""); + if ((deleteType == "network")&&(id.length() == 16)) { + eraseNetwork(Utils::hexStrToU64(id.c_str())); + } else if ((deleteType == "member")&&(id.length() == 10)) { + const std::string networkId(OSUtils::jsonString(j["deleteNetworkId"],"")); + const uint64_t nwid = Utils::hexStrToU64(networkId.c_str()); + const uint64_t mid = Utils::hexStrToU64(id.c_str()); + if (networkId.length() == 16) + eraseNetworkMember(nwid,mid,true); + _parent->onNetworkMemberDeauthorize(nwid,mid); + } + } } } catch ( ... ) {} @@ -455,7 +481,7 @@ bool JSONDB::_load(const std::string &p) std::string buf; if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) { try { - _add(OSUtils::jsonParse(buf)); + _addOrUpdate(OSUtils::jsonParse(buf)); } catch ( ... ) {} } } else { diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 66d0138a..44f4d7f5 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -157,7 +157,7 @@ public: throw(); private: - bool _add(const nlohmann::json &j); + bool _addOrUpdate(const nlohmann::json &j); bool _load(const std::string &p); void _recomputeSummaryInfo(const uint64_t networkId); std::string _genPath(const std::string &n,bool create); -- cgit v1.2.3 From 106dff0d5312ec7908dade36d537fdf85538413c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Aug 2017 13:52:10 -0700 Subject: Make remote trace target null by default, which is probably what we want. --- controller/EmbeddedNetworkController.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index f5bfce4e..0f342fa5 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1391,7 +1391,7 @@ void EmbeddedNetworkController::_request( if (rtt.length() == 10) { nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str())); } else { - nc->remoteTraceTarget = _signingId.address(); + nc->remoteTraceTarget.zero(); } } -- cgit v1.2.3 From 5bf5d5e9cbf799987897fd3ea07e212c1ff1ab6c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 30 Aug 2017 17:22:25 -0700 Subject: Minor controller stuff. --- controller/EmbeddedNetworkController.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0f342fa5..8d5febd9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1242,7 +1242,7 @@ void EmbeddedNetworkController::_request( _initMember(member); { - std::string haveIdStr(OSUtils::jsonString(member["identity"],"")); + const std::string haveIdStr(OSUtils::jsonString(member["identity"],"")); if (haveIdStr.length() > 0) { // If we already know this member's identity perform a full compare. This prevents // a "collision" from being able to auth onto our network in place of an already @@ -1308,8 +1308,6 @@ void EmbeddedNetworkController::_request( member["lastAuthorizedTime"] = now; member["lastAuthorizedCredentialType"] = autoAuthCredentialType; member["lastAuthorizedCredential"] = autoAuthCredential; - json &revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } if (authorized) { @@ -1338,6 +1336,7 @@ void EmbeddedNetworkController::_request( if (fromAddr) ms.physicalAddr = fromAddr; + char tmpip[64]; if (ms.physicalAddr) member["physicalAddr"] = ms.physicalAddr.toString(tmpip); @@ -1346,8 +1345,11 @@ void EmbeddedNetworkController::_request( } else { // If they are not authorized, STOP! _removeMemberNonPersistedFields(member); - if (origMember != member) + if (origMember != member) { + json &revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetworkMember(nwid,identity.address().toInt(),member); + } _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); return; } @@ -1710,8 +1712,11 @@ void EmbeddedNetworkController::_request( } _removeMemberNonPersistedFields(member); - if (member != origMember) + if (member != origMember) { + json &revj = member["revision"]; + member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetworkMember(nwid,identity.address().toInt(),member); + } _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } -- cgit v1.2.3 From 283e8d5bc00c13f821c67ed7a431af4bd7694113 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 31 Aug 2017 18:01:21 -0400 Subject: Start threads in Central harnessed mode. --- controller/EmbeddedNetworkController.cpp | 16 ++++++++++++++++ controller/EmbeddedNetworkController.hpp | 15 +-------------- 2 files changed, 17 insertions(+), 14 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8d5febd9..1d46d5e6 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -434,6 +434,8 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa _db(dbPath,this), _node(node) { + if ((dbPath[0] == '-')&&(dbPath[1] == 0)) + _startThreads(); // start threads now in Central harnessed mode } EmbeddedNetworkController::~EmbeddedNetworkController() @@ -1721,4 +1723,18 @@ void EmbeddedNetworkController::_request( _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } +void EmbeddedNetworkController::_startThreads() +{ + Mutex::Lock _l(_threads_m); + if (_threads.size() == 0) { + long hwc = (long)std::thread::hardware_concurrency(); + if (hwc < 1) + hwc = 1; + else if (hwc > 16) + hwc = 16; + for(long i=0;i &metaData); - - inline void _startThreads() - { - Mutex::Lock _l(_threads_m); - if (_threads.size() == 0) { - long hwc = (long)std::thread::hardware_concurrency(); - if (hwc < 1) - hwc = 1; - else if (hwc > 16) - hwc = 16; - for(long i=0;i Date: Mon, 2 Oct 2017 15:52:57 -0700 Subject: timestamps changed from uint64_t to int64_t There were cases in the code where time calculations and comparisons were overflowing and causing connection instability. This will keep time calculations within expected ranges. --- controller/EmbeddedNetworkController.cpp | 14 +++--- controller/EmbeddedNetworkController.hpp | 8 ++-- controller/JSONDB.cpp | 4 +- controller/JSONDB.hpp | 2 +- include/ZeroTierOne.h | 12 ++--- node/CertificateOfMembership.hpp | 2 +- node/IncomingPacket.cpp | 12 ++--- node/IncomingPacket.hpp | 4 +- node/Membership.cpp | 8 ++-- node/Membership.hpp | 14 +++--- node/Multicaster.cpp | 8 ++-- node/Multicaster.hpp | 12 ++--- node/Network.cpp | 10 ++--- node/Network.hpp | 4 +- node/Node.cpp | 34 +++++++------- node/Node.hpp | 26 +++++------ node/OutboundMulticast.hpp | 2 +- node/Path.cpp | 2 +- node/Path.hpp | 22 ++++----- node/Peer.cpp | 35 ++++++++------- node/Peer.hpp | 76 ++++++++++++++++---------------- node/Revocation.hpp | 4 +- node/SelfAwareness.cpp | 6 +-- node/SelfAwareness.hpp | 4 +- node/Switch.cpp | 20 ++++----- node/Switch.hpp | 14 +++--- node/Topology.cpp | 7 +-- node/Topology.hpp | 4 +- one.cpp | 6 +-- osdep/OSUtils.hpp | 6 +-- osdep/PortMapper.cpp | 2 +- service/OneService.cpp | 22 +++++---- service/SoftwareUpdater.cpp | 2 +- service/SoftwareUpdater.hpp | 2 +- 34 files changed, 209 insertions(+), 201 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 1d46d5e6..20f81966 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -535,7 +535,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } else { // Get network - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); JSONDB::NetworkSummaryInfo ns; _db.getNetworkSummaryInfo(nwid,ns); _addNetworkNonPersistedFields(network,now,ns); @@ -602,7 +602,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( responseContentType = "application/json"; return 400; } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); if (path[0] == "network") { @@ -1087,7 +1087,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); OSUtils::ztsnprintf(id,sizeof(id),"%.10llx-%.10llx-%.16llx-%.8lx",_signingId.address().toInt(),rt.origin,now,++idCounter); d["id"] = id; d["objtype"] = "trace"; @@ -1129,7 +1129,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId) { // Send an update to all members of the network that are online - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); Mutex::Lock _l(_memberStatus_m); for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { if ((i->first.networkId == networkId)&&(i->second.online(now))&&(i->second.lastRequestMetaData)) @@ -1150,7 +1150,7 @@ void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,c void EmbeddedNetworkController::onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId) { - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); Revocation rev((uint32_t)_node->prng(),networkId,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(memberId),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); { @@ -1224,7 +1224,7 @@ void EmbeddedNetworkController::_request( if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) return; - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); if (requestPacketId) { Mutex::Lock _l(_memberStatus_m); @@ -1360,7 +1360,7 @@ void EmbeddedNetworkController::_request( // If we made it this far, they are authorized. // ------------------------------------------------------------------------- - uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; + int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; if (now > ns.mostRecentDeauthTime) { // If we recently de-authorized a member, shrink credential TTL/max delta to // be below the threshold required to exclude it. Cap this to a min/max to diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 19469164..fce56065 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -166,7 +166,7 @@ private: } network["objtype"] = "network"; } - inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const JSONDB::NetworkSummaryInfo &ns) + inline void _addNetworkNonPersistedFields(nlohmann::json &network,int64_t now,const JSONDB::NetworkSummaryInfo &ns) { network["clock"] = now; network["authorizedMemberCount"] = ns.authorizedMemberCount; @@ -182,7 +182,7 @@ private: // legacy fields network.erase("lastModified"); } - inline void _addMemberNonPersistedFields(uint64_t nwid,uint64_t nodeId,nlohmann::json &member,uint64_t now) + inline void _addMemberNonPersistedFields(uint64_t nwid,uint64_t nodeId,nlohmann::json &member,int64_t now) { member["clock"] = now; Mutex::Lock _l(_memberStatus_m); @@ -197,7 +197,7 @@ private: member.erase("lastRequestMetaData"); } - const uint64_t _startTime; + const int64_t _startTime; volatile bool _running; BlockingQueue<_RQEntry *> _queue; @@ -230,7 +230,7 @@ private: Dictionary lastRequestMetaData; Identity identity; InetAddress physicalAddr; // last known physical address - inline bool online(const uint64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); } + inline bool online(const int64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); } }; struct _MemberStatusHash { diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index f362acf3..67b13393 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -323,7 +323,7 @@ void JSONDB::threadMain() _networks_m.unlock(); } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); try { Mutex::Lock _l(_networks_m); for(std::vector::iterator ii(todo.begin());ii!=todo.end();++ii) { @@ -373,7 +373,7 @@ void JSONDB::threadMain() } catch ( ... ) {} } else { try { - ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,OSUtils::jsonInt(member["lastDeauthorizedTime"],0ULL)); + ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,(int64_t)OSUtils::jsonInt(member["lastDeauthorizedTime"],0LL)); } catch ( ... ) {} } ++ns.totalMemberCount; diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 44f4d7f5..db909cb0 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -57,7 +57,7 @@ public: unsigned long authorizedMemberCount; unsigned long activeMemberCount; unsigned long totalMemberCount; - uint64_t mostRecentDeauthTime; + int64_t mostRecentDeauthTime; }; JSONDB(const std::string &basePath,EmbeddedNetworkController *parent); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index cf6b21fd..8adbc4d1 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1587,7 +1587,7 @@ struct ZT_Node_Callbacks * @param now Current clock in milliseconds * @return OK (0) or error code if a fatal error condition has occurred */ -ZT_SDK_API enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); +ZT_SDK_API enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now); /** * Delete a node and free all resources it consumes @@ -1615,12 +1615,12 @@ ZT_SDK_API void ZT_Node_delete(ZT_Node *node); ZT_SDK_API enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); /** * Process a frame from a virtual network port (tap) @@ -1641,7 +1641,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processWirePacket( ZT_SDK_API enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -1649,7 +1649,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); /** * Perform periodic background operations @@ -1660,7 +1660,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( * @param nextBackgroundTaskDeadline Value/result: set to deadline for next call to processBackgroundTasks() * @return OK (0) or error code if a fatal error condition has occurred */ -ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); +ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline); /** * Join a network diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 3ffa814f..0105fade 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -176,7 +176,7 @@ public: /** * @return Timestamp for this cert and maximum delta for timestamp */ - inline uint64_t timestamp() const + inline int64_t timestamp() const { for(unsigned int i=0;i<_qualifierCount;++i) { if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 685f2f09..c0409c91 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -169,7 +169,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar // Peers can send this in response to frames if they do not have a recent enough COM from us networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); const SharedPtr network(RR->node->network(networkId)); - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) network->pushCredentialsNow(tPtr,peer->address(),now); } break; @@ -202,7 +202,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); const uint64_t pid = packetId(); const Address fromAddress(source()); @@ -210,7 +210,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION]; const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION]; const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO_IDX_REVISION); - const uint64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); + const int64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); Identity id; unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); @@ -725,7 +725,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); uint64_t authOnNetwork[256]; // cache for approved network IDs unsigned int authOnNetworkCount = 0; @@ -1082,7 +1082,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { @@ -1186,7 +1186,7 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); if (peer->rateGateOutgoingComRequest(now)) { Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR); outp.append((uint8_t)verb()); diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 45a0166d..c8f52721 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -77,7 +77,7 @@ public: * @param now Current time * @throws std::out_of_range Range error processing packet */ - IncomingPacket(const void *data,unsigned int len,const SharedPtr &path,uint64_t now) : + IncomingPacket(const void *data,unsigned int len,const SharedPtr &path,int64_t now) : Packet(data,len), _receiveTime(now), _path(path) @@ -93,7 +93,7 @@ public: * @param now Current time * @throws std::out_of_range Range error processing packet */ - inline void init(const void *data,unsigned int len,const SharedPtr &path,uint64_t now) + inline void init(const void *data,unsigned int len,const SharedPtr &path,int64_t now) { copyFrom(data,len); _receiveTime = now; diff --git a/node/Membership.cpp b/node/Membership.cpp index 17de6554..740f4e68 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -51,7 +51,7 @@ Membership::Membership() : resetPushState(); } -void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) +void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) { bool sendCom = ( (nconf.com) && ( ((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); @@ -127,13 +127,13 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com) { - const uint64_t newts = com.timestamp(); + const int64_t newts = com.timestamp(); if (newts <= _comRevocationThreshold) { RR->t->credentialRejected(tPtr,com,"revoked"); return ADD_REJECTED; } - const uint64_t oldts = _com.timestamp(); + const int64_t oldts = _com.timestamp(); if (newts < oldts) { RR->t->credentialRejected(tPtr,com,"old"); return ADD_REJECTED; @@ -227,7 +227,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -void Membership::clean(const uint64_t now,const NetworkConfig &nconf) +void Membership::clean(const int64_t now,const NetworkConfig &nconf) { _cleanCredImpl(nconf,_remoteTags); _cleanCredImpl(nconf,_remoteCaps); diff --git a/node/Membership.hpp b/node/Membership.hpp index c6e2b803..5612858a 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -80,7 +80,7 @@ public: * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none * @param force If true, send objects regardless of last push time */ - void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); + void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** * Check whether we should push MULTICAST_LIKEs to this peer, and update last sent time if true @@ -88,7 +88,7 @@ public: * @param now Current time * @return True if we should update multicasts */ - inline bool multicastLikeGate(const uint64_t now) + inline bool multicastLikeGate(const int64_t now) { if ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD) { _lastUpdatedMulticast = now; @@ -110,7 +110,7 @@ public: return nconf.com.agreesWith(_com); } - inline bool recentlyAssociated(const uint64_t now) const + inline bool recentlyAssociated(const int64_t now) const { return ((_com)&&((now - _com.timestamp()) < ZT_PEER_ACTIVITY_TIMEOUT)); } @@ -180,7 +180,7 @@ public: * @param now Current time * @param nconf Current network configuration */ - void clean(const uint64_t now,const NetworkConfig &nconf); + void clean(const int64_t now,const NetworkConfig &nconf); /** * Reset last pushed time for local credentials @@ -223,13 +223,13 @@ private: } // Last time we pushed MULTICAST_LIKE(s) - uint64_t _lastUpdatedMulticast; + int64_t _lastUpdatedMulticast; // Last time we pushed our COM to this peer - uint64_t _lastPushedCom; + int64_t _lastPushedCom; // Revocation threshold for COM or 0 if none - uint64_t _comRevocationThreshold; + int64_t _comRevocationThreshold; // Remote member's latest network COM CertificateOfMembership _com; diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index e8c8613a..fa6f7bd1 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -51,7 +51,7 @@ Multicaster::~Multicaster() { } -void Multicaster::addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) +void Multicaster::addMultiple(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) { const unsigned char *p = (const unsigned char *)addresses; const unsigned char *e = p + (5 * count); @@ -160,7 +160,7 @@ std::vector
Multicaster::getMembers(uint64_t nwid,const MulticastGroup void Multicaster::send( void *tPtr, unsigned int limit, - uint64_t now, + int64_t now, uint64_t nwid, bool disableCompression, const std::vector
&alwaysSendTo, @@ -309,7 +309,7 @@ void Multicaster::send( delete [] indexes; } -void Multicaster::clean(uint64_t now) +void Multicaster::clean(int64_t now) { { Mutex::Lock _l(_groups_m); @@ -367,7 +367,7 @@ void Multicaster::addCredential(void *tPtr,const CertificateOfMembership &com,bo } } -void Multicaster::_add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) +void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) { // assumes _groups_m is locked diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 69a6645d..08c96485 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -98,7 +98,7 @@ public: * @param mg Multicast group * @param member New member address */ - inline void add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) + inline void add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) { Mutex::Lock _l(_groups_m); _add(tPtr,now,nwid,mg,_groups[Multicaster::Key(nwid,mg)],member); @@ -117,7 +117,7 @@ public: * @param count Number of addresses * @param totalKnown Total number of known addresses as reported by peer */ - void addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); + void addMultiple(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); /** * Remove a multicast group member (if present) @@ -174,7 +174,7 @@ public: void send( void *tPtr, unsigned int limit, - uint64_t now, + int64_t now, uint64_t nwid, bool disableCompression, const std::vector
&alwaysSendTo, @@ -190,7 +190,7 @@ public: * @param RR Runtime environment * @param now Current time */ - void clean(uint64_t now); + void clean(int64_t now); /** * Add an authorization credential @@ -212,7 +212,7 @@ public: * @param now Current time * @return True if GATHER and LIKE should be allowed */ - bool cacheAuthorized(const Address &a,const uint64_t nwid,const uint64_t now) const + bool cacheAuthorized(const Address &a,const uint64_t nwid,const int64_t now) const { Mutex::Lock _l(_gatherAuth_m); const uint64_t *p = _gatherAuth.get(_GatherAuthKey(nwid,a)); @@ -220,7 +220,7 @@ public: } private: - void _add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); + void _add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); const RuntimeEnvironment *RR; diff --git a/node/Network.cpp b/node/Network.cpp index 16155c33..111e4736 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -592,7 +592,7 @@ bool Network::filterOutgoingPacket( const unsigned int etherType, const unsigned int vlanId) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); Address ztFinalDest(ztDest); int localCapabilityIndex = -1; int accept = 0; @@ -1164,7 +1164,7 @@ void Network::requestConfiguration(void *tPtr) bool Network::gate(void *tPtr,const SharedPtr &peer) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); Mutex::Lock _l(_lock); try { if (_config) { @@ -1192,7 +1192,7 @@ bool Network::recentlyAssociatedWith(const Address &addr) void Network::clean() { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); Mutex::Lock _l(_lock); if (_destroyed) @@ -1257,7 +1257,7 @@ void Network::learnBridgeRoute(const MAC &mac,const Address &addr) } } -void Network::learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now) +void Network::learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,int64_t now) { Mutex::Lock _l(_lock); const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); @@ -1377,7 +1377,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup) { // Assumes _lock is locked - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); std::vector groups; if (newMulticastGroup) diff --git a/node/Network.hpp b/node/Network.hpp index d4d217f2..1b4da7d2 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -305,7 +305,7 @@ public: * @param mg Multicast group * @param now Current time */ - void learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now); + void learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,int64_t now); /** * Validate a credential and learn it if it passes certificate and other checks @@ -357,7 +357,7 @@ public: * @param to Destination peer address * @param now Current time */ - inline void pushCredentialsNow(void *tPtr,const Address &to,const uint64_t now) + inline void pushCredentialsNow(void *tPtr,const Address &to,const int64_t now) { Mutex::Lock _l(_lock); _membership(to).pushCredentials(RR,tPtr,now,to,_config,-1,true); diff --git a/node/Node.cpp b/node/Node.cpp index cc076e4d..31ee8f19 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -54,7 +54,7 @@ namespace ZeroTier { /* Public Node interface (C++, exposed via CAPI bindings) */ /****************************************************************************/ -Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : +Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now) : _RR(this), RR(&_RR), _uPtr(uptr), @@ -139,12 +139,12 @@ Node::~Node() ZT_ResultCode Node::processWirePacket( void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { _now = now; RR->sw->onRemotePacket(tptr,localSocket,*(reinterpret_cast(remoteAddress)),packetData,packetLength); @@ -153,7 +153,7 @@ ZT_ResultCode Node::processWirePacket( ZT_ResultCode Node::processVirtualNetworkFrame( void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -161,7 +161,7 @@ ZT_ResultCode Node::processVirtualNetworkFrame( unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { _now = now; SharedPtr nw(this->network(nwid)); @@ -175,7 +175,7 @@ ZT_ResultCode Node::processVirtualNetworkFrame( class _PingPeersThatNeedPing { public: - _PingPeersThatNeedPing(const RuntimeEnvironment *renv,void *tPtr,Hashtable< Address,std::vector > &upstreamsToContact,uint64_t now) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,void *tPtr,Hashtable< Address,std::vector > &upstreamsToContact,int64_t now) : lastReceiveFromUpstream(0), RR(renv), _tPtr(tPtr), @@ -185,7 +185,7 @@ public: { } - uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay + int64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay inline void operator()(Topology &t,const SharedPtr &p) { @@ -234,17 +234,17 @@ private: const RuntimeEnvironment *RR; void *_tPtr; Hashtable< Address,std::vector > &_upstreamsToContact; - const uint64_t _now; + const int64_t _now; const SharedPtr _bestCurrentUpstream; }; -ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline) { _now = now; Mutex::Lock bl(_backgroundTasksLock); unsigned long timeUntilNextPingCheck = ZT_PING_CHECK_INVERVAL; - const uint64_t timeSinceLastPingCheck = now - _lastPingCheck; + const int64_t timeSinceLastPingCheck = now - _lastPingCheck; if (timeSinceLastPingCheck >= ZT_PING_CHECK_INVERVAL) { try { _lastPingCheck = now; @@ -305,7 +305,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint } try { - *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); + *nextBackgroundTaskDeadline = now + (int64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } @@ -689,7 +689,7 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des extern "C" { -enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now) { *node = (ZT_Node *)0; try { @@ -714,12 +714,12 @@ void ZT_Node_delete(ZT_Node *node) enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { try { return reinterpret_cast(node)->processWirePacket(tptr,now,localSocket,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); @@ -733,7 +733,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -741,7 +741,7 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { try { return reinterpret_cast(node)->processVirtualNetworkFrame(tptr,now,nwid,sourceMac,destMac,etherType,vlanId,frameData,frameLength,nextBackgroundTaskDeadline); @@ -752,7 +752,7 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( } } -enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline) { try { return reinterpret_cast(node)->processBackgroundTasks(tptr,now,nextBackgroundTaskDeadline); diff --git a/node/Node.hpp b/node/Node.hpp index 1aa01c9a..ae7976d4 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -64,7 +64,7 @@ class World; class Node : public NetworkController::Sender { public: - Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); + Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now); virtual ~Node(); // Get rid of alignment warnings on 32-bit Windows and possibly improve performance @@ -77,15 +77,15 @@ public: ZT_ResultCode processWirePacket( void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); ZT_ResultCode processVirtualNetworkFrame( void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -93,8 +93,8 @@ public: unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline); - ZT_ResultCode processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); + ZT_ResultCode processBackgroundTasks(void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline); ZT_ResultCode join(uint64_t nwid,void *uptr,void *tptr); ZT_ResultCode leave(uint64_t nwid,void **uptr,void *tptr); ZT_ResultCode multicastSubscribe(void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); @@ -114,7 +114,7 @@ public: // Internal functions ------------------------------------------------------ - inline uint64_t now() const { return _now; } + inline int64_t now() const { return _now; } inline bool putPacket(void *tPtr,const int64_t localSocket,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) { @@ -243,7 +243,7 @@ public: * @param from Source address of packet * @return True if within rate limits */ - inline bool rateGateIdentityVerification(const uint64_t now,const InetAddress &from) + inline bool rateGateIdentityVerification(const int64_t now,const InetAddress &from) { unsigned long iph = from.rateGateHash(); if ((now - _lastIdentityVerification[iph]) >= ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT) { @@ -270,7 +270,7 @@ private: uint32_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1]; // Time of last identity verification indexed by InetAddress.rateGateHash() -- used in IncomingPacket::_doHELLO() via rateGateIdentityVerification() - uint64_t _lastIdentityVerification[16384]; + int64_t _lastIdentityVerification[16384]; Hashtable< uint64_t,SharedPtr > _networks; Mutex _networks_m; @@ -281,10 +281,10 @@ private: Mutex _backgroundTasksLock; Address _remoteTraceTarget; - uint64_t _now; - uint64_t _lastPingCheck; - uint64_t _lastHousekeepingRun; - volatile uint64_t _prngState[2]; + int64_t _now; + int64_t _lastPingCheck; + int64_t _lastHousekeepingRun; + volatile int64_t _prngState[2]; bool _online; }; diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 486b66ff..2f6d8338 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -96,7 +96,7 @@ public: * @param now Current time * @return True if this multicast is expired (has exceeded transmit timeout) */ - inline bool expired(uint64_t now) const { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); } + inline bool expired(int64_t now) const { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); } /** * @return True if this outbound multicast has been sent to enough peers diff --git a/node/Path.cpp b/node/Path.cpp index 9dc9aba5..ca366e39 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -30,7 +30,7 @@ namespace ZeroTier { -bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now) +bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,int64_t now) { if (RR->node->putPacket(tPtr,_localSocket,_addr,data,len)) { _lastOut = now; diff --git a/node/Path.hpp b/node/Path.hpp index ac8e4c0e..050fb6e2 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -179,14 +179,14 @@ public: * @param now Current time * @return True if transport reported success */ - bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now); + bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,int64_t now); /** * Manually update last sent time * * @param t Time of send */ - inline void sent(const uint64_t t) { _lastOut = t; } + inline void sent(const int64_t t) { _lastOut = t; } /** * @return Local socket as specified by external code @@ -206,7 +206,7 @@ public: /** * @return True if path has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms */ - inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } + inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } /** * @return Preference rank, higher == better @@ -261,27 +261,27 @@ public: /** * @return True if path appears alive */ - inline bool alive(const uint64_t now) const { return ((now - _lastIn) <= ZT_PATH_ALIVE_TIMEOUT); } + inline bool alive(const int64_t now) const { return ((now - _lastIn) <= ZT_PATH_ALIVE_TIMEOUT); } /** * @return True if this path needs a heartbeat */ - inline bool needsHeartbeat(const uint64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); } + inline bool needsHeartbeat(const int64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); } /** * @return Last time we sent something */ - inline uint64_t lastOut() const { return _lastOut; } + inline int64_t lastOut() const { return _lastOut; } /** * @return Last time we received anything */ - inline uint64_t lastIn() const { return _lastIn; } + inline int64_t lastIn() const { return _lastIn; } /** * @return Time last trust-established packet was received */ - inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } + inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } /** * Return and increment outgoing packet counter (used with Packet::armor()) @@ -291,9 +291,9 @@ public: inline unsigned int nextOutgoingCounter() { return _outgoingPacketCounter++; } private: - volatile uint64_t _lastOut; - volatile uint64_t _lastIn; - volatile uint64_t _lastTrustEstablishedPacketReceived; + volatile int64_t _lastOut; + volatile int64_t _lastIn; + volatile int64_t _lastTrustEstablishedPacketReceived; volatile uint64_t _incomingLinkQualityFastLog; int64_t _localSocket; volatile unsigned long _incomingLinkQualitySlowLogPtr; diff --git a/node/Peer.cpp b/node/Peer.cpp index a954b716..60661592 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -34,6 +34,7 @@ #include "SelfAwareness.hpp" #include "Packet.hpp" #include "Trace.hpp" +#include "InetAddress.hpp" namespace ZeroTier { @@ -75,7 +76,7 @@ void Peer::received( const bool trustEstablished, const uint64_t networkId) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); /* #ifdef ZT_ENABLE_CLUSTER @@ -263,14 +264,14 @@ void Peer::received( } } -bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) +bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,int64_t now,bool force) { Mutex::Lock _l(_paths_m); - uint64_t v6lr = 0; + int64_t v6lr = 0; if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) v6lr = _v6Path.p->lastIn(); - uint64_t v4lr = 0; + int64_t v4lr = 0; if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) v4lr = _v4Path.p->lastIn(); @@ -289,16 +290,18 @@ bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now, return false; } -SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) +SharedPtr Peer::getBestPath(int64_t now,bool includeExpired) { Mutex::Lock _l(_paths_m); - uint64_t v6lr = 0; - if ( ( includeExpired || ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v6Path.p) ) + int64_t v6lr = 0; + if ((includeExpired || ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)) && (_v6Path.p)) { v6lr = _v6Path.p->lastIn(); - uint64_t v4lr = 0; - if ( ( includeExpired || ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v4Path.p) ) + } + int64_t v4lr = 0; + if ((includeExpired || ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)) && (_v4Path.p)) { v4lr = _v4Path.p->lastIn(); + } if (v6lr > v4lr) { return _v6Path.p; @@ -309,7 +312,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) return SharedPtr(); } -void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,unsigned int counter) +void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,unsigned int counter) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); @@ -357,7 +360,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA } } -void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) +void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,bool sendFullHello,unsigned int counter) { if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); @@ -369,7 +372,7 @@ void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAdd } } -void Peer::tryMemorizedPath(void *tPtr,uint64_t now) +void Peer::tryMemorizedPath(void *tPtr,int64_t now) { if ((now - _lastTriedMemorizedPath) >= ZT_TRY_MEMORIZED_PATH_INTERVAL) { _lastTriedMemorizedPath = now; @@ -379,15 +382,15 @@ void Peer::tryMemorizedPath(void *tPtr,uint64_t now) } } -bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) +bool Peer::doPingAndKeepalive(void *tPtr,int64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); if (inetAddressFamily < 0) { - uint64_t v6lr = 0; + int64_t v6lr = 0; if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) v6lr = _v6Path.p->lastIn(); - uint64_t v4lr = 0; + int64_t v4lr = 0; if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) v4lr = _v4Path.p->lastIn(); @@ -423,7 +426,7 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) return false; } -void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const uint64_t now) +void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now) { if ((remoteAddress.ss_family != AF_INET)&&(remoteAddress.ss_family != AF_INET6)) // sanity check return; diff --git a/node/Peer.hpp b/node/Peer.hpp index af9163a5..e08f7d36 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -120,7 +120,7 @@ public: * @param addr Remote address * @return True if we have an active path to this destination */ - inline bool hasActivePathTo(uint64_t now,const InetAddress &addr) const + inline bool hasActivePathTo(int64_t now,const InetAddress &addr) const { Mutex::Lock _l(_paths_m); return ( ((addr.ss_family == AF_INET)&&(_v4Path.p)&&(_v4Path.p->address() == addr)&&(_v4Path.p->alive(now))) || ((addr.ss_family == AF_INET6)&&(_v6Path.p)&&(_v6Path.p->address() == addr)&&(_v6Path.p->alive(now))) ); @@ -136,7 +136,7 @@ public: * @param force If true, send even if path is not alive * @return True if we actually sent something */ - bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force); + bool sendDirect(void *tPtr,const void *data,unsigned int len,int64_t now,bool force); /** * Get the best current direct path @@ -148,7 +148,7 @@ public: * @param includeExpired If true, include even expired paths * @return Best current path or NULL if none */ - SharedPtr getBestPath(uint64_t now,bool includeExpired); + SharedPtr getBestPath(int64_t now,bool includeExpired); /** * Send a HELLO to this peer at a specified physical address @@ -161,7 +161,7 @@ public: * @param now Current time * @param counter Outgoing packet counter */ - void sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,unsigned int counter); + void sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,unsigned int counter); /** * Send ECHO (or HELLO for older peers) to this peer at the given address @@ -175,7 +175,7 @@ public: * @param sendFullHello If true, always send a full HELLO instead of just an ECHO * @param counter Outgoing packet counter */ - void attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); + void attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,bool sendFullHello,unsigned int counter); /** * Try a memorized or statically defined path if any are known @@ -185,7 +185,7 @@ public: * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time */ - void tryMemorizedPath(void *tPtr,uint64_t now); + void tryMemorizedPath(void *tPtr,int64_t now); /** * Send pings or keepalives depending on configured timeouts @@ -195,7 +195,7 @@ public: * @param inetAddressFamily Keep this address family alive, or -1 for any * @return True if we have at least one direct path of the given family (or any if family is -1) */ - bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); + bool doPingAndKeepalive(void *tPtr,int64_t now,int inetAddressFamily); /** * Specify remote path for this peer and forget others @@ -209,7 +209,7 @@ public: * @param remoteAddress Remote address * @param now Current time */ - void redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const uint64_t now); + void redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now); /** * Reset paths within a given IP scope and address family @@ -222,7 +222,7 @@ public: * @param inetAddressFamily Family e.g. AF_INET * @param now Current time */ - inline void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) + inline void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now) { Mutex::Lock _l(_paths_m); if ((inetAddressFamily == AF_INET)&&(_v4Path.lr)&&(_v4Path.p->address().ipScope() == scope)) { @@ -243,7 +243,7 @@ public: * @param v4 Result parameter to receive active IPv4 address, if any * @param v6 Result parameter to receive active IPv6 address, if any */ - inline void getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const + inline void getRendezvousAddresses(int64_t now,InetAddress &v4,InetAddress &v6) const { Mutex::Lock _l(_paths_m); if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) @@ -256,7 +256,7 @@ public: * @param now Current time * @return All known paths to this peer */ - inline std::vector< SharedPtr > paths(const uint64_t now) const + inline std::vector< SharedPtr > paths(const int64_t now) const { std::vector< SharedPtr > pp; Mutex::Lock _l(_paths_m); @@ -270,17 +270,17 @@ public: /** * @return Time of last receive of anything, whether direct or relayed */ - inline uint64_t lastReceive() const { return _lastReceive; } + inline int64_t lastReceive() const { return _lastReceive; } /** * @return True if we've heard from this peer in less than ZT_PEER_ACTIVITY_TIMEOUT */ - inline bool isAlive(const uint64_t now) const { return ((now - _lastReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline bool isAlive(const int64_t now) const { return ((now - _lastReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return True if this peer has sent us real network traffic recently */ - inline uint64_t isActive(uint64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline int64_t isActive(int64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return Latency in milliseconds or 0 if unknown @@ -298,7 +298,7 @@ public: * * @return Relay quality score computed from latency and other factors, lower is better */ - inline unsigned int relayQuality(const uint64_t now) const + inline unsigned int relayQuality(const int64_t now) const { const uint64_t tsr = now - _lastReceive; if (tsr >= ZT_PEER_ACTIVITY_TIMEOUT) @@ -353,12 +353,12 @@ public: /** * @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms */ - inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } + inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } /** * Rate limit gate for VERB_PUSH_DIRECT_PATHS */ - inline bool rateGatePushDirectPaths(const uint64_t now) + inline bool rateGatePushDirectPaths(const int64_t now) { if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME) ++_directPathPushCutoffCount; @@ -370,7 +370,7 @@ public: /** * Rate limit gate for VERB_NETWORK_CREDENTIALS */ - inline bool rateGateCredentialsReceived(const uint64_t now) + inline bool rateGateCredentialsReceived(const int64_t now) { if ((now - _lastCredentialsReceived) <= ZT_PEER_CREDENTIALS_CUTOFF_TIME) ++_credentialsCutoffCount; @@ -382,7 +382,7 @@ public: /** * Rate limit gate for sending of ERROR_NEED_MEMBERSHIP_CERTIFICATE */ - inline bool rateGateRequestCredentials(const uint64_t now) + inline bool rateGateRequestCredentials(const int64_t now) { if ((now - _lastCredentialRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastCredentialRequestSent = now; @@ -394,7 +394,7 @@ public: /** * Rate limit gate for inbound WHOIS requests */ - inline bool rateGateInboundWhoisRequest(const uint64_t now) + inline bool rateGateInboundWhoisRequest(const int64_t now) { if ((now - _lastWhoisRequestReceived) >= ZT_PEER_WHOIS_RATE_LIMIT) { _lastWhoisRequestReceived = now; @@ -406,7 +406,7 @@ public: /** * Rate limit gate for inbound ECHO requests */ - inline bool rateGateEchoRequest(const uint64_t now) + inline bool rateGateEchoRequest(const int64_t now) { if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastEchoRequestReceived = now; @@ -418,7 +418,7 @@ public: /** * Rate gate incoming requests for network COM */ - inline bool rateGateIncomingComRequest(const uint64_t now) + inline bool rateGateIncomingComRequest(const int64_t now) { if ((now - _lastComRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastComRequestReceived = now; @@ -430,7 +430,7 @@ public: /** * Rate gate outgoing requests for network COM */ - inline bool rateGateOutgoingComRequest(const uint64_t now) + inline bool rateGateOutgoingComRequest(const int64_t now) { if ((now - _lastComRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastComRequestSent = now; @@ -484,7 +484,7 @@ public: } template - inline static SharedPtr deserializeFromCache(uint64_t now,void *tPtr,Buffer &b,const RuntimeEnvironment *renv) + inline static SharedPtr deserializeFromCache(int64_t now,void *tPtr,Buffer &b,const RuntimeEnvironment *renv) { try { unsigned int ptr = 0; @@ -527,8 +527,8 @@ private: struct _PeerPath { _PeerPath() : lr(0),sticky(0),p() {} - uint64_t lr; // time of last valid ZeroTier packet - uint64_t sticky; // time last set as sticky + int64_t lr; // time of last valid ZeroTier packet + int64_t sticky; // time last set as sticky SharedPtr p; }; @@ -536,18 +536,18 @@ private: const RuntimeEnvironment *RR; - uint64_t _lastReceive; // direct or indirect - uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. - uint64_t _lastTriedMemorizedPath; - uint64_t _lastDirectPathPushSent; - uint64_t _lastDirectPathPushReceive; - uint64_t _lastCredentialRequestSent; - uint64_t _lastWhoisRequestReceived; - uint64_t _lastEchoRequestReceived; - uint64_t _lastComRequestReceived; - uint64_t _lastComRequestSent; - uint64_t _lastCredentialsReceived; - uint64_t _lastTrustEstablishedPacketReceived; + int64_t _lastReceive; // direct or indirect + int64_t _lastNontrivialReceive; // frames, things like netconf, etc. + int64_t _lastTriedMemorizedPath; + int64_t _lastDirectPathPushSent; + int64_t _lastDirectPathPushReceive; + int64_t _lastCredentialRequestSent; + int64_t _lastWhoisRequestReceived; + int64_t _lastEchoRequestReceived; + int64_t _lastComRequestReceived; + int64_t _lastComRequestSent; + int64_t _lastCredentialsReceived; + int64_t _lastTrustEstablishedPacketReceived; uint16_t _vProto; uint16_t _vMajor; diff --git a/node/Revocation.hpp b/node/Revocation.hpp index a28da0ab..7f7498bb 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -85,7 +85,7 @@ public: inline uint32_t id() const { return _id; } inline uint32_t credentialId() const { return _credentialId; } inline uint64_t networkId() const { return _networkId; } - inline uint64_t threshold() const { return _threshold; } + inline int64_t threshold() const { return _threshold; } inline const Address &target() const { return _target; } inline const Address &signer() const { return _signedBy; } inline Credential::Type type() const { return _type; } @@ -184,7 +184,7 @@ private: uint32_t _id; uint32_t _credentialId; uint64_t _networkId; - uint64_t _threshold; + int64_t _threshold; uint64_t _flags; Address _target; Address _signedBy; diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 0af0d691..83cd89c9 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -49,7 +49,7 @@ namespace ZeroTier { class _ResetWithinScope { public: - _ResetWithinScope(void *tPtr,uint64_t now,int inetAddressFamily,InetAddress::IpScope scope) : + _ResetWithinScope(void *tPtr,int64_t now,int inetAddressFamily,InetAddress::IpScope scope) : _now(now), _tPtr(tPtr), _family(inetAddressFamily), @@ -70,7 +70,7 @@ SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) : { } -void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) +void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,int64_t now) { const InetAddress::IpScope scope = myPhysicalAddress.ipScope(); @@ -112,7 +112,7 @@ void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receive } } -void SelfAwareness::clean(uint64_t now) +void SelfAwareness::clean(int64_t now) { Mutex::Lock _l(_phy_m); Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index 35e0ad39..7ddba465 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -55,14 +55,14 @@ public: * @param trusted True if this peer is trusted as an authority to inform us of external address changes * @param now Current time */ - void iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); + void iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,int64_t now); /** * Clean up database periodically * * @param now Current time */ - void clean(uint64_t now); + void clean(int64_t now); /** * If we appear to be behind a symmetric NAT, get predictions for possible external endpoints diff --git a/node/Switch.cpp b/node/Switch.cpp index f46b3e73..cc022b6b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -57,7 +57,7 @@ Switch::Switch(const RuntimeEnvironment *renv) : void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddress &fromAddr,const void *data,unsigned int len) { try { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); const SharedPtr path(RR->topology->getPath(localSocket,fromAddr)); path->received(now); @@ -557,14 +557,14 @@ void Switch::send(void *tPtr,Packet &packet,bool encrypt) } } -void Switch::requestWhois(void *tPtr,const uint64_t now,const Address &addr) +void Switch::requestWhois(void *tPtr,const int64_t now,const Address &addr) { if (addr == RR->identity.address()) return; { Mutex::Lock _l(_lastSentWhoisRequest_m); - uint64_t &last = _lastSentWhoisRequest[addr]; + int64_t &last = _lastSentWhoisRequest[addr]; if ((now - last) < ZT_WHOIS_RETRY_DELAY) return; else last = now; @@ -586,7 +586,7 @@ void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) _lastSentWhoisRequest.erase(peer->address()); } - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); for(unsigned int ptr=0;ptrtimestamp)&&(rq->complete)) { @@ -611,7 +611,7 @@ void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) } } -unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) +unsigned long Switch::doTimerTasks(void *tPtr,int64_t now) { const uint64_t timeSinceLastCheck = now - _lastCheckedQueues; if (timeSinceLastCheck < ZT_WHOIS_RETRY_DELAY) @@ -663,9 +663,9 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) { Mutex::Lock _l(_lastSentWhoisRequest_m); - Hashtable< Address,uint64_t >::Iterator i(_lastSentWhoisRequest); + Hashtable< Address,int64_t >::Iterator i(_lastSentWhoisRequest); Address *a = (Address *)0; - uint64_t *ts = (uint64_t *)0; + int64_t *ts = (int64_t *)0; while (i.next(a,ts)) { if ((now - *ts) > (ZT_WHOIS_RETRY_DELAY * 2)) _lastSentWhoisRequest.erase(*a); @@ -675,7 +675,7 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) return ZT_WHOIS_RETRY_DELAY; } -bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address &destination) +bool Switch::_shouldUnite(const int64_t now,const Address &source,const Address &destination) { Mutex::Lock _l(_lastUniteAttempt_m); uint64_t &ts = _lastUniteAttempt[_LastUniteKey(source,destination)]; @@ -689,7 +689,7 @@ bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) { SharedPtr viaPath; - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); const Address destination(packet.destination()); const SharedPtr peer(RR->topology->getPeer(tPtr,destination)); @@ -703,7 +703,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) viaPath = peer->getBestPath(now,false); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { - if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { + if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(int64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { peer->attemptToContactAt(tPtr,viaPath->localSocket(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); viaPath->sent(now); } diff --git a/node/Switch.hpp b/node/Switch.hpp index c258a255..b42389fc 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -114,7 +114,7 @@ public: * @param now Current time * @param addr Address to look up */ - void requestWhois(void *tPtr,const uint64_t now,const Address &addr); + void requestWhois(void *tPtr,const int64_t now,const Address &addr); /** * Run any processes that are waiting for this peer's identity @@ -136,25 +136,25 @@ public: * @param now Current time * @return Number of milliseconds until doTimerTasks() should be run again */ - unsigned long doTimerTasks(void *tPtr,uint64_t now); + unsigned long doTimerTasks(void *tPtr,int64_t now); private: - bool _shouldUnite(const uint64_t now,const Address &source,const Address &destination); + bool _shouldUnite(const int64_t now,const Address &source,const Address &destination); bool _trySend(void *tPtr,Packet &packet,bool encrypt); // packet is modified if return is true const RuntimeEnvironment *const RR; - uint64_t _lastBeaconResponse; - volatile uint64_t _lastCheckedQueues; + int64_t _lastBeaconResponse; + volatile int64_t _lastCheckedQueues; // Time we last sent a WHOIS request for each address - Hashtable< Address,uint64_t > _lastSentWhoisRequest; + Hashtable< Address,int64_t > _lastSentWhoisRequest; Mutex _lastSentWhoisRequest_m; // Packets waiting for WHOIS replies or other decode info or missing fragments struct RXQueueEntry { RXQueueEntry() : timestamp(0) {} - volatile uint64_t timestamp; // 0 if entry is not in use + volatile int64_t timestamp; // 0 if entry is not in use volatile uint64_t packetId; IncomingPacket frag0; // head of packet Packet::Fragment frags[ZT_MAX_PACKET_FRAGMENTS - 1]; // later fragments (if any) diff --git a/node/Topology.cpp b/node/Topology.cpp index 8a830b93..f884e9c3 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -133,8 +133,9 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) if (ap) return ap; ap = Peer::deserializeFromCache(RR->node->now(),tPtr,buf,RR); - if (!ap) + if (!ap) { _peers.erase(zta); + } return SharedPtr(); } } catch ( ... ) {} // ignore invalid identities or other strage failures @@ -157,7 +158,7 @@ Identity Topology::getIdentity(void *tPtr,const Address &zta) SharedPtr Topology::getUpstreamPeer() { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); unsigned int bestq = ~((unsigned int)0); const SharedPtr *best = (const SharedPtr *)0; @@ -365,7 +366,7 @@ void Topology::removeMoon(void *tPtr,const uint64_t id) _memoizeUpstreams(tPtr); } -void Topology::doPeriodicTasks(void *tPtr,uint64_t now) +void Topology::doPeriodicTasks(void *tPtr,int64_t now) { { Mutex::Lock _l1(_peers_m); diff --git a/node/Topology.hpp b/node/Topology.hpp index 34df28a1..c3a218e3 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -286,13 +286,13 @@ public: /** * Clean and flush database */ - void doPeriodicTasks(void *tPtr,uint64_t now); + void doPeriodicTasks(void *tPtr,int64_t now); /** * @param now Current time * @return Number of peers with active direct paths */ - inline unsigned long countActive(uint64_t now) const + inline unsigned long countActive(int64_t now) const { unsigned long cnt = 0; Mutex::Lock _l(_peers_m); diff --git a/one.cpp b/one.cpp index b1a19e8c..de16cc2d 100644 --- a/one.cpp +++ b/one.cpp @@ -364,9 +364,9 @@ static int cli(int argc,char **argv) if (path["preferred"]) { char tmp[256]; std::string addr = path["address"]; - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); const double lq = (path.count("linkQuality")) ? (double)path["linkQuality"] : -1.0; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%lld;%lld;%1.2f",addr.c_str(),now - (int64_t)path["lastSend"],now - (int64_t)path["lastReceive"],lq); bestPath = tmp; break; } @@ -864,7 +864,7 @@ static int idtool(int argc,char **argv) } std::sort(roots.begin(),roots.end()); - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); World w(World::make(t,id,now,updatesMustBeSignedBy,roots,signingKey)); Buffer wbuf; w.serialize(wbuf); diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 8683ba25..8f66f850 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -200,7 +200,7 @@ public: /** * @return Current time in milliseconds since epoch */ - static inline uint64_t now() + static inline int64_t now() { #ifdef __WINDOWS__ FILETIME ft; @@ -210,7 +210,7 @@ public: SystemTimeToFileTime(&st,&ft); tmp.LowPart = ft.dwLowDateTime; tmp.HighPart = ft.dwHighDateTime; - return ( ((tmp.QuadPart - 116444736000000000ULL) / 10000L) + st.wMilliseconds ); + return (int64_t)( ((tmp.QuadPart - 116444736000000000LL) / 10000L) + st.wMilliseconds ); #else struct timeval tv; #ifdef __LINUX__ @@ -218,7 +218,7 @@ public: #else gettimeofday(&tv,(struct timezone *)0); #endif - return ( (1000ULL * (uint64_t)tv.tv_sec) + (uint64_t)(tv.tv_usec / 1000) ); + return ( (1000LL * (int64_t)tv.tv_sec) + (int64_t)(tv.tv_usec / 1000) ); #endif }; diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index 0da00653..825972b0 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -131,7 +131,7 @@ public: InetAddress publicAddress; sendpublicaddressrequest(&natpmp); - uint64_t myTimeout = OSUtils::now() + 5000; + int64_t myTimeout = OSUtils::now() + 5000; do { fd_set fds; struct timeval timeout; diff --git a/service/OneService.cpp b/service/OneService.cpp index 66e9a9c8..fb185ee7 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -276,6 +276,10 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) pa.push_back(j); } pj["paths"] = pa; + + if (peer->address == 0xda6c71a1ad) { + fprintf(stdout, "%s\n", pj.dump(2).c_str()); + } } static void _moonToJson(nlohmann::json &mj,const World &world) @@ -436,7 +440,7 @@ public: uint64_t _lastRestart; // Deadline for the next background task service function - volatile uint64_t _nextBackgroundTaskDeadline; + volatile int64_t _nextBackgroundTaskDeadline; // Configured networks struct NetworkState @@ -755,12 +759,12 @@ public: // Main I/O loop _nextBackgroundTaskDeadline = 0; - uint64_t clockShouldBe = OSUtils::now(); + int64_t clockShouldBe = OSUtils::now(); _lastRestart = clockShouldBe; - uint64_t lastTapMulticastGroupCheck = 0; - uint64_t lastBindRefresh = 0; - uint64_t lastUpdateCheck = clockShouldBe; - uint64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle + int64_t lastTapMulticastGroupCheck = 0; + int64_t lastBindRefresh = 0; + int64_t lastUpdateCheck = clockShouldBe; + int64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle for(;;) { _run_m.lock(); if (!_run) { @@ -773,7 +777,7 @@ public: _run_m.unlock(); } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); // Attempt to detect sleep/wake events by detecting delay overruns bool restarted = false; @@ -809,7 +813,7 @@ public: } // Run background task processor in core if it's time to do so - uint64_t dl = _nextBackgroundTaskDeadline; + int64_t dl = _nextBackgroundTaskDeadline; if (dl <= now) { _node->processBackgroundTasks((void *)0,now,&_nextBackgroundTaskDeadline); dl = _nextBackgroundTaskDeadline; @@ -2152,7 +2156,7 @@ public: // Engage TCP tunnel fallback if we haven't received anything valid from a global // IP address in ZT_TCP_FALLBACK_AFTER milliseconds. If we do start getting // valid direct traffic we'll stop using it and close the socket after a while. - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); if (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER)) { if (_tcpFallbackTunnel) { bool flushNow = false; diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 11005945..39833c90 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -303,7 +303,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void } } -bool SoftwareUpdater::check(const uint64_t now) +bool SoftwareUpdater::check(const int64_t now) { if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { _lastCheckTime = now; diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index ff3e36df..f16c99a0 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -167,7 +167,7 @@ public: * * @return True if we've downloaded and verified an update */ - bool check(const uint64_t now); + bool check(const int64_t now); /** * @return Meta-data for downloaded update or NULL if none -- cgit v1.2.3 From 395d8b31392309d2861cd2d50377aaa8953f42cd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 24 Oct 2017 13:33:53 -0700 Subject: Full and clearer implementation of GitHub issue #588 --- controller/EmbeddedNetworkController.cpp | 29 +++++++++++++++++++++++++++++ include/ZeroTierOne.h | 12 ++++++------ node/Capability.hpp | 13 +++++++++++++ node/Network.cpp | 29 +++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 6 deletions(-) (limited to 'controller') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 20f81966..1a7cdbc7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -227,6 +227,16 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: + r["type"] = "INTEGER_RANGE"; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.intRange.start); + r["start"] = tmp; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.intRange.start + (uint64_t)rule.v.intRange.end); + r["end"] = tmp; + r["idx"] = rule.v.intRange.idx; + r["little"] = ((rule.v.intRange.format & 0x80) != 0); + r["bits"] = (rule.v.intRange.format & 63) + 1; + break; default: break; } @@ -417,7 +427,26 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_TAG_RECEIVER") { rule.t |= ZT_NETWORK_RULE_MATCH_TAG_RECEIVER; tag = true; + } else if (t == "INTEGER_RANGE") { + json &s = r["start"]; + if (s.is_string()) { + std::string tmp = s; + rule.v.intRange.start = Utils::hexStrToU64(tmp.c_str()); + } else { + rule.v.intRange.start = OSUtils::jsonInt(s,0ULL); + } + json &e = r["end"]; + if (e.is_string()) { + std::string tmp = e; + rule.v.intRange.end = (uint32_t)(Utils::hexStrToU64(tmp.c_str()) - rule.v.intRange.start); + } else { + rule.v.intRange.end = (uint32_t)(OSUtils::jsonInt(e,0ULL) - rule.v.intRange.start); + } + rule.v.intRange.idx = (uint16_t)OSUtils::jsonInt(r["idx"],0ULL); + rule.v.intRange.format = (OSUtils::jsonBool(r["little"],false)) ? 0x80 : 0x00; + rule.v.intRange.format |= (uint8_t)((OSUtils::jsonInt(r["bits"],1ULL) - 1) & 63); } + if (tag) { rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 8adbc4d1..02bb283b 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -52,13 +52,13 @@ #else #define ZT_SDK_API __declspec(dllimport) #ifdef _DEBUG -#ifdef _WIN64 +#ifdef _WIN64 #pragma comment(lib, "ZeroTierOne_x64d.lib") #else #pragma comment(lib, "ZeroTierOne_x86d.lib") #endif #else -#ifdef _WIN64 +#ifdef _WIN64 #pragma comment(lib, "ZeroTierOne_x64.lib") #else #pragma comment(lib, "ZeroTierOne_x86.lib") @@ -750,6 +750,7 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 48, ZT_NETWORK_RULE_MATCH_TAG_SENDER = 49, ZT_NETWORK_RULE_MATCH_TAG_RECEIVER = 50, + ZT_NETWORK_RULE_MATCH_INTEGER_RANGE = 51, /** * Maximum ID allowed for a MATCH entry in the rules table @@ -770,7 +771,7 @@ enum ZT_VirtualNetworkRuleType */ typedef struct { - /** + /** * Type and flags * * Bits are: NOTTTTTT @@ -812,10 +813,9 @@ typedef struct */ struct { uint64_t start; // integer range start - int32_t delta; // +/- offset from start of integer range end + uint32_t end; // end of integer range (relative to start, inclusive, 0 for equality w/start) uint16_t idx; // index in packet of integer - uint8_t bits; // number of bits in integer (range: 1-64) - uint8_t endian; // endianness of integer in packet (0 == big, 1 == little) + uint8_t format; // bits in integer (range 1-64, ((format&63)+1)) and endianness (MSB 1 for little, 0 for big) } intRange; /** diff --git a/node/Capability.hpp b/node/Capability.hpp index c0336243..407884ad 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -278,6 +278,13 @@ public: b.append((uint32_t)rules[i].v.tag.id); b.append((uint32_t)rules[i].v.tag.value); break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: + b.append((uint8_t)19); + b.append((uint64_t)rules[i].v.intRange.start); + b.append((uint64_t)(rules[i].v.intRange.start + (uint64_t)rules[i].v.intRange.end)); // more future-proof + b.append((uint16_t)rules[i].v.intRange.idx); + b.append((uint8_t)rules[i].v.intRange.format); + break; } } } @@ -366,6 +373,12 @@ public: rules[ruleCount].v.tag.id = b.template at(p); rules[ruleCount].v.tag.value = b.template at(p + 4); break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: + rules[ruleCount].v.intRange.start = b.template at(p); + rules[ruleCount].v.intRange.end = (uint32_t)(b.template at(p + 8) - rules[ruleCount].v.intRange.start); + rules[ruleCount].v.intRange.idx = b.template at(p + 16); + rules[ruleCount].v.intRange.format = (uint8_t)b[p + 18]; + break; } p += fieldLen; ++ruleCount; diff --git a/node/Network.cpp b/node/Network.cpp index 111e4736..a9e8539e 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -493,6 +493,35 @@ static _doZtFilterResult _doZtFilter( } } } break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: { + uint64_t integer = 0; + const unsigned int bits = (rules[rn].v.intRange.format & 63) + 1; + const unsigned int bytes = ((bits + 8 - 1) / 8); // integer ceiling of division by 8 + if ((rules[rn].v.intRange.format & 0x80) == 0) { + // Big-endian + unsigned int idx = rules[rn].v.intRange.idx + (8 - bytes); + const unsigned int eof = idx + bytes; + if (eof <= frameLen) { + while (idx < eof) { + integer <<= 8; + integer |= frameData[idx++]; + } + } + integer &= 0xffffffffffffffffULL >> (64 - bits); + } else { + // Little-endian + unsigned int idx = rules[rn].v.intRange.idx; + const unsigned int eof = idx + bytes; + if (eof <= frameLen) { + while (idx < eof) { + integer >>= 8; + integer |= ((uint64_t)frameData[idx++]) << 56; + } + } + integer >>= (64 - bits); + } + thisRuleMatches = (uint8_t)((integer >= rules[rn].v.intRange.start)&&(integer <= (rules[rn].v.intRange.start + (uint64_t)rules[rn].v.intRange.end))); + } break; // The result of an unsupported MATCH is configurable at the network // level via a flag. -- cgit v1.2.3 From 4e88c80a22b6ca982341413ee806ade0df57b4b7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 2 Nov 2017 07:05:11 -0700 Subject: RethinkDB native connector work, minor fixes. --- .gitignore | 4 + controller/RethinkDB.cpp | 308 +++ controller/RethinkDB.hpp | 101 + ext/librethinkdbxx/.travis.yml | 11 + ext/librethinkdbxx/COPYRIGHT | 16 + ext/librethinkdbxx/Makefile | 126 + ext/librethinkdbxx/README.md | 72 + ext/librethinkdbxx/reql/add_docs.py | 80 + ext/librethinkdbxx/reql/gen.py | 33 + ext/librethinkdbxx/reql/python_docs.txt | 189 ++ ext/librethinkdbxx/reql/ql2.proto | 843 +++++++ ext/librethinkdbxx/src/connection.cc | 431 ++++ ext/librethinkdbxx/src/connection.h | 59 + ext/librethinkdbxx/src/connection_p.h | 133 + ext/librethinkdbxx/src/cursor.cc | 221 ++ ext/librethinkdbxx/src/cursor.h | 76 + ext/librethinkdbxx/src/cursor_p.h | 29 + ext/librethinkdbxx/src/datum.cc | 449 ++++ ext/librethinkdbxx/src/datum.h | 287 +++ ext/librethinkdbxx/src/error.h | 46 + ext/librethinkdbxx/src/exceptions.h | 13 + ext/librethinkdbxx/src/json.cc | 62 + ext/librethinkdbxx/src/json_p.h | 19 + ext/librethinkdbxx/src/rapidjson-config.h | 8 + ext/librethinkdbxx/src/rapidjson/allocators.h | 263 ++ ext/librethinkdbxx/src/rapidjson/document.h | 2575 ++++++++++++++++++++ ext/librethinkdbxx/src/rapidjson/encodedstream.h | 299 +++ ext/librethinkdbxx/src/rapidjson/encodings.h | 716 ++++++ ext/librethinkdbxx/src/rapidjson/error/en.h | 74 + ext/librethinkdbxx/src/rapidjson/error/error.h | 155 ++ ext/librethinkdbxx/src/rapidjson/filereadstream.h | 99 + ext/librethinkdbxx/src/rapidjson/filewritestream.h | 104 + ext/librethinkdbxx/src/rapidjson/fwd.h | 151 ++ .../src/rapidjson/internal/biginteger.h | 290 +++ ext/librethinkdbxx/src/rapidjson/internal/diyfp.h | 258 ++ ext/librethinkdbxx/src/rapidjson/internal/dtoa.h | 245 ++ .../src/rapidjson/internal/ieee754.h | 78 + ext/librethinkdbxx/src/rapidjson/internal/itoa.h | 304 +++ ext/librethinkdbxx/src/rapidjson/internal/meta.h | 181 ++ ext/librethinkdbxx/src/rapidjson/internal/pow10.h | 55 + ext/librethinkdbxx/src/rapidjson/internal/regex.h | 701 ++++++ ext/librethinkdbxx/src/rapidjson/internal/stack.h | 230 ++ .../src/rapidjson/internal/strfunc.h | 55 + ext/librethinkdbxx/src/rapidjson/internal/strtod.h | 269 ++ ext/librethinkdbxx/src/rapidjson/internal/swap.h | 46 + ext/librethinkdbxx/src/rapidjson/istreamwrapper.h | 115 + ext/librethinkdbxx/src/rapidjson/memorybuffer.h | 70 + ext/librethinkdbxx/src/rapidjson/memorystream.h | 71 + .../src/rapidjson/msinttypes/inttypes.h | 316 +++ .../src/rapidjson/msinttypes/stdint.h | 300 +++ ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h | 81 + ext/librethinkdbxx/src/rapidjson/pointer.h | 1358 +++++++++++ ext/librethinkdbxx/src/rapidjson/prettywriter.h | 249 ++ ext/librethinkdbxx/src/rapidjson/rapidjson.h | 615 +++++ ext/librethinkdbxx/src/rapidjson/reader.h | 1879 ++++++++++++++ ext/librethinkdbxx/src/rapidjson/schema.h | 2006 +++++++++++++++ ext/librethinkdbxx/src/rapidjson/stream.h | 179 ++ ext/librethinkdbxx/src/rapidjson/stringbuffer.h | 117 + ext/librethinkdbxx/src/rapidjson/writer.h | 609 +++++ ext/librethinkdbxx/src/term.cc | 285 +++ ext/librethinkdbxx/src/term.h | 592 +++++ ext/librethinkdbxx/src/types.cc | 47 + ext/librethinkdbxx/src/types.h | 53 + ext/librethinkdbxx/src/utils.cc | 153 ++ ext/librethinkdbxx/src/utils.h | 19 + ext/librethinkdbxx/test/bench.cc | 58 + ext/librethinkdbxx/test/gen_index_cxx.py | 11 + ext/librethinkdbxx/test/test.cc | 114 + ext/librethinkdbxx/test/testlib.cc | 356 +++ ext/librethinkdbxx/test/testlib.h | 231 ++ ext/librethinkdbxx/test/upstream/aggregation.yaml | 575 +++++ ext/librethinkdbxx/test/upstream/arity.yaml | 316 +++ .../test/upstream/changefeeds/edge.yaml | 142 ++ .../test/upstream/changefeeds/geo.rb.yaml | 27 + .../test/upstream/changefeeds/idxcopy.yaml | 38 + .../test/upstream/changefeeds/include_states.yaml | 58 + .../test/upstream/changefeeds/point.yaml | 147 ++ .../test/upstream/changefeeds/sindex.yaml | 50 + .../test/upstream/changefeeds/squash.yaml | 62 + .../test/upstream/changefeeds/table.yaml | 101 + ext/librethinkdbxx/test/upstream/control.yaml | 297 +++ ext/librethinkdbxx/test/upstream/datum/array.yaml | 133 + ext/librethinkdbxx/test/upstream/datum/binary.yaml | 363 +++ ext/librethinkdbxx/test/upstream/datum/bool.yaml | 47 + ext/librethinkdbxx/test/upstream/datum/null.yaml | 18 + ext/librethinkdbxx/test/upstream/datum/number.yaml | 125 + ext/librethinkdbxx/test/upstream/datum/object.yaml | 85 + ext/librethinkdbxx/test/upstream/datum/string.yaml | 329 +++ ext/librethinkdbxx/test/upstream/datum/typeof.yaml | 14 + ext/librethinkdbxx/test/upstream/datum/uuid.yaml | 20 + ext/librethinkdbxx/test/upstream/default.yaml | 270 ++ .../test/upstream/geo/constructors.yaml | 64 + ext/librethinkdbxx/test/upstream/geo/geojson.yaml | 31 + ext/librethinkdbxx/test/upstream/geo/indexing.yaml | 208 ++ .../test/upstream/geo/intersection_inclusion.yaml | 119 + .../test/upstream/geo/operations.yaml | 97 + .../test/upstream/geo/primitives.yaml | 50 + ext/librethinkdbxx/test/upstream/joins.yaml | 133 + ext/librethinkdbxx/test/upstream/json.yaml | 74 + ext/librethinkdbxx/test/upstream/limits.yaml | 128 + ext/librethinkdbxx/test/upstream/match.yaml | 38 + .../test/upstream/math_logic/add.yaml | 65 + .../test/upstream/math_logic/aliases.yaml | 46 + .../test/upstream/math_logic/comparison.yaml | 477 ++++ .../test/upstream/math_logic/div.yaml | 52 + .../test/upstream/math_logic/floor_ceil_round.yaml | 114 + .../test/upstream/math_logic/logic.yaml | 169 ++ .../test/upstream/math_logic/math.yaml | 11 + .../test/upstream/math_logic/mod.yaml | 34 + .../test/upstream/math_logic/mul.yaml | 60 + .../test/upstream/math_logic/sub.yaml | 37 + .../test/upstream/meta/composite.py.yaml | 14 + ext/librethinkdbxx/test/upstream/meta/dbs.yaml | 51 + ext/librethinkdbxx/test/upstream/meta/table.yaml | 365 +++ .../test/upstream/mutation/atomic_get_set.yaml | 93 + .../test/upstream/mutation/delete.yaml | 50 + .../test/upstream/mutation/insert.yaml | 239 ++ .../test/upstream/mutation/replace.yaml | 110 + .../test/upstream/mutation/sync.yaml | 52 + .../test/upstream/mutation/update.yaml | 170 ++ ext/librethinkdbxx/test/upstream/parsePolyglot.py | 164 ++ ext/librethinkdbxx/test/upstream/polymorphism.yaml | 33 + ext/librethinkdbxx/test/upstream/random.yaml | 180 ++ ext/librethinkdbxx/test/upstream/range.yaml | 53 + .../test/upstream/regression/1001.yaml | 20 + .../test/upstream/regression/1005.yaml | 19 + .../test/upstream/regression/1023.yaml | 65 + .../test/upstream/regression/1081.yaml | 39 + .../test/upstream/regression/1132.yaml | 4 + .../test/upstream/regression/1133.yaml | 19 + .../test/upstream/regression/1155.yaml | 5 + .../test/upstream/regression/1179.yaml | 26 + .../test/upstream/regression/1468.yaml | 7 + .../test/upstream/regression/1789.yaml | 22 + .../test/upstream/regression/2052.yaml | 10 + .../test/upstream/regression/2399.rb.yaml | 45 + .../test/upstream/regression/2639.rb.yaml | 8 + .../test/upstream/regression/2696.yaml | 6 + .../test/upstream/regression/2697.yaml | 31 + .../test/upstream/regression/2709.yaml | 21 + .../test/upstream/regression/2710.yaml | 6 + .../test/upstream/regression/2766.yaml | 25 + .../test/upstream/regression/2767.yaml | 20 + .../test/upstream/regression/2774.yaml | 99 + .../test/upstream/regression/2838.py.yaml | 16 + .../test/upstream/regression/2930.yaml | 17 + .../test/upstream/regression/3057.yaml | 10 + .../test/upstream/regression/3059.yaml | 7 + .../test/upstream/regression/309.yaml | 15 + .../test/upstream/regression/3444.yaml | 38 + .../test/upstream/regression/3449.js.yaml | 21 + .../test/upstream/regression/354.yaml | 20 + .../test/upstream/regression/3637.yaml | 51 + .../test/upstream/regression/370.yaml | 19 + .../test/upstream/regression/3745.yaml | 17 + .../test/upstream/regression/3759.yaml | 26 + .../test/upstream/regression/4030.yaml | 48 + .../test/upstream/regression/4063.js.yaml | 9 + .../test/upstream/regression/4132.yaml | 12 + .../test/upstream/regression/4146.yaml | 14 + .../test/upstream/regression/4431.yaml | 10 + .../test/upstream/regression/4462.yaml | 24 + .../test/upstream/regression/4465.py.yaml.disabled | 8 + .../test/upstream/regression/4501.yaml | 5 + .../test/upstream/regression/453.yaml | 16 + .../test/upstream/regression/4582.yaml | 9 + .../test/upstream/regression/4591.yaml | 4 + .../test/upstream/regression/46.yaml | 11 + .../test/upstream/regression/469.yaml | 139 ++ .../test/upstream/regression/4729.yaml | 13 + .../test/upstream/regression/5092.js.yaml | 14 + .../test/upstream/regression/5130.js.yaml | 5 + .../test/upstream/regression/522.py.yaml | 7 + .../test/upstream/regression/5222.py.yaml | 23 + .../test/upstream/regression/5241.yaml | 28 + .../test/upstream/regression/5383.rb.yaml | 20 + .../test/upstream/regression/5438.yaml | 16 + .../test/upstream/regression/545.yaml | 15 + .../test/upstream/regression/546.yaml | 27 + .../test/upstream/regression/5481.py.yaml | 5 + .../test/upstream/regression/5535.rb.yaml | 12 + .../test/upstream/regression/5542.py.yaml | 11 + .../test/upstream/regression/568.yaml | 10 + .../test/upstream/regression/578.yaml | 25 + .../test/upstream/regression/579.yaml | 15 + .../test/upstream/regression/619.yaml | 15 + .../test/upstream/regression/665.yaml | 19 + .../test/upstream/regression/678.yaml | 10 + .../test/upstream/regression/718.yaml | 7 + .../test/upstream/regression/730.yaml | 4 + .../test/upstream/regression/757.yaml | 9 + .../test/upstream/regression/763.js.yaml | 33 + .../test/upstream/regression/767.yaml | 10 + .../test/upstream/regression/831.yaml | 5 + ext/librethinkdbxx/test/upstream/selection.yaml | 355 +++ ext/librethinkdbxx/test/upstream/sindex/api.yaml | 1348 ++++++++++ .../test/upstream/sindex/nullsinstrings.yaml | 21 + .../test/upstream/sindex/status.yaml | 76 + .../test/upstream/sindex/trunc_array.rb.yaml | 70 + .../test/upstream/sindex/truncation.rb.yaml | 57 + ext/librethinkdbxx/test/upstream/timeout.yaml | 39 + ext/librethinkdbxx/test/upstream/times/api.yaml | 144 ++ .../test/upstream/times/constructors.yaml | 96 + ext/librethinkdbxx/test/upstream/times/index.yaml | 151 ++ .../test/upstream/times/portions.yaml | 45 + ext/librethinkdbxx/test/upstream/times/shim.yaml | 22 + .../test/upstream/times/time_arith.yaml | 197 ++ .../test/upstream/times/timezones.yaml | 87 + .../test/upstream/transform/array.yaml | 303 +++ .../test/upstream/transform/fold.yaml | 49 + .../test/upstream/transform/map.yaml | 106 + .../test/upstream/transform/object.yaml | 147 ++ .../test/upstream/transform/table.yaml | 19 + .../test/upstream/transform/unordered_map.yaml | 63 + .../test/upstream/transformation.yaml | 543 +++++ ext/librethinkdbxx/test/yaml_to_cxx.py | 467 ++++ node/InetAddress.hpp | 7 + osdep/OSUtils.cpp | 15 + osdep/OSUtils.hpp | 1 + 219 files changed, 33295 insertions(+) create mode 100644 controller/RethinkDB.cpp create mode 100644 controller/RethinkDB.hpp create mode 100644 ext/librethinkdbxx/.travis.yml create mode 100644 ext/librethinkdbxx/COPYRIGHT create mode 100644 ext/librethinkdbxx/Makefile create mode 100644 ext/librethinkdbxx/README.md create mode 100644 ext/librethinkdbxx/reql/add_docs.py create mode 100644 ext/librethinkdbxx/reql/gen.py create mode 100644 ext/librethinkdbxx/reql/python_docs.txt create mode 100644 ext/librethinkdbxx/reql/ql2.proto create mode 100644 ext/librethinkdbxx/src/connection.cc create mode 100644 ext/librethinkdbxx/src/connection.h create mode 100644 ext/librethinkdbxx/src/connection_p.h create mode 100644 ext/librethinkdbxx/src/cursor.cc create mode 100644 ext/librethinkdbxx/src/cursor.h create mode 100644 ext/librethinkdbxx/src/cursor_p.h create mode 100644 ext/librethinkdbxx/src/datum.cc create mode 100644 ext/librethinkdbxx/src/datum.h create mode 100644 ext/librethinkdbxx/src/error.h create mode 100644 ext/librethinkdbxx/src/exceptions.h create mode 100644 ext/librethinkdbxx/src/json.cc create mode 100644 ext/librethinkdbxx/src/json_p.h create mode 100644 ext/librethinkdbxx/src/rapidjson-config.h create mode 100644 ext/librethinkdbxx/src/rapidjson/allocators.h create mode 100644 ext/librethinkdbxx/src/rapidjson/document.h create mode 100644 ext/librethinkdbxx/src/rapidjson/encodedstream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/encodings.h create mode 100644 ext/librethinkdbxx/src/rapidjson/error/en.h create mode 100644 ext/librethinkdbxx/src/rapidjson/error/error.h create mode 100644 ext/librethinkdbxx/src/rapidjson/filereadstream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/filewritestream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/fwd.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/biginteger.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/diyfp.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/dtoa.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/ieee754.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/itoa.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/meta.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/pow10.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/regex.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/stack.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/strfunc.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/strtod.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/swap.h create mode 100644 ext/librethinkdbxx/src/rapidjson/istreamwrapper.h create mode 100644 ext/librethinkdbxx/src/rapidjson/memorybuffer.h create mode 100644 ext/librethinkdbxx/src/rapidjson/memorystream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h create mode 100644 ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h create mode 100644 ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h create mode 100644 ext/librethinkdbxx/src/rapidjson/pointer.h create mode 100644 ext/librethinkdbxx/src/rapidjson/prettywriter.h create mode 100644 ext/librethinkdbxx/src/rapidjson/rapidjson.h create mode 100644 ext/librethinkdbxx/src/rapidjson/reader.h create mode 100644 ext/librethinkdbxx/src/rapidjson/schema.h create mode 100644 ext/librethinkdbxx/src/rapidjson/stream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/stringbuffer.h create mode 100644 ext/librethinkdbxx/src/rapidjson/writer.h create mode 100644 ext/librethinkdbxx/src/term.cc create mode 100644 ext/librethinkdbxx/src/term.h create mode 100644 ext/librethinkdbxx/src/types.cc create mode 100644 ext/librethinkdbxx/src/types.h create mode 100644 ext/librethinkdbxx/src/utils.cc create mode 100644 ext/librethinkdbxx/src/utils.h create mode 100644 ext/librethinkdbxx/test/bench.cc create mode 100644 ext/librethinkdbxx/test/gen_index_cxx.py create mode 100644 ext/librethinkdbxx/test/test.cc create mode 100644 ext/librethinkdbxx/test/testlib.cc create mode 100644 ext/librethinkdbxx/test/testlib.h create mode 100644 ext/librethinkdbxx/test/upstream/aggregation.yaml create mode 100644 ext/librethinkdbxx/test/upstream/arity.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/point.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/table.yaml create mode 100644 ext/librethinkdbxx/test/upstream/control.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/array.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/binary.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/bool.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/null.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/number.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/object.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/string.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/typeof.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/uuid.yaml create mode 100644 ext/librethinkdbxx/test/upstream/default.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/constructors.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/geojson.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/indexing.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/operations.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/primitives.yaml create mode 100644 ext/librethinkdbxx/test/upstream/joins.yaml create mode 100644 ext/librethinkdbxx/test/upstream/json.yaml create mode 100644 ext/librethinkdbxx/test/upstream/limits.yaml create mode 100644 ext/librethinkdbxx/test/upstream/match.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/add.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/div.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/logic.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/math.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/mod.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/mul.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/sub.yaml create mode 100644 ext/librethinkdbxx/test/upstream/meta/composite.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/meta/dbs.yaml create mode 100644 ext/librethinkdbxx/test/upstream/meta/table.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/delete.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/insert.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/replace.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/sync.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/update.yaml create mode 100755 ext/librethinkdbxx/test/upstream/parsePolyglot.py create mode 100644 ext/librethinkdbxx/test/upstream/polymorphism.yaml create mode 100644 ext/librethinkdbxx/test/upstream/random.yaml create mode 100644 ext/librethinkdbxx/test/upstream/range.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1001.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1005.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1023.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1081.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1132.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1133.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1155.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1179.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1468.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1789.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2052.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2696.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2697.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2709.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2710.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2766.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2767.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2774.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2838.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2930.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3057.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3059.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/309.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3444.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3449.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/354.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3637.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/370.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3745.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3759.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4030.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4063.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4132.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4146.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4431.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4462.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4465.py.yaml.disabled create mode 100644 ext/librethinkdbxx/test/upstream/regression/4501.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/453.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4582.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4591.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/46.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/469.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4729.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5092.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5130.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/522.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5222.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5241.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5383.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5438.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/545.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/546.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5481.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5535.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5542.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/568.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/578.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/579.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/619.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/665.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/678.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/718.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/730.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/757.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/763.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/767.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/831.yaml create mode 100644 ext/librethinkdbxx/test/upstream/selection.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/api.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/nullsinstrings.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/status.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/trunc_array.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/truncation.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/timeout.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/api.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/constructors.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/index.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/portions.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/shim.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/time_arith.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/timezones.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/array.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/fold.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/map.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/object.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/table.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/unordered_map.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transformation.yaml create mode 100644 ext/librethinkdbxx/test/yaml_to_cxx.py (limited to 'controller') diff --git a/.gitignore b/.gitignore index 61dd1c85..936de092 100755 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,7 @@ build/ !default.perspectivev3 *.xccheckout xcuserdata/ +ext/librethinkdbxx/build +.vscode +__pycache__ +*~ diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp new file mode 100644 index 00000000..5d93bf72 --- /dev/null +++ b/controller/RethinkDB.cpp @@ -0,0 +1,308 @@ +#include "RethinkDB.hpp" + +#include +#include +#include + +#include "../ext/librethinkdbxx/build/include/rethinkdb.h" + +namespace R = RethinkDB; +using nlohmann::json; + +namespace ZeroTier { + +RethinkDB::RethinkDB(const Address &myAddress,const char *host,const int port,const char *db,const char *auth) : + _myAddress(myAddress), + _host(host ? host : "127.0.0.1"), + _db(db), + _auth(auth ? auth : ""), + _port((port > 0) ? port : 28015), + _ready(2), // two tables need to be synchronized before we're ready + _run(1) +{ + _readyLock.lock(); + + { + char tmp[32]; + _myAddress.toString(tmp); + _myAddressStr = tmp; + } + + _membersDbWatcher = std::thread([this]() { + while (_run == 1) { + try { + auto rdb = R::connect(this->_host,this->_port,this->_auth); + if (rdb) { + _membersDbWatcherConnection = (void *)rdb.get(); + auto cur = R::db(this->_db).table("Member").get_all(this->_myAddressStr,R::optargs("index","controllerId")).changes(R::optargs("squash",0.1,"include_initial",true,"include_types",true,"include_states",true)).run(*rdb); + while (cur.has_next()) { + if (_run != 1) break; + json tmp(json::parse(cur.next().as_json())); + if ((tmp["type"] == "state")&&(tmp["state"] == "ready")) { + if (--this->_ready == 0) + this->_readyLock.unlock(); + } else { + try { + this->_memberChanged(tmp["old_val"],tmp["new_val"]); + } catch ( ... ) {} // ignore bad records + } + } + } + } catch (std::exception &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.what()); + } catch (R::Error &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.message.c_str()); + } catch ( ... ) { + fprintf(stderr,"ERROR: controller RethinkDB: unknown exception" ZT_EOL_S); + } + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + }); + + _networksDbWatcher = std::thread([this]() { + while (_run == 1) { + try { + auto rdb = R::connect(this->_host,this->_port,this->_auth); + if (rdb) { + _membersDbWatcherConnection = (void *)rdb.get(); + auto cur = R::db(this->_db).table("Network").get_all(this->_myAddressStr,R::optargs("index","controllerId")).changes(R::optargs("squash",0.1,"include_initial",true,"include_types",true,"include_states",true)).run(*rdb); + while (cur.has_next()) { + if (_run != 1) break; + json tmp(json::parse(cur.next().as_json())); + if ((tmp["type"] == "state")&&(tmp["state"] == "ready")) { + if (--this->_ready == 0) + this->_readyLock.unlock(); + } else { + try { + this->_networkChanged(tmp["old_val"],tmp["new_val"]); + } catch ( ... ) {} // ignore bad records + } + } + } + } catch (std::exception &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.what()); + } catch (R::Error &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.message.c_str()); + } catch ( ... ) { + fprintf(stderr,"ERROR: controller RethinkDB: unknown exception" ZT_EOL_S); + } + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + }); +} + +RethinkDB::~RethinkDB() +{ + // FIXME: not totally safe but will generally work, and only happens on shutdown anyway + _run = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (_membersDbWatcherConnection) + ((R::Connection *)_membersDbWatcherConnection)->close(); + if (_networksDbWatcherConnection) + ((R::Connection *)_networksDbWatcherConnection)->close(); + _membersDbWatcher.join(); + _networksDbWatcher.join(); +} + +inline bool RethinkDB::get(const uint64_t networkId,nlohmann::json &network) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + network = nw->config; + + return true; +} + +inline bool RethinkDB::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + auto m = nw->members.find(memberId); + if (m == nw->members.end()) + return false; + network = nw->config; + member = m->second; + _fillSummaryInfo(nw,info); + + return true; +} + +inline bool RethinkDB::get(const uint64_t networkId,nlohmann::json &network,std::vector &members) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + network = nw->config; + for(auto m=nw->members.begin();m!=nw->members.end();++m) + members.push_back(m->second); + + return true; +} + +inline bool RethinkDB::summary(const uint64_t networkId,NetworkSummaryInfo &info) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + _fillSummaryInfo(nw,info); + + return true; +} + +void RethinkDB::_memberChanged(nlohmann::json &old,nlohmann::json &member) +{ + uint64_t memberId = 0; + uint64_t networkId = 0; + std::shared_ptr<_Network> nw; + + if (old.is_object()) { + json &config = old["config"]; + if (config.is_object()) { + memberId = OSUtils::jsonIntHex(config["id"],0ULL); + networkId = OSUtils::jsonIntHex(config["nwid"],0ULL); + if ((memberId)&&(networkId)) { + { + std::lock_guard l(_networks_l); + auto nw2 = _networks.find(networkId); + if (nw2 != _networks.end()) + nw = nw2->second; + } + if (nw) { + std::lock_guard l(nw->lock); + if (OSUtils::jsonBool(config["activeBridge"],false)) + nw->activeBridgeMembers.erase(memberId); + if (OSUtils::jsonBool(config["authorized"],false)) + nw->authorizedMembers.erase(memberId); + json &ips = config["ipAssignments"]; + if (ips.is_array()) { + for(unsigned long i=0;iallocatedIps.erase(ipa); + } + } + } + } + } + } + } + + if (member.is_object()) { + json &config = member["config"]; + if (config.is_object()) { + if (!nw) { + memberId = OSUtils::jsonIntHex(config["id"],0ULL); + networkId = OSUtils::jsonIntHex(config["nwid"],0ULL); + if ((!memberId)||(!networkId)) + return; + std::lock_guard l(_networks_l); + std::shared_ptr<_Network> &nw2 = _networks[networkId]; + if (!nw2) + nw2.reset(new _Network); + nw = nw2; + } + std::lock_guard l(nw->lock); + + nw->members[memberId] = config; + + if (OSUtils::jsonBool(config["activeBridge"],false)) + nw->activeBridgeMembers.insert(memberId); + const bool isAuth = OSUtils::jsonBool(config["authorized"],false); + if (isAuth) + nw->authorizedMembers.insert(memberId); + json &ips = config["ipAssignments"]; + if (ips.is_array()) { + for(unsigned long i=0;iallocatedIps.insert(ipa); + } + } + } + + if (!isAuth) { + const int64_t ldt = (int64_t)OSUtils::jsonInt(config["lastDeauthorizedTime"],0ULL); + if (ldt > nw->mostRecentDeauthTime) + nw->mostRecentDeauthTime = ldt; + } + } + } +} + +void RethinkDB::_networkChanged(nlohmann::json &old,nlohmann::json &network) +{ + if (network.is_object()) { + json &config = network["config"]; + if (config.is_object()) { + const std::string ids = config["id"]; + const uint64_t id = Utils::hexStrToU64(ids.c_str()); + if (id) { + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + std::shared_ptr<_Network> &nw2 = _networks[id]; + if (!nw2) + nw2.reset(new _Network); + nw = nw2; + } + std::lock_guard l2(nw->lock); + nw->config = config; + } + } + } else if (old.is_object()) { + const std::string ids = old["id"]; + const uint64_t id = Utils::hexStrToU64(ids.c_str()); + if (id) { + std::lock_guard l(_networks_l); + _networks.erase(id); + } + } +} + +} // namespace ZeroTier + +/* +int main(int argc,char **argv) +{ + ZeroTier::RethinkDB db(ZeroTier::Address(0x8056c2e21cULL),"10.6.6.188",28015,"ztc",""); + db.waitForReady(); + printf("ready.\n"); + pause(); +} +*/ diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp new file mode 100644 index 00000000..7ed0e0a8 --- /dev/null +++ b/controller/RethinkDB.hpp @@ -0,0 +1,101 @@ +#ifndef ZT_CONTROLLER_RETHINKDB_HPP +#define ZT_CONTROLLER_RETHINKDB_HPP + +#include "../node/Constants.hpp" +#include "../node/Address.hpp" +#include "../node/InetAddress.hpp" +#include "../osdep/OSUtils.hpp" + +#include +#include +#include +#include +#include +#include + +#include "../ext/json/json.hpp" + +namespace ZeroTier +{ + +class RethinkDB +{ +public: + struct NetworkSummaryInfo + { + NetworkSummaryInfo() : authorizedMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} + std::vector
activeBridges; + std::vector allocatedIps; + unsigned long authorizedMemberCount; + unsigned long totalMemberCount; + int64_t mostRecentDeauthTime; + }; + + RethinkDB(const Address &myAddress,const char *host,const int port,const char *db,const char *auth); + ~RethinkDB(); + + inline bool ready() const { return (_ready <= 0); } + + inline void waitForReady() const + { + while (_ready > 0) { + _readyLock.lock(); + _readyLock.unlock(); + } + } + + bool get(const uint64_t networkId,nlohmann::json &network); + bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info); + bool get(const uint64_t networkId,nlohmann::json &network,std::vector &members); + bool summary(const uint64_t networkId,NetworkSummaryInfo &info); + +private: + struct _Network + { + _Network() : mostRecentDeauthTime(0) {} + nlohmann::json config; + std::unordered_map members; + std::unordered_set activeBridgeMembers; + std::unordered_set authorizedMembers; + std::unordered_set allocatedIps; + int64_t mostRecentDeauthTime; + std::mutex lock; + }; + + void _memberChanged(nlohmann::json &old,nlohmann::json &member); + void _networkChanged(nlohmann::json &old,nlohmann::json &network); + + inline void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info) + { + for(auto ab=nw->activeBridgeMembers.begin();ab!=nw->activeBridgeMembers.end();++ab) + info.activeBridges.push_back(Address(*ab)); + for(auto ip=nw->allocatedIps.begin();ip!=nw->allocatedIps.end();++ip) + info.allocatedIps.push_back(*ip); + info.authorizedMemberCount = (unsigned long)nw->authorizedMembers.size(); + info.totalMemberCount = (unsigned long)nw->members.size(); + info.mostRecentDeauthTime = nw->mostRecentDeauthTime; + } + + const Address _myAddress; + std::string _myAddressStr; + std::string _host; + std::string _db; + std::string _auth; + const int _port; + + void *_networksDbWatcherConnection; + void *_membersDbWatcherConnection; + std::thread _networksDbWatcher; + std::thread _membersDbWatcher; + + std::unordered_map< uint64_t,std::shared_ptr<_Network> > _networks; + std::mutex _networks_l; + + mutable std::mutex _readyLock; // locked until ready + std::atomic _ready; + std::atomic _run; +}; + +} // namespace ZeroTier + +#endif diff --git a/ext/librethinkdbxx/.travis.yml b/ext/librethinkdbxx/.travis.yml new file mode 100644 index 00000000..b306a410 --- /dev/null +++ b/ext/librethinkdbxx/.travis.yml @@ -0,0 +1,11 @@ +sudo: required +dist: trusty + +python: + - "3.4.3" + +addons: + rethinkdb: "2.3" + +script: + - make test diff --git a/ext/librethinkdbxx/COPYRIGHT b/ext/librethinkdbxx/COPYRIGHT new file mode 100644 index 00000000..c25145d5 --- /dev/null +++ b/ext/librethinkdbxx/COPYRIGHT @@ -0,0 +1,16 @@ +RethinkDB Language Drivers + +Copyright 2010-2012 RethinkDB + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this product except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/ext/librethinkdbxx/Makefile b/ext/librethinkdbxx/Makefile new file mode 100644 index 00000000..ba319c16 --- /dev/null +++ b/ext/librethinkdbxx/Makefile @@ -0,0 +1,126 @@ +# Customisable build settings + +CXX ?= clang++ +CXXFLAGS ?= +INCLUDE_PYTHON_DOCS ?= no +DEBUG ?= no +PYTHON ?= python3 + +# Required build settings + +ifneq (no,$(DEBUG)) + CXXFLAGS += -ggdb +else + CXXFLAGS += -O3 # -flto +endif + +CXXFLAGS += -std=c++11 -I'build/gen' -Wall -pthread -fPIC + +prefix ?= /usr +DESTDIR ?= + +.DELETE_ON_ERROR: +SHELL := /bin/bash + +modules := connection datum json term cursor types utils +headers := utils error exceptions types datum connection cursor term + +o_files := $(patsubst %, build/obj/%.o, $(modules)) +d_files := $(patsubst %, build/dep/%.d, $(modules)) + +skip_tests := regression/1133 regression/767 regression/1005 # python-only +skip_tests += arity # arity errors are compile-time +skip_tests += geo # geo types not implemented yet +skip_tests += limits # possibly broken tests: https://github.com/rethinkdb/rethinkdb/issues/5940 + +upstream_tests := \ + $(filter-out %.rb.%, \ + $(filter-out $(patsubst %,test/upstream/%%, $(skip_tests)), \ + $(filter test/upstream/$(test_filter)%, \ + $(shell find test/upstream -name '*.yaml' | egrep -v '.(rb|js).yaml$$')))) +upstream_tests_cc := $(patsubst %.yaml, build/tests/%.cc, $(upstream_tests)) +upstream_tests_o := $(patsubst %.cc, %.o, $(upstream_tests_cc)) + +.PRECIOUS: $(upstream_tests_cc) $(upstream_tests_o) + +default: build/librethinkdb++.a build/include/rethinkdb.h build/librethinkdb++.so + +all: default build/test + +build/librethinkdb++.a: $(o_files) + ar rcs $@ $^ + +build/librethinkdb++.so: $(o_files) + $(CXX) -o $@ $(CXXFLAGS) -shared $^ + +build/obj/%.o: src/%.cc build/gen/protocol_defs.h + @mkdir -p $(dir $@) + @mkdir -p $(dir build/dep/$*.d) + $(CXX) -o $@ $(CXXFLAGS) -c $< -MP -MQ $@ -MD -MF build/dep/$*.d + +build/gen/protocol_defs.h: reql/ql2.proto reql/gen.py | build/gen/. + $(PYTHON) reql/gen.py $< > $@ + +clean: + rm -rf build + +ifneq (no,$(INCLUDE_PYTHON_DOCS)) +build/include/rethinkdb.h: build/rethinkdb.nodocs.h reql/add_docs.py reql/python_docs.txt | build/include/. + $(PYTHON) reql/add_docs.py reql/python_docs.txt < $< > $@ +else +build/include/rethinkdb.h: build/rethinkdb.nodocs.h | build/include/. + cp $< $@ +endif + +build/rethinkdb.nodocs.h: build/gen/protocol_defs.h $(patsubst %, src/%.h, $(headers)) + ( echo "// Auto-generated file, built from $^"; \ + echo '#pragma once'; \ + cat $^ | \ + grep -v '^#pragma once' | \ + grep -v '^#include "'; \ + ) > $@ + +build/tests/%.cc: %.yaml test/yaml_to_cxx.py + @mkdir -p $(dir $@) + $(PYTHON) test/yaml_to_cxx.py $< > $@ + +build/tests/upstream_tests.cc: $(upstream_tests) test/gen_index_cxx.py FORCE | build/tests/. + @echo '$(PYTHON) test/gen_index_cxx.py $(wordlist 1,5,$(upstream_tests)) ... > $@' + @$(PYTHON) test/gen_index_cxx.py $(upstream_tests) > $@ + +build/tests/%.o: build/tests/%.cc build/include/rethinkdb.h test/testlib.h | build/tests/. + $(CXX) -o $@ $(CXXFLAGS) -isystem build/include -I test -c $< -Wno-unused-variable + +build/tests/%.o: test/%.cc test/testlib.h build/include/rethinkdb.h | build/tests/. + $(CXX) -o $@ $(CXXFLAGS) -isystem build/include -I test -c $< + +build/test: build/tests/testlib.o build/tests/test.o build/tests/upstream_tests.o $(upstream_tests_o) build/librethinkdb++.a + @echo $(CXX) -o $@ $(CXXFLAGS) $(wordlist 1,5,$^) ... + @$(CXX) -o $@ $(CXXFLAGS) build/librethinkdb++.a $^ + +.PHONY: test +test: build/test + build/test + +build/bench: build/tests/bench.o build/librethinkdb++.a + @$(CXX) -o $@ $(CXXFLAGS) -isystem build/include build/librethinkdb++.a $^ + +.PHONY: bench +bench: build/bench + build/bench + +.PHONY: install +install: build/librethinkdb++.a build/include/rethinkdb.h build/librethinkdb++.so + install -m755 -d $(DESTDIR)$(prefix)/lib + install -m755 -d $(DESTDIR)$(prefix)/include + install -m644 build/librethinkdb++.a $(DESTDIR)$(prefix)/lib/librethinkdb++.a + install -m644 build/librethinkdb++.so $(DESTDIR)$(prefix)/lib/librethinkdb++.so + install -m644 build/include/rethinkdb.h $(DESTDIR)$(prefix)/include/rethinkdb.h + +%/.: + mkdir -p $* + +.PHONY: FORCE +FORCE: + +-include $(d_files) diff --git a/ext/librethinkdbxx/README.md b/ext/librethinkdbxx/README.md new file mode 100644 index 00000000..92fa9136 --- /dev/null +++ b/ext/librethinkdbxx/README.md @@ -0,0 +1,72 @@ +# RethinkDB driver for C++ + +This driver is compatible with RethinkDB 2.0. It is based on the +official RethinkDB Python driver. + +* [RethinkDB server](http://rethinkdb.com/) +* [RethinkDB API docs](http://rethinkdb.com/api/python/) + +## Example + +``` +#include +#include +#include + +namespace R = RethinkDB; + +int main() { + std::unique_ptr conn = R::connect("localhost", 28015); + R::Cursor cursor = R::table("users").filter(R::row["age"] > 14).run(*conn); + for (R::Datum& user : cursor) { + printf("%s\n", user.as_json().c_str()); + } +} +``` + +## Build + +Requires a modern C++ compiler. to build and install, run: + +``` +make +make install +``` + +Will build `include/rethinkdb.h`, `librethinkdb++.a` and +`librethinkdb++.so` into the `build/` directory. + +To include documentation from the Python driver in the header file, +pass the following argument to make. + +``` +make INCLUDE_PYTHON_DOCS=yes +``` + +To build in debug mode: + +``` +make DEBUG=yes +``` + +To install to a specific location: + +``` +make install prefix=/usr/local DESTDIR= +``` + +## Status + +Still in early stages of development. + +## Tests + +This driver is tested against the upstream ReQL tests from the +RethinkDB repo, which are programmatically translated from Python to +C++. As of 34dc13c, all tests pass: + +``` +$ make test +... +SUCCESS: 2053 tests passed +``` diff --git a/ext/librethinkdbxx/reql/add_docs.py b/ext/librethinkdbxx/reql/add_docs.py new file mode 100644 index 00000000..67f08df8 --- /dev/null +++ b/ext/librethinkdbxx/reql/add_docs.py @@ -0,0 +1,80 @@ +from sys import stdin, stderr, stdout, argv +from re import match, sub + +docs = {} + +for line in open(argv[1]): + res = match('^\t\(([^,]*), (.*)\),$', line) + if res: + fullname = res.group(1) + docs[fullname.split('.')[-1]] = eval(res.group(2)).decode('utf-8') + +translate_name = { + 'name': None, + 'delete_': 'delete', + 'union_': 'union', + 'operator[]': '__getitem__', + 'operator+': '__add__', + 'operator-': '__sub__', + 'operator*': '__mul__', + 'operator/': '__div__', + 'operator%': '__mod__', + 'operator&&': 'and_', + 'operator||': 'or_', + 'operator==': '__eq__', + 'operator!=': '__ne__', + 'operator>': '__gt__', + 'operator>=': '__ge__', + 'operator<': '__lt__', + 'operator<=': '__le__', + 'operator!': 'not_', + 'default_': 'default', + 'array': None, + 'desc': None, + 'asc': None, + 'maxval': None, + 'minval': None, + 'january': None, + 'february': None, + 'march': None, + 'april': None, + 'may': None, + 'june': None, + 'july': None, + 'august': None, + 'september': None, + 'october': None, + 'november': None, + 'december': None, + 'monday': None, + 'tuesday': None, + 'wednesday': None, + 'thursday': None, + 'friday': None, + 'saturday': None, + 'sunday': None, +} + +def print_docs(name, line): + py_name = translate_name.get(name, name) + if py_name in docs: + indent = match("^( *)", line).group(1) + stdout.write('\n') + # TODO: convert the examples to C++ + for line in docs[py_name].split('\n'): + stdout.write(indent + "// " + line + '\n') + elif py_name: + stderr.write('Warning: no docs for ' + py_name + ': ' + line) + +stdout.write('// Contains documentation copied as-is from the Python driver') + +for line in stdin: + res = match("^ *CO?[0-9_]+\(([^,)]+)|extern Query (\w+)|^ *// *(\$)doc\((\w+)\) *$", line) + if res: + name = res.group(1) or res.group(2) or res.group(4) + print_docs(name, line) + if not res.group(3): + stdout.write(line) + else: + stdout.write(line) + diff --git a/ext/librethinkdbxx/reql/gen.py b/ext/librethinkdbxx/reql/gen.py new file mode 100644 index 00000000..2b1fe9fc --- /dev/null +++ b/ext/librethinkdbxx/reql/gen.py @@ -0,0 +1,33 @@ +from sys import argv +from re import sub, finditer, VERBOSE + +def gen(defs): + indent = 0 + enum = False + def p(s): print(" " * (indent * 4) + s) + for item in finditer(""" + (?P message|enum) \\s+ (?P \\w+) \\s* \\{ | + (?P \\w+) \\s* = \\s* (?P \\w+) \\s* ; | + \\} + """, defs, flags=VERBOSE): + if item.group(0) == "}": + indent = indent - 1 + p("};" if enum else "}") + enum = False; + elif item.group('type') == 'enum': + p("enum class %s {" % item.group('name')) + indent = indent + 1 + enum = True + elif item.group('type') == 'message': + p("namespace %s {" % item.group('name')) + indent = indent + 1 + enum = False + else: + if enum: + p("%s = %s," % (item.group('var'), item.group('val'))) + +print("// Auto-generated by reql/gen.py") +print("#pragma once") +print("namespace RethinkDB { namespace Protocol {") +gen(sub("//.*", "", open(argv[1]).read())) +print("} }") diff --git a/ext/librethinkdbxx/reql/python_docs.txt b/ext/librethinkdbxx/reql/python_docs.txt new file mode 100644 index 00000000..41a5c84e --- /dev/null +++ b/ext/librethinkdbxx/reql/python_docs.txt @@ -0,0 +1,189 @@ +# This file was generated by _scripts/gen_python.py from the rethinkdb documentation in http://github.com/rethinkdb/docs +# hash: "3d13a937cfdacb7ffa3dab2ce3ebdf25bd3c192e" + +import rethinkdb + +docsSource = [ + + (rethinkdb.net.Connection.close, b'conn.close(noreply_wait=True)\n\nClose an open connection.\n\nClosing a connection normally waits until all outstanding requests have finished and then frees any open resources associated with the connection. By passing `False` to the `noreply_wait` optional argument, the connection will be closed immediately, possibly aborting any outstanding noreply writes.\n\nA noreply query is executed by passing the `noreply` option to the [run](http://rethinkdb.com/api/python/run/) command, indicating that `run()` should not wait for the query to complete before returning. You may also explicitly wait for a noreply query to complete by using the [noreply_wait](http://rethinkdb.com/api/python/noreply_wait) command.\n\n*Example* Close an open connection, waiting for noreply writes to finish.\n\n conn.close()\n\n*Example* Close an open connection immediately.\n\n conn.close(noreply_wait=False)\n'), + (rethinkdb.connect, b'r.connect(host="localhost", port=28015, db="test", auth_key="", timeout=20) -> connection\nr.connect(host) -> connection\n\nCreate a new connection to the database server. The keyword arguments are:\n\n- `host`: host of the RethinkDB instance. The default value is `localhost`.\n- `port`: the driver port, by default `28015`.\n- `db`: the database used if not explicitly specified in a query, by default `test`.\n- `auth_key`: the authentication key, by default the empty string.\n- `timeout`: timeout period in seconds for the connection to be opened (default `20`).\n\nIf the connection cannot be established, a `RqlDriverError` exception will be thrown.\n\nThe authentication key can be set from the RethinkDB command line tool. Once set, client connections must provide the key as an option to `run` in order to make the connection. For more information, read "Using the RethinkDB authentication system" in the documentation on [securing your cluster](http://rethinkdb.com/docs/security/).\n\n__Note:__ Currently, the Python driver is not thread-safe. Each thread or multiprocessing PID should be given its own connection object. (This is likely to change in a future release of RethinkDB; you can track issue [#2427](https://github.com/rethinkdb/rethinkdb/issues/2427) for progress.)\n\n*Example* Opens a connection using the default host and port but specifying the default database.\n\n conn = r.connect(db=\'marvel\')\n\n*Example* Opens a new connection to the database.\n\n conn = r.connect(host = \'localhost\',\n port = 28015,\n db = \'heroes\',\n auth_key = \'hunter2\')\n\n'), + (rethinkdb.net.Connection.noreply_wait, b'conn.noreply_wait()\n\n`noreply_wait` ensures that previous queries with the `noreply` flag have been processed\nby the server. Note that this guarantee only applies to queries run on the given connection.\n\n*Example* We have previously run queries with the `noreply` argument set to `True`. Now\nwait until the server has processed them.\n\n conn.noreply_wait()\n\n'), + (rethinkdb, b'r -> r\n\nThe top-level ReQL namespace.\n\n*Example* Setup your top-level namespace.\n\n import rethinkdb as r\n\n'), + (rethinkdb.net.Connection.reconnect, b'conn.reconnect(noreply_wait=True)\n\nClose and reopen a connection.\n\nClosing a connection normally waits until all outstanding requests have finished and then frees any open resources associated with the connection. By passing `False` to the `noreply_wait` optional argument, the connection will be closed immediately, possibly aborting any outstanding noreply writes.\n\nA noreply query is executed by passing the `noreply` option to the [run](http://rethinkdb.com/api/python/run/) command, indicating that `run()` should not wait for the query to complete before returning. You may also explicitly wait for a noreply query to complete by using the [noreply_wait](http://rethinkdb.com/api/python/noreply_wait) command.\n\n*Example* Cancel outstanding requests/queries that are no longer needed.\n\n conn.reconnect(noreply_wait=False)\n'), + (rethinkdb.net.Connection.repl, b"conn.repl()\n\nSet the default connection to make REPL use easier. Allows calling\n`.run()` on queries without specifying a connection.\n\n__Note:__ Avoid using `repl` in application code. RethinkDB connection objects are not thread-safe, and calls to `connect` from multiple threads may change the global connection object used by `repl`. Applications should specify connections explicitly.\n\n*Example* Set the default connection for the REPL, then call\n`run()` without specifying the connection.\n\n r.connect(db='marvel').repl()\n r.table('heroes').run()\n"), + (rethinkdb.ast.RqlQuery.run, b'query.run(conn, use_outdated=False, time_format=\'native\', profile=False, durability="hard") -> cursor\nquery.run(conn, use_outdated=False, time_format=\'native\', profile=False, durability="hard") -> object\n\nRun a query on a connection, returning either a single JSON result or\na cursor, depending on the query.\n\nThe optional arguments are:\n\n- `use_outdated`: whether or not outdated reads are OK (default: `False`).\n- `time_format`: what format to return times in (default: `\'native\'`).\n Set this to `\'raw\'` if you want times returned as JSON objects for exporting.\n- `profile`: whether or not to return a profile of the query\'s\n execution (default: `False`).\n- `durability`: possible values are `\'hard\'` and `\'soft\'`. In soft durability mode RethinkDB\nwill acknowledge the write immediately after receiving it, but before the write has\nbeen committed to disk.\n- `group_format`: what format to return `grouped_data` and `grouped_streams` in (default: `\'native\'`).\n Set this to `\'raw\'` if you want the raw pseudotype.\n- `noreply`: set to `True` to not receive the result object or cursor and return immediately.\n- `db`: the database to run this query against as a string. The default is the database specified in the `db` parameter to [connect](http://rethinkdb.com/api/python/connect/) (which defaults to `test`). The database may also be specified with the [db](http://rethinkdb.com/api/python/db/) command.\n- `array_limit`: the maximum numbers of array elements that can be returned by a query (default: 100,000). This affects all ReQL commands that return arrays. Note that it has no effect on the size of arrays being _written_ to the database; those always have an upper limit of 100,000 elements.\n- `binary_format`: what format to return binary data in (default: `\'native\'`). Set this to `\'raw\'` if you want the raw pseudotype.\n- `min_batch_rows`: minimum number of rows to wait for before batching a result set (default: 8). This is an integer.\n- `max_batch_rows`: maximum number of rows to wait for before batching a result set (default: unlimited). This is an integer.\n- `max_batch_bytes`: maximum number of bytes to wait for before batching a result set (default: 1024). This is an integer.\n- `max_batch_seconds`: maximum number of seconds to wait before batching a result set (default: 0.5). This is a float (not an integer) and may be specified to the microsecond.\n- `first_batch_scaledown_factor`: factor to scale the other parameters down by on the first batch (default: 4). For example, with this set to 8 and `max_batch_rows` set to 80, on the first batch `max_batch_rows` will be adjusted to 10 (80 / 8). This allows the first batch to return faster.\n\n*Example* Run a query on the connection `conn` and print out every\nrow in the result.\n\n for doc in r.table(\'marvel\').run(conn):\n print doc\n\n*Example* If you are OK with potentially out of date data from all\nthe tables involved in this query and want potentially faster reads,\npass a flag allowing out of date data in an options object. Settings\nfor individual tables will supercede this global setting for all\ntables in the query.\n\n r.table(\'marvel\').run(conn, use_outdated=True)\n\n*Example* If you just want to send a write and forget about it, you\ncan set `noreply` to true in the options. In this case `run` will\nreturn immediately.\n\n r.table(\'marvel\').run(conn, noreply=True)\n\n*Example* If you want to specify whether to wait for a write to be\nwritten to disk (overriding the table\'s default settings), you can set\n`durability` to `\'hard\'` or `\'soft\'` in the options.\n\n r.table(\'marvel\')\n .insert({ \'superhero\': \'Iron Man\', \'superpower\': \'Arc Reactor\' })\n .run(conn, noreply=True, durability=\'soft\')\n\n*Example* If you do not want a time object to be converted to a\nnative date object, you can pass a `time_format` flag to prevent it\n(valid flags are "raw" and "native"). This query returns an object\nwith two fields (`epoch_time` and `$reql_type$`) instead of a native date\nobject.\n\n r.now().run(conn, time_format="raw")\n\n*Example* Specify the database to use for the query.\n\n for doc in r.table(\'marvel\').run(conn, db=\'heroes\'):\n print doc\n\nThis is equivalent to using the `db` command to specify the database:\n\n r.db(\'heroes\').table(\'marvel\').run(conn) ...\n\n*Example* Change the batching parameters for this query.\n\n r.table(\'marvel\').run(conn, max_batch_rows=16, max_batch_bytes=2048)\n'), + (rethinkdb.set_loop_type, b'r.set_loop_type(string)\n\nSet an asynchronous event loop model. Currently, the only event loop model RethinkDB supports is `"tornado"`, for use with the [Tornado web framework](http://www.tornadoweb.org). After setting the event loop to `"tornado"`, the [connect](http://rethinkdb.com/api/python/connect) and [run](http://rethinkdb.com/api/python/run) commands will return Tornado `Future` objects.\n\n*Example* Read a table\'s data using Tornado.\n\n r.set_loop_type("tornado")\n conn = r.connect(host=\'localhost\', port=28015)\n \n @gen.coroutine\n def use_cursor(conn):\n # Print every row in the table.\n cursor = yield r.table(\'test\').order_by(index="id").run(yield conn)\n while (yield cursor.fetch_next()):\n item = yield cursor.next()\n print(item)\n\nFor a longer discussion with Tornado examples, see the documentation article on [Asynchronous connections][ac].\n\n[ac]: /docs/async-connections/\n'), + (rethinkdb.net.Connection.use, b"conn.use(db_name)\n\nChange the default database on this connection.\n\n*Example* Change the default database so that we don't need to\nspecify the database when referencing a table.\n\n conn.use('marvel')\n r.table('heroes').run(conn) # refers to r.db('marvel').table('heroes')\n"), + (rethinkdb.ast.Table.config, b'table.config() -> selection<object>\ndatabase.config() -> selection<object>\n\nQuery (read and/or update) the configurations for individual tables or databases.\n\nThe `config` command is a shorthand way to access the `table_config` or `db_config` [System tables](http://rethinkdb.com/docs/system-tables/). It will return the single row from the system that corresponds to the database or table configuration, as if [get](http://rethinkdb.com/api/python/get) had been called on the system table with the UUID of the database or table in question.\n\n*Example* Get the configuration for the `users` table.\n\n r.table(\'users\').config().run(conn)\n \n {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "users",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "a", "replicas": ["a", "b"]},\n {"primary_replica": "d", "replicas": ["c", "d"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n\n*Example* Change the write acknowledgement requirement of the `users` table.\n\n r.table(\'users\').config().update({\'write_acks\': \'single\'}).run(conn)\n'), + (rethinkdb.ast.Table.rebalance, b'table.rebalance() -> object\ndatabase.rebalance() -> object\n\nRebalances the shards of a table. When called on a database, all the tables in that database will be rebalanced.\n\nThe `rebalance` command operates by measuring the distribution of primary keys within a table and picking split points that will give each shard approximately the same number of documents. It won\'t change the number of shards within a table, or change any other configuration aspect for the table or the database.\n\nA table will lose availability temporarily after `rebalance` is called; use the [wait](http://rethinkdb.com/api/python/wait) command to wait for the table to become available again, or [status](http://rethinkdb.com/api/python/status) to check if the table is available for writing.\n\nRethinkDB automatically rebalances tables when the number of shards are increased, and as long as your documents have evenly distributed primary keys—such as the default UUIDs—it is rarely necessary to call `rebalance` manually. Cases where `rebalance` may need to be called include:\n\n* Tables with unevenly distributed primary keys, such as incrementing integers\n* Changing a table\'s primary key type\n* Increasing the number of shards on an empty table, then using non-UUID primary keys in that table\n\nThe [web UI](http://rethinkdb.com/docs/administration-tools/) (and the [info](http://rethinkdb.com/api/python/info) command) can be used to tell you when a table\'s shards need to be rebalanced.\n\nThe return value of `rebalance` is an object with two fields:\n\n* `rebalanced`: the number of tables rebalanced.\n* `status_changes`: a list of new and old table status values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `rebalance` was executed. \n * `new_val`: The table\'s `status` value after `rebalance` was executed. (This value will almost always indicate the table is unavailable.)\n\nSee the [status](http://rethinkdb.com/api/python/status) command for an explanation of the objects returned in the `old_val` and `new_val` fields.\n\n*Example* Rebalance a table.\n\n r.table(\'superheroes\').rebalance().run(conn)\n \n {\n "rebalanced": 1,\n "status_changes": [\n {\n "old_val": {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": True,\n "ready_for_outdated_reads": True,\n "ready_for_reads": True,\n "ready_for_writes": True\n }\n },\n "new_val": {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "transitioning"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "transitioning"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": False,\n "ready_for_outdated_reads": False,\n "ready_for_reads": False,\n "ready_for_writes": False\n }\n }\n \n }\n ]\n }\n'), + (rethinkdb.ast.Table.reconfigure, b'table.reconfigure(shards=, replicas=[, primary_replica_tag=, dry_run=False]) -> object\ndatabase.reconfigure(shards=, replicas=[, primary_replica_tag=, dry_run=False]) -> object\n\nReconfigure a table\'s sharding and replication.\n\n* `shards`: the number of shards, an integer from 1-32. Required.\n* `replicas`: either an integer or a mapping object. Required.\n * If `replicas` is an integer, it specifies the number of replicas per shard. Specifying more replicas than there are servers will return an error.\n * If `replicas` is an object, it specifies key-value pairs of server tags and the number of replicas to assign to those servers: `{"tag1": 2, "tag2": 4, "tag3": 2, ...}`. For more information about server tags, read [Administration tools](http://rethinkdb.com/docs/administration-tools/).\n* `primary_replica_tag`: the primary server specified by its server tag. Required if `replicas` is an object; the tag must be in the object. This must *not* be specified if `replicas` is an integer.\n* `dry_run`: if `True` the generated configuration will not be applied to the table, only returned.\n\nThe return value of `reconfigure` is an object with three fields:\n\n* `reconfigured`: the number of tables reconfigured. This will be `0` if `dry_run` is `True`.\n* `config_changes`: a list of new and old table configuration values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [config](http://rethinkdb.com/api/python/config) value before `reconfigure` was executed. \n * `new_val`: The table\'s `config` value after `reconfigure` was executed.\n* `status_changes`: a list of new and old table status values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `reconfigure` was executed. \n * `new_val`: The table\'s `status` value after `reconfigure` was executed.\n\nFor `config_changes` and `status_changes`, see the [config](http://rethinkdb.com/api/python/config) and [status](http://rethinkdb.com/api/python/status) commands for an explanation of the objects returned in the `old_val` and `new_val` fields.\n\nA table will lose availability temporarily after `reconfigure` is called; use the [table_status](http://rethinkdb.com/api/python/table_status) command to determine when the table is available again.\n\n**Note:** Whenever you call `reconfigure`, the write durability will be set to `hard` and the write acknowledgments will be set to `majority`; these can be changed by using the `config` command on the table.\n\nIf `reconfigure` is called on a database, all the tables in the database will have their configurations affected. The return value will be an array of the objects described above, one per table.\n\nRead [Sharding and replication](http://rethinkdb.com/docs/sharding-and-replication/) for a complete discussion of the subject, including advanced topics.\n\n*Example* Reconfigure a table.\n\n r.table(\'superheroes\').reconfigure(shards=2, replicas=1).run(conn)\n \n {\n "reconfigured": 1,\n "config_changes": [\n {\n "new_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "jeeves", "replicas": ["jeeves"]},\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n },\n "old_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n }\n ],\n "status_changes": [\n {\n "new_val": (status object),\n "old_val": (status object)\n }\n ]\n }\n\n*Example* Reconfigure a table, specifying replicas by server tags.\n\n r.table(\'superheroes\').reconfigure(shards=2, replicas={\'wooster\': 1, \'wayne\': 1}, primary_replica_tag=\'wooster\').run(conn)\n \n {\n "reconfigured": 1,\n "config_changes": [\n {\n "new_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "jeeves", "replicas": ["jeeves", "alfred"]},\n {"primary_replica": "jeeves", "replicas": ["jeeves", "alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n },\n "old_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n }\n ],\n "status_changes": [\n {\n "new_val": (status object),\n "old_val": (status object)\n }\n ]\n }\n'), + (rethinkdb.ast.Table.status, b'table.status() -> selection<object>\n\nReturn the status of a table.\n\nThe return value is an object providing information about the table\'s shards, replicas and replica readiness states. For a more complete discussion of the object fields, read about the `table_status` table in [System tables](http://rethinkdb.com/docs/system-tables/).\n\n* `db`: database name.\n* `name`: table name.\n* `id`: table UUID.\n* `shards`: an array of objects, one for each shard, with the following keys per object:\n * `primary_replica`: name of the shard\'s primary server.\n * `replicas`: an array of objects showing the status of each replica, with the following keys:\n * `server`: name of the replica server.\n * `state`: one of `ready`, `disconnected`, `backfilling_data`, `offloading_data`, `erasing_data`, `looking_for_primary_replica` or `transitioning`.\n* `status`: an object with the following boolean keys:\n * `all_replicas_ready`: `True` if all backfills have finished.\n * `ready_for_outdated_reads`: `True` if the table is ready for read queries with the `use_outdated` flag set to `True`.\n * `ready_for_reads`: `True` if the table is ready for read queries with current data (with the `use_outdated` flag set to `False` or unspecified).\n * `ready_for_writes`: `True` if the table is ready for write queries.\n\n*Example* Get a table\'s status.\n\n r.table(\'superheroes\').status().run(conn)\n \n {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": True,\n "ready_for_outdated_reads": True,\n "ready_for_reads": True,\n "ready_for_writes": True\n }\n }\n'), + (rethinkdb.ast.Table.wait, b'table.wait([wait_for=\'ready_for_writes\', timeout=]) -> object\ndatabase.wait([wait_for=\'ready_for_writes\', timeout=]) -> object\nr.wait([wait_for=\'ready_for_writes\', timeout=]) -> object\n\nWait for a table or all the tables in a database to be ready. A table may be temporarily unavailable after creation, rebalancing or reconfiguring. The `wait` command blocks until the given table (or database) is fully up to date.\n\nThe `wait` command takes two optional arguments:\n\n* `wait_for`: a string indicating a table [status](http://rethinkdb.com/api/python/status) to wait on before returning, one of `ready_for_outdated_reads`, `ready_for_reads`, `ready_for_writes`, or `all_replicas_ready`. The default is `ready_for_writes`. \n* `timeout`: a number indicating maximum time to wait for in seconds before returning. The default is no timeout.\n\nThe return value is an object consisting of two key/value pairs:\n\n* `ready`: an integer indicating the number of tables waited for. It will always be `1` when `wait` is called on a table, and the total number of tables when called on a database.\n* `status_changes`: a list with one entry for each of the tables. Each member of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `wait` was executed. \n * `new_val`: The table\'s `status` value after `wait` finished.\n\nSee [status](http://rethinkdb.com/api/python/status) and [System tables](http://rethinkdb.com/docs/system-tables/) for a description of the fields within `status_changes`.\n\nIf `wait` is called with no table or database specified (the `r.wait()` form), it will wait on all the tables in the default database (set with the [connect](http://rethinkdb.com/api/python/connect/) command\'s `db` parameter, which defaults to `test`).\n\n*Example* Wait on a table to be ready.\n\n r.table(\'superheroes\').wait().run(conn)\n \n {\n "ready": 1,\n "status_changes": [\n \t{\n \t "old_val": {\n \t\t"db": "database",\n \t\t"id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n \t\t"name": "superheroes",\n \t\t"shards": [\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t },\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t }\n \t\t],\n \t\t"status": {\n \t\t "all_replicas_ready": True,\n \t\t "ready_for_outdated_reads": True,\n \t\t "ready_for_reads": True,\n \t\t "ready_for_writes": True\n \t\t}\n \t },\n \t "new_val": {\n \t\t"db": "database",\n \t\t"id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n \t\t"name": "superheroes",\n \t\t"shards": [\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t },\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t }\n \t\t],\n \t\t"status": {\n \t\t "all_replicas_ready": True,\n \t\t "ready_for_outdated_reads": True,\n \t\t "ready_for_reads": True,\n \t\t "ready_for_writes": True\n \t\t}\n \t }\n \t}\n ]\n }\n'), + (rethinkdb.ast.RqlQuery.avg, b"sequence.avg([field_or_function]) -> number\n\nAverages all the elements of a sequence. If called with a field name,\naverages all the values of that field in the sequence, skipping\nelements of the sequence that lack that field. If called with a\nfunction, calls that function on every element of the sequence and\naverages the results, skipping elements of the sequence where that\nfunction returns `None` or a non-existence error.\n\nProduces a non-existence error when called on an empty sequence. You\ncan handle this case with `default`.\n\n*Example* What's the average of 3, 5, and 7?\n\n r.expr([3, 5, 7]).avg().run(conn)\n\n*Example* What's the average number of points scored in a game?\n\n r.table('games').avg('points').run(conn)\n\n*Example* What's the average number of points scored in a game,\ncounting bonus points?\n\n r.table('games').avg(lambda game:\n game['points'] + game['bonus_points']\n ).run(conn)\n\n*Example* What's the average number of points scored in a game?\n(But return `None` instead of raising an error if there are no games where\npoints have been scored.)\n\n r.table('games').avg('points').default(None).run(conn)\n"), + (rethinkdb.ast.RqlQuery.contains, b"sequence.contains(value|predicate[, value|predicate, ...]) -> bool\n\nWhen called with values, returns `True` if a sequence contains all the\nspecified values. When called with predicate functions, returns `True`\nif for each predicate there exists at least one element of the stream\nwhere that predicate returns `True`.\n\nValues and predicates may be mixed freely in the argument list.\n\n*Example* Has Iron Man ever fought Superman?\n\n r.table('marvel').get('ironman')['opponents'].contains('superman').run(conn)\n\n*Example* Has Iron Man ever defeated Superman in battle?\n\n r.table('marvel').get('ironman')['battles'].contains(lambda battle:\n (battle['winner'] == 'ironman') & (battle['loser'] == 'superman')\n ).run(conn)\n\n*Example* Use `contains` with a predicate function to simulate an `or`. Return the Marvel superheroes who live in Detroit, Chicago or Hoboken.\n\n r.table('marvel').filter(\n lambda hero: r.expr(['Detroit', 'Chicago', 'Hoboken']).contains(hero['city'])\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.count, b"sequence.count([value_or_predicate]) -> number\nbinary.count() -> number\n\nCounts the number of elements in a sequence. If called with a value,\ncounts the number of times that value occurs in the sequence. If\ncalled with a predicate function, counts the number of elements in the\nsequence where that function returns `True`.\n\nIf `count` is called on a [binary](http://rethinkdb.com/api/python/binary) object, it will return the size of the object in bytes.\n\n*Example* Count the number of users.\n\n r.table('users').count().run(conn)\n\n*Example* Count the number of 18 year old users.\n\n r.table('users')['age'].count(18).run(conn)\n\n*Example* Count the number of users over 18.\n\n r.table('users')['age'].count(lambda age: age > 18).run(conn)\n\n r.table('users').count(lambda user: user['age'] > 18).run(conn)\n"), + (rethinkdb.ast.RqlQuery.distinct, b"sequence.distinct() -> array\ntable.distinct([index=]) -> stream\n\nRemoves duplicate elements from a sequence.\n\nThe `distinct` command can be called on any sequence or table with an index.\n\n{% infobox %}\nWhile `distinct` can be called on a table without an index, the only effect will be to convert the table into a stream; the content of the stream will not be affected.\n{% endinfobox %}\n\n*Example* Which unique villains have been vanquished by Marvel heroes?\n\n r.table('marvel').concat_map(\n lambda hero: hero['villain_list']).distinct().run(conn)\n\n*Example* Topics in a table of messages have a secondary index on them, and more than one message can have the same topic. What are the unique topics in the table?\n\n r.table('messages').distinct(index='topics').run(conn)\n\nThe above structure is functionally identical to:\n\n r.table('messages')['topics'].distinct().run(conn)\n\nHowever, the first form (passing the index as an argument to `distinct`) is faster, and won't run into array limit issues since it's returning a stream.\n"), + (rethinkdb.ast.RqlQuery.group, b'sequence.group(field_or_function..., [index=\'index_name\', multi=False]) -> grouped_stream\n\nTakes a stream and partitions it into multiple groups based on the\nfields or functions provided.\n\nWith the `multi` flag single documents can be assigned to multiple groups, similar to the behavior of [multi-indexes](http://rethinkdb.com/docs/secondary-indexes/python). When `multi` is `True` and the grouping value is an array, documents will be placed in each group that corresponds to the elements of the array. If the array is empty the row will be ignored.\n\n*Example* Grouping games by player.\n\nSuppose that the table `games` has the following data:\n\n [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ]\n\nGrouping games by player can be done with:\n\n > r.table(\'games\').group(\'player\').run(conn)\n \n {\n "Alice": [\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ],\n "Bob": [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n\nCommands chained after `group` will be called on each of these grouped\nsub-streams, producing grouped data.\n\n*Example* What is each player\'s best game?\n\n > r.table(\'games\').group(\'player\').max(\'points\').run(conn)\n \n {\n "Alice": {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n "Bob": {"id": 2, "player": "Bob", "points": 15, "type": "ranked"}\n }\n\nCommands chained onto grouped data will operate on each grouped datum,\nproducing more grouped data.\n\n*Example* What is the maximum number of points scored by each player?\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].run(conn)\n \n {\n "Alice": 7,\n "Bob": 15\n }\n\nYou can also group by more than one field.\n\n*Example* What is the maximum number of points scored by each\nplayer for each game type?\n\n > r.table(\'games\').group(\'player\', \'type\').max(\'points\')[\'points\'].run(conn)\n \n {\n ("Alice", "free"): 7,\n ("Bob", "free"): 10,\n ("Bob", "ranked"): 15\n }\n\nYou can also group by a function.\n\n*Example* What is the maximum number of points scored by each\nplayer for each game type?\n\n > r.table(\'games\')\n .group(lambda game:\n game.pluck(\'player\', \'type\')\n ).max(\'points\')[\'points\'].run(conn)\n \n {\n frozenset([(\'player\', \'Alice\'), (\'type\', \'free\')]): 7,\n frozenset([(\'player\', \'Bob\'), (\'type\', \'free\')]): 10,\n frozenset([(\'player\', \'Bob\'), (\'type\', \'ranked\')]): 15,\n }\n\nUsing a function, you can also group by date on a ReQL [date field](http://rethinkdb.com/docs/dates-and-times/javascript/).\n\n*Example* How many matches have been played this year by month?\n\n > r.table(\'matches\').group(\n lambda match: [match[\'date\'].year(), match[\'date\'].month()]\n ).count().run(conn)\n \n {\n (2014, 2): 2,\n (2014, 3): 2,\n (2014, 4): 1,\n (2014, 5): 3\n }\n\nYou can also group on an index (primary key or secondary).\n\n*Example* What is the maximum number of points scored by game type?\n\n > r.table(\'games\').group(index=\'type\').max(\'points\')[\'points\'].run(conn)\n \n {\n "free": 10,\n "ranked": 15\n }\n\nSuppose that the table `games2` has the following data:\n\n [\n { \'id\': 1, \'matches\': {\'a\': [1, 2, 3], \'b\': [4, 5, 6]} },\n { \'id\': 2, \'matches\': {\'b\': [100], \'c\': [7, 8, 9]} },\n { \'id\': 3, \'matches\': {\'a\': [10, 20], \'c\': [70, 80]} }\n ]\n\nUsing the `multi` option we can group data by match A, B or C.\n\n > r.table(\'games2\').group(r.row[\'matches\'].keys(), multi=True).run(conn)\n \n [\n {\n \'group\': \'a\',\n \'reduction\': [ , ]\n },\n {\n \'group\': \'b\',\n \'reduction\': [ , ]\n },\n {\n \'group\': \'c\',\n \'reduction\': [ , ]\n }\n ]\n\n(The full result set is abbreviated in the figure; `, ` and `` would be the entire documents matching those keys.)\n\n*Example* Use [map](http://rethinkdb.com/api/python/map) and [sum](http://rethinkdb.com/api/python/sum) to get the total points scored for each match.\n\n r.table(\'games2\').group(r.row[\'matches\'].keys(), multi=True).ungroup().map(\n lambda doc: { \'match\': doc[\'group\'], \'total\': doc[\'reduction\'].sum(\n lambda set: set[\'matches\'][doc[\'group\']].sum()\n )}).run(conn)\n \n [\n { \'match\': \'a\', \'total\': 36 },\n { \'match\': \'b\', \'total\': 115 },\n { \'match\': \'c\', \'total\': 174 }\n ]\n\nThe inner `sum` adds the scores by match within each document; the outer `sum` adds those results together for a total across all the documents.\n\nIf you want to operate on all the groups rather than operating on each\ngroup (e.g. if you want to order the groups by their reduction), you\ncan use [ungroup](http://rethinkdb.com/api/python/ungroup/) to turn a grouped stream or\ngrouped data into an array of objects representing the groups.\n\n*Example* Ungrouping grouped data.\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].ungroup().run(conn)\n \n [\n {\n "group": "Alice",\n "reduction": 7\n },\n {\n "group": "Bob",\n "reduction": 15\n }\n ]\n\nUngrouping is useful e.g. for ordering grouped data, or for inserting\ngrouped data into a table.\n\n*Example* What is the maximum number of points scored by each\nplayer, with the highest scorers first?\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].ungroup().order_by(\n r.desc(\'reduction\')).run(conn)\n \n [\n {\n "group": "Bob",\n "reduction": 15\n },\n {\n "group": "Alice",\n "reduction": 7\n }\n ]\n\nWhen grouped data are returned to the client, they are transformed\ninto a client-specific native type. (Something similar is done with\n[times](http://rethinkdb.com/docs/dates-and-times/).) In Python, grouped data are\ntransformed into a `dictionary`. If the group value is an `array`, the\nkey is converted to a `tuple`. If the group value is a `dictionary`,\nit will be converted to a `frozenset`.\n\nIf you instead want to receive the raw\npseudotype from the server (e.g. if you\'re planning to serialize the\nresult as JSON), you can specify `group_format: \'raw\'` as an optional\nargument to `run`:\n\n*Example* Get back the raw `GROUPED_DATA` pseudotype.\n\n > r.table(\'games\').group(\'player\').avg(\'points\').run(conn, group_format=\'raw\')\n \n {\n "$reql_type$": "GROUPED_DATA",\n "data": [\n ["Alice", 4.5],\n ["Bob", 12.5]\n ]\n }\n\nNot passing the `group_format` flag would return:\n\n {\n "Alice": 4.5,\n "Bob": 12.5\n }\n\nYou might also want to use the [ungroup](http://rethinkdb.com/api/python/ungroup/)\ncommand (see above), which will turn the grouped data into an array of\nobjects on the server.\n\nIf you run a query that returns a grouped stream, it will be\nautomatically converted to grouped data before being sent back to you\n(there is currently no efficient way to stream groups from RethinkDB).\nThis grouped data is subject to the array size limit (see [run](http://rethinkdb.com/api/python/run)).\n\nIn general, operations on grouped streams will be efficiently\ndistributed, and operations on grouped data won\'t be. You can figure\nout what you\'re working with by putting `type_of` on the end of your\nquery. Below are efficient and inefficient examples.\n\n*Example* Efficient operation.\n\n # r.table(\'games\').group(\'player\').type_of().run(conn)\n # Returns "GROUPED_STREAM"\n r.table(\'games\').group(\'player\').min(\'points\').run(conn) # EFFICIENT\n\n*Example* Inefficient operation.\n\n # r.table(\'games\').group(\'player\').order_by(\'score\').type_of().run(conn)\n # Returns "GROUPED_DATA"\n r.table(\'games\').group(\'player\').order_by(\'score\').nth(0).run(conn) # INEFFICIENT\n\nWhat does it mean to be inefficient here? When operating on grouped\ndata rather than a grouped stream, *all* of the data has to be\navailable on the node processing the query. This means that the\noperation will only use one server\'s resources, and will require\nmemory proportional to the size of the grouped data it\'s operating\non. (In the case of the `order_by` in the inefficient example, that\nmeans memory proportional **to the size of the table**.) The array\nlimit is also enforced for grouped data, so the `order_by` example\nwould fail for tables with more than 100,000 rows unless you used the `array_limit` option with `run`.\n\n*Example* What is the maximum number of points scored by each\nplayer in free games?\n\n > r.table(\'games\').filter(lambda game:\n game[\'type\'] = \'free\'\n ).group(\'player\').max(\'points\')[\'points\'].run(conn)\n \n {\n "Alice": 7,\n "Bob": 10\n }\n\n*Example* What is each player\'s highest even and odd score?\n\n > r.table(\'games\')\n .group(\'name\', lambda game:\n game[\'points\'] % 2\n ).max(\'points\')[\'points\'].run(conn)\n \n {\n ("Alice", 1): 7,\n ("Bob", 0): 10,\n ("Bob", 1): 15\n }\n'), + (rethinkdb.ast.RqlQuery.max, b"sequence.max(field_or_function) -> element\nsequence.max(index='index') -> element\n\nFinds the maximum element of a sequence. The `max` command can be called with:\n\n* a **field name**, to return the element of the sequence with the largest value in that field;\n* an **index** (the primary key or a secondary index), to return the element of the sequence with the largest value in that index;\n* a **function**, to apply the function to every element within the sequence and return the element which returns the largest value from the function, ignoring any elements where the function produces a non-existence error.\n\nFor more information on RethinkDB's sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nCalling `max` on an empty sequence will throw a non-existence error; this can be handled using the [default](http://rethinkdb.com/api/python/default/) command.\n\n*Example* Return the maximum value in the list `[3, 5, 7]`.\n\n r.expr([3, 5, 7]).max().run(conn)\n\n*Example* Return the user who has scored the most points.\n\n r.table('users').max('points').run(conn)\n\n*Example* The same as above, but using a secondary index on the `points` field.\n\n r.table('users').max(index='points').run(conn)\n\n*Example* Return the user who has scored the most points, adding in bonus points from a separate field using a function.\n\n r.table('users').max(lambda user:\n user['points'] + user['bonus_points']\n ).run(conn)\n\n*Example* Return the highest number of points any user has ever scored. This returns the value of that `points` field, not a document.\n\n r.table('users').max('points')['points'].run(conn)\n\n*Example* Return the user who has scored the most points, but add a default `None` return value to prevent an error if no user has ever scored points.\n\n r.table('users').max('points').default(None).run(conn)\n"), + (rethinkdb.ast.RqlQuery.min, b"sequence.min(field_or_function) -> element\nsequence.min(index='index') -> element\n\nFinds the minimum element of a sequence. The `min` command can be called with:\n\n* a **field name**, to return the element of the sequence with the smallest value in that field;\n* an **index** (the primary key or a secondary index), to return the element of the sequence with the smallest value in that index;\n* a **function**, to apply the function to every element within the sequence and return the element which returns the smallest value from the function, ignoring any elements where the function produces a non-existence error.\n\nFor more information on RethinkDB's sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nCalling `min` on an empty sequence will throw a non-existence error; this can be handled using the [default](http://rethinkdb.com/api/python/default/) command.\n\n*Example* Return the minimum value in the list `[3, 5, 7]`.\n\n r.expr([3, 5, 7]).min().run(conn)\n\n*Example* Return the user who has scored the fewest points.\n\n r.table('users').min('points').run(conn)\n\n*Example* The same as above, but using a secondary index on the `points` field.\n\n r.table('users').min(index='points').run(conn)\n\n*Example* Return the user who has scored the fewest points, adding in bonus points from a separate field using a function.\n\n r.table('users').min(lambda user:\n user['points'] + user['bonus_points']\n ).run(conn)\n\n*Example* Return the smallest number of points any user has ever scored. This returns the value of that `points` field, not a document.\n\n r.table('users').min('points')['points'].run(conn)\n\n*Example* Return the user who has scored the fewest points, but add a default `None` return value to prevent an error if no user has ever scored points.\n\n r.table('users').min('points').default(None).run(conn)\n"), + (rethinkdb.ast.RqlQuery.reduce, b'sequence.reduce(reduction_function) -> value\n\nProduce a single value from a sequence through repeated application of a reduction\nfunction. \nThe reduction function can be called on:\n\n- two elements of the sequence\n- one element of the sequence and one result of a previous reduction\n- two results of previous reductions\n\nThe reduction function can be called on the results of two previous reductions because the\n`reduce` command is distributed and parallelized across shards and CPU cores. A common\nmistaken when using the `reduce` command is to suppose that the reduction is executed\nfrom left to right. Read the [map-reduce in RethinkDB](http://rethinkdb.com/docs/map-reduce/) article to\nsee an example.\n\nIf the sequence is empty, the server will produce a `RqlRuntimeError` that can be\ncaught with `default`. \nIf the sequence has only one element, the first element will be returned.\n\n*Example* Return the number of documents in the table `posts`.\n\n r.table("posts").map(lambda doc: 1)\n .reduce(lambda left, right: left+right)\n .default(0).run(conn)\n\nA shorter way to execute this query is to use [count](http://rethinkdb.com/api/python/count).\n\n*Example* Suppose that each `post` has a field `comments` that is an array of\ncomments. \nReturn the number of comments for all posts.\n\n r.table("posts").map(lambda doc:\n doc["comments"].count()\n ).reduce(lambda left, right:\n left+right\n ).default(0).run(conn)\n\n*Example* Suppose that each `post` has a field `comments` that is an array of\ncomments. \nReturn the maximum number comments per post.\n\n r.table("posts").map(lambda doc:\n doc["comments"].count()\n ).reduce(lambda left, right:\n r.branch(\n left > right,\n left,\n right\n )\n ).default(0).run(conn)\n\nA shorter way to execute this query is to use [max](http://rethinkdb.com/api/python/max).\n'), + (rethinkdb.ast.RqlQuery.sum, b"sequence.sum([field_or_function]) -> number\n\nSums all the elements of a sequence. If called with a field name,\nsums all the values of that field in the sequence, skipping elements\nof the sequence that lack that field. If called with a function,\ncalls that function on every element of the sequence and sums the\nresults, skipping elements of the sequence where that function returns\n`None` or a non-existence error.\n\nReturns `0` when called on an empty sequence.\n\n*Example* What's 3 + 5 + 7?\n\n r.expr([3, 5, 7]).sum().run(conn)\n\n*Example* How many points have been scored across all games?\n\n r.table('games').sum('points').run(conn)\n\n*Example* How many points have been scored across all games,\ncounting bonus points?\n\n r.table('games').sum(lambda game:\n game['points'] + game['bonus_points']\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.ungroup, b'grouped_stream.ungroup() -> array\ngrouped_data.ungroup() -> array\n\nTakes a grouped stream or grouped data and turns it into an array of\nobjects representing the groups. Any commands chained after `ungroup`\nwill operate on this array, rather than operating on each group\nindividually. This is useful if you want to e.g. order the groups by\nthe value of their reduction.\n\nThe format of the array returned by `ungroup` is the same as the\ndefault native format of grouped data in the JavaScript driver and\ndata explorer.\n\n*Example* What is the maximum number of points scored by each\nplayer, with the highest scorers first?\n\nSuppose that the table `games` has the following data:\n\n [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ]\n\nWe can use this query:\n\n r.table(\'games\')\n .group(\'player\').max(\'points\')[\'points\']\n .ungroup().order_by(r.desc(\'reduction\')).run(conn)\n\nResult: \n\n [\n {\n "group": "Bob",\n "reduction": 15\n },\n {\n "group": "Alice",\n "reduction": 7\n }\n ]\n\n*Example* Select one random player and all their games.\n\n r.table(\'games\').group(\'player\').ungroup().sample(1).run(conn)\n\nResult:\n\n [\n {\n "group": "Bob",\n "reduction": [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n ]\n\nNote that if you didn\'t call `ungroup`, you would instead select one\nrandom game from each player:\n\n r.table(\'games\').group(\'player\').sample(1).run(conn)\n\nResult:\n\n {\n "Alice": [\n {"id": 5, "player": "Alice", "points": 7, "type": "free"}\n ],\n "Bob": [\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n\n*Example* Types!\n\n r.table(\'games\').group(\'player\').type_of().run(conn) # Returns "GROUPED_STREAM"\n r.table(\'games\').group(\'player\').ungroup().type_of().run(conn) # Returns "ARRAY"\n r.table(\'games\').group(\'player\').avg(\'points\').run(conn) # Returns "GROUPED_DATA"\n r.table(\'games\').group(\'player\').avg(\'points\').ungroup().run(conn) #Returns "ARRAY"\n'), + (rethinkdb.args, b"r.args(array) -> special\n\n`r.args` is a special term that's used to splice an array of arguments\ninto another term. This is useful when you want to call a variadic\nterm such as `get_all` with a set of arguments produced at runtime.\n\nThis is analogous to unpacking argument lists in Python.\n\n*Example* Get Alice and Bob from the table `people`.\n\n r.table('people').get_all('Alice', 'Bob').run(conn)\n # or\n r.table('people').get_all(r.args(['Alice', 'Bob'])).run(conn)\n\n*Example* Get all of Alice's children from the table `people`.\n\n # r.table('people').get('Alice') returns {'id': 'Alice', 'children': ['Bob', 'Carol']}\n r.table('people').get_all(r.args(r.table('people').get('Alice')['children'])).run(conn)\n"), + (rethinkdb.binary, b'r.binary(data) -> binary\n\nEncapsulate binary data within a query.\n\nThe type of data `binary` accepts depends on the client language. In Python, it expects a parameter of `bytes` type. Using a `bytes` object within a query implies the use of `binary` and the ReQL driver will automatically perform the coercion (in Python 3 only).\n\nBinary objects returned to the client in JavaScript will also be of the `bytes` type. This can be changed with the `binary_format` option provided to [run](http://rethinkdb.com/api/python/run) to return "raw" objects.\n\nOnly a limited subset of ReQL commands may be chained after `binary`:\n\n* [coerce_to](http://rethinkdb.com/api/python/coerce_to/) can coerce `binary` objects to `string` types\n* [count](http://rethinkdb.com/api/python/count/) will return the number of bytes in the object\n* [slice](http://rethinkdb.com/api/python/slice/) will treat bytes like array indexes (i.e., `slice(10,20)` will return bytes 10–19)\n* [type_of](http://rethinkdb.com/api/python/type_of) returns `PTYPE`\n* [info](http://rethinkdb.com/api/python/info) will return information on a binary object.\n\n*Example* Save an avatar image to a existing user record.\n\n f = open(\'./default_avatar.png\', \'rb\')\n avatar_image = f.read()\n f.close()\n r.table(\'users\').get(100).update({\'avatar\': r.binary(avatar_image)}).run(conn)\n\n*Example* Get the size of an existing avatar image.\n\n r.table(\'users\').get(100)[\'avatar\'].count().run(conn)\n \n 14156\n\nRead more details about RethinkDB\'s binary object support: [Storing binary objects](http://rethinkdb.com/docs/storing-binary/).\n'), + (rethinkdb.branch, b'r.branch(test, true_branch, false_branch) -> any\n\nIf the `test` expression returns `False` or `None`, the `false_branch` will be evaluated.\nOtherwise, the `true_branch` will be evaluated.\n \nThe `branch` command is effectively an `if` renamed due to language constraints.\n\n*Example* Return heroes and superheroes.\n\n r.table(\'marvel\').map(\n r.branch(\n r.row[\'victories\'] > 100,\n r.row[\'name\'] + \' is a superhero\',\n r.row[\'name\'] + \' is a hero\'\n )\n ).run(conn)\n\nIf the documents in the table `marvel` are:\n\n [{\n "name": "Iron Man",\n "victories": 214\n },\n {\n "name": "Jubilee",\n "victories": 9\n }]\n\nThe results will be:\n\n [\n "Iron Man is a superhero",\n "Jubilee is a hero"\n ]\n'), + (rethinkdb.ast.RqlQuery.coerce_to, b"sequence.coerce_to('array') -> array\nvalue.coerce_to('string') -> string\nstring.coerce_to('number') -> number\narray.coerce_to('object') -> object\nobject.coerce_to('array') -> array\nbinary.coerce_to('string') -> string\nstring.coerce_to('binary') -> binary\n\nConvert a value of one type into another.\n\n* a sequence, selection or object can be coerced to an array\n* an array of key-value pairs can be coerced to an object\n* a string can be coerced to a number\n* any datum (single value) can be coerced to a string\n* a binary object can be coerced to a string and vice-versa\n\n*Example* Coerce a stream to an array to store its output in a field. (A stream cannot be stored in a field directly.)\n\n r.table('posts').map(lambda post: post.merge(\n { 'comments': r.table('comments').get_all(post['id'], index='post_id').coerce_to('array') }\n )).run(conn)\n\n*Example* Coerce an array of pairs into an object.\n\n r.expr([['name', 'Ironman'], ['victories', 2000]]).coerce_to('object').run(conn)\n\n__Note:__ To coerce a list of key-value pairs like `['name', 'Ironman', 'victories', 2000]` to an object, use the [object](http://rethinkdb.com/api/python/object) command.\n\n*Example* Coerce a number to a string.\n\n r.expr(1).coerce_to('string').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.default, b'value.default(default_value) -> any\nsequence.default(default_value) -> any\n\nHandle non-existence errors. Tries to evaluate and return its first argument. If an\nerror related to the absence of a value is thrown in the process, or if its first\nargument returns `None`, returns its second argument. (Alternatively, the second argument\nmay be a function which will be called with either the text of the non-existence error\nor `None`.)\n\n*Example* Suppose we want to retrieve the titles and authors of the table `posts`.\nIn the case where the author field is missing or `None`, we want to retrieve the string\n`Anonymous`.\n\n r.table("posts").map(lambda post:\n {\n "title": post["title"],\n "author": post["author"].default("Anonymous")\n }\n ).run(conn)\n\nWe can rewrite the previous query with `r.branch` too.\n\n r.table("posts").map(lambda post:\n r.branch(\n post.has_fields("author"),\n {\n "title": post["title"],\n "author": post["author"]\n },\n {\n "title": post["title"],\n "author": "Anonymous" \n }\n )\n ).run(conn)\n\n*Example* The `default` command can be useful to filter documents too. Suppose\nwe want to retrieve all our users who are not grown-ups or whose age is unknown\n(i.e the field `age` is missing or equals `None`). We can do it with this query:\n\n r.table("users").filter(lambda user:\n (user["age"] < 18).default(True)\n ).run(conn)\n\nOne more way to write the previous query is to set the age to be `-1` when the\nfield is missing.\n\n r.table("users").filter(lambda user:\n user["age"].default(-1) < 18\n ).run(conn)\n\nOne last way to do the same query is to use `has_fields`.\n\n r.table("users").filter(lambda user:\n user.has_fields("age").not_() | (user["age"] < 18)\n ).run(conn)\n\nThe body of every `filter` is wrapped in an implicit `.default(False)`. You can overwrite\nthe value `False` by passing an option in filter, so the previous query can also be\nwritten like this.\n\n r.table("users").filter(\n lambda user: (user["age"] < 18).default(True),\n default=True\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.do, b"any.do(function) -> any\nr.do([args]*, function) -> any\nany.do(expr) -> any\nr.do([args]*, expr) -> any\n\nCall an anonymous function using return values from other ReQL commands or queries as arguments.\n\nThe last argument to `do` (or, in some forms, the only argument) is an expression or an anonymous function which receives values from either the previous arguments or from prefixed commands chained before `do`. The `do` command is essentially a single-element [map](http://rethinkdb.com/api/python/map/), letting you map a function over just one document. This allows you to bind a query result to a local variable within the scope of `do`, letting you compute the result just once and reuse it in a complex expression or in a series of ReQL commands.\n\nArguments passed to the `do` function must be basic data types, and cannot be streams or selections. (Read about [ReQL data types](http://rethinkdb.com/docs/data-types/).) While the arguments will all be evaluated before the function is executed, they may be evaluated in any order, so their values should not be dependent on one another. The type of `do`'s result is the type of the value returned from the function or last expression.\n\n*Example* Compute a golfer's net score for a game.\n\n r.table('players').get('86be93eb-a112-48f5-a829-15b2cb49de1d').do(\n lambda player: player['gross_score'] - player['course_handicap']\n ).run(conn)\n\n*Example* Return the name of the best scoring player in a two-player golf match.\n\n r.do(r.table('players').get(id1), r.table('players').get(id2),\n (lambda player1, player2:\n r.branch(player1['gross_score'].lt(player2['gross_score']),\n player1, player2))\n ).run(conn)\n\nNote that `branch`, the ReQL conditional command, must be used instead of `if`. See the `branch` [documentation](http://rethinkdb.com/api/python/branch) for more.\n\n*Example* Take different actions based on the result of a ReQL [insert](http://rethinkdb.com/api/python/insert) command.\n\n new_data = {\n 'id': 100,\n 'name': 'Agatha',\n 'gross_score': 57,\n 'course_handicap': 4\n }\n r.table('players').insert(new_data).do(lambda doc:\n r.branch((doc['inserted'] != 0),\n r.table('log').insert({'time': r.now(), 'response': doc, 'result': 'ok'}),\n r.table('log').insert({'time': r.now(), 'response': doc, 'result': 'error'}))\n ).run(conn)\n"), + (rethinkdb.error, b"r.error(message) -> error\n\nThrow a runtime error. If called with no arguments inside the second argument to `default`, re-throw the current error.\n\n*Example* Iron Man can't possibly have lost a battle:\n\n r.table('marvel').get('IronMan').do(\n lambda ironman: r.branch(ironman['victories'] < ironman['battles'],\n r.error('impossible code path'),\n ironman)\n ).run(conn)\n\n"), + (rethinkdb.expr, b"r.expr(value) -> value\n\nConstruct a ReQL JSON object from a native object.\n\nIf the native object is of the `bytes` type, then `expr` will return a binary object. See [binary](http://rethinkdb.com/api/python/binary) for more information.\n\n*Example* Objects wrapped with expr can then be manipulated by ReQL API functions.\n\n r.expr({'a':'b'}).merge({'b':[1,2,3]}).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.for_each, b"sequence.for_each(write_query) -> object\n\nLoop over a sequence, evaluating the given write query for each element.\n\n*Example* Now that our heroes have defeated their villains, we can safely remove them from the villain table.\n\n r.table('marvel').for_each(\n lambda hero: r.table('villains').get(hero['villainDefeated']).delete()\n ).run(conn)\n\n"), + (rethinkdb.http, b'r.http(url[, options]) -> value\nr.http(url[, options]) -> stream\n\nRetrieve data from the specified URL over HTTP. The return type depends on the `result_format` option, which checks the `Content-Type` of the response by default.\n\n*Example* Perform an HTTP `GET` and store the result in a table.\n\n r.table(\'posts\').insert(r.http(\'http://httpbin.org/get\')).run(conn)\n\nSee [the tutorial](http://rethinkdb.com/docs/external-api-access/) on `r.http` for more examples on how to use this command.\n\n* `timeout`: Number of seconds to wait before timing out and aborting the operation. Default: 30.\n\n* `reattempts`: An integer giving the number of attempts to make in cast of connection errors or potentially-temporary HTTP errors. Default: 5.\n\n* `redirects`: An integer giving the number of redirects and location headers to follow. Default: 1.\n\n* `verify`: Verify the server\'s SSL certificate, specified as a boolean. Default: True.\n\n* `result_format`: The format the result should be returned in. The values can be `\'text\'` (always return as a string), `\'json\'` (parse the result as JSON, raising an error if the parsing fails), `\'jsonp\'` (parse the result as [padded JSON](http://www.json-p.org/)), `\'binary\'` (return a binary object), or `\'auto\'` . The default is `\'auto\'`.\n\n When `result_format` is `\'auto\'`, the response body will be parsed according to the `Content-Type` of the response:\n * `application/json`: parse as `\'json\'`\n * `application/json-p`, `text/json-p`, `text/javascript`: parse as `\'jsonp\'`\n * `audio/*`, `video/*`, `image/*`, `application/octet-stream`: return a binary object\n * Anything else: parse as `\'text\'`\n\n* `method`: HTTP method to use for the request. One of `GET`, `POST`, `PUT`, `PATCH`, `DELETE` or `HEAD`. Default: `GET`.\n\n* `auth`: Authentication information in the form of an object with key/value pairs indicating the authentication type (in the `type` key) and any required information. Types currently supported are `basic` and `digest` for HTTP Basic and HTTP Digest authentication respectively. If `type` is omitted, `basic` is assumed. Example:\n\n\t```py\n\tr.http(\'http://httpbin.org/basic-auth/fred/mxyzptlk\',\n auth={ \'type\': \'basic\', \'user\': \'fred\', \'pass\': \'mxyzptlk\' }).run(conn)\n\t```\n\n* `params`: URL parameters to append to the URL as encoded key/value pairs, specified as an object. For example, `{ \'query\': \'banana\', \'limit\': 2 }` will be appended as `?query=banana&limit=2`. Default: none.\n\n* `header`: Extra header lines to include. The value may be an array of strings or an object. Default: none.\n\n Unless specified otherwise, `r.http` will by default use the headers `Accept-Encoding: deflate=1;gzip=0.5` and `User-Agent: RethinkDB/VERSION`.\n\n* `data`: Data to send to the server on a `POST`, `PUT`, `PATCH`, or `DELETE` request.\n\n For `PUT`, `PATCH` and `DELETE` requests, the value will be serialized to JSON and placed in the request body, and the `Content-Type` will be set to `application/json`.\n\n\tFor `POST` requests, data may be either an object or a string. Objects will be written to the body as form-encoded key/value pairs (values must be numbers, strings, or `None`). Strings will be put directly into the body. If `data` is not a string or an object, an error will be thrown.\n\n If `data` is not specified, no data will be sent.\n\n`r.http` supports depagination, which will request multiple pages in a row and aggregate the results into a stream. The use of this feature is controlled by the optional arguments `page` and `page_limit`. Either none or both of these arguments must be provided.\n\n* `page`: This option may specify either a built-in pagination strategy (as a string), or a function to provide the next URL and/or `params` to request.\n\n At the moment, the only supported built-in is `\'link-next\'`, which is equivalent to `lambda info: info[\'header\'][\'link\'][\'rel="next"\'].default(None)`.\n\n *Example* Perform a GitHub search and collect up to 3 pages of results.\n\n ```py\n r.http("https://api.github.com/search/code?q=addClass+user:mozilla",\n page=\'link-next\', page_limit=3).run(conn)\n ```\n\n As a function, `page` takes one parameter, an object of the format:\n\n ```py\n {\n \'params\': object, # the URL parameters used in the last request\n \'header\': object, # the HTTP headers of the last response as key/value pairs\n \'body\': value # the body of the last response in the format specified by `result_format`\n }\n ```\n\n The `header` field will be a parsed version of the header with fields lowercased, like so:\n\n ```py\n {\n \'content-length\': \'1024\',\n \'content-type\': \'application/json\',\n \'date\': \'Thu, 1 Jan 1970 00:00:00 GMT\',\n \'link\': {\n \'rel="last"\': \'http://example.com/?page=34\',\n \'rel="next"\': \'http://example.com/?page=2\'\n }\n }\n ```\n\n The `page` function may return a string corresponding to the next URL to request, `None` indicating that there is no more to get, or an object of the format:\n\n ```py\n {\n \'url\': string, # the next URL to request, or None for no more pages\n \'params\': object # new URL parameters to use, will be merged with the previous request\'s params\n }\n ```\n\n* `page_limit`: An integer specifying the maximum number of requests to issue using the `page` functionality. This is to prevent overuse of API quotas, and must be specified with `page`.\n * `-1`: no limit\n * `0`: no requests will be made, an empty stream will be returned\n * `n`: `n` requests will be made\n\n# Examples\n\n*Example* Perform multiple requests with different parameters.\n\n r.expr([1, 2, 3]).map(lambda i: r.http(\'http://httpbin.org/get\',\n params={ \'user\': i })).run(conn)\n\n*Example* Perform a `PUT` request for each item in a table.\n\n r.table(\'data\').map(lambda row: r.http(\'http://httpbin.org/put\',\n method=\'PUT\', data=row)).run(conn)\n\n*Example* Perform a `POST` request with accompanying data.\n\nUsing form-encoded data:\n\n r.http(\'http://httpbin.org/post\',\n method=\'POST\',\n data={ \'player\': \'Bob\', \'game\': \'tic tac toe\' }).run(conn)\n\nUsing JSON data:\n\n r.http(\'http://httpbin.org/post\',\n method=\'POST\',\n data=r.expr(value).coerce_to(\'string\'),\n header={ \'Content-Type\': \'application/json\' }).run(conn)\n\n*Example* Perform depagination with a custom `page` function.\n\n r.http(\'example.com/pages\',\n page=lambda info: info[\'body\'][\'meta\'][\'next\'].default(None),\n page_limit=5).run(conn)\n\n# Learn more\n\nSee [the tutorial](http://rethinkdb.com/docs/external-api-access/) on `r.http` for more examples on how to use this command.\n'), + (rethinkdb.ast.RqlQuery.info, b"any.info() -> object\n\nGet information about a ReQL value.\n\n*Example* Get information about a table such as primary key, or cache size.\n\n r.table('marvel').info().run(conn)\n\n"), + (rethinkdb.js, b'r.js(js_string[, timeout=]) -> value\n\nCreate a javascript expression.\n\n*Example* Concatenate two strings using JavaScript.\n\n`timeout` is the number of seconds before `r.js` times out. The default value is 5 seconds.\n\n{% infobox %}\nWhenever possible, you should use native ReQL commands rather than `r.js` for better performance.\n{% endinfobox %}\n\n r.js("\'str1\' + \'str2\'").run(conn)\n\n*Example* Select all documents where the \'magazines\' field is greater than 5 by running JavaScript on the server.\n\n r.table(\'marvel\').filter(\n r.js(\'(function (row) { return row.magazines.length > 5; })\')\n ).run(conn)\n\n*Example* You may also specify a timeout in seconds (defaults to 5).\n\n r.js(\'while(true) {}\', timeout=1.3).run(conn)\n\n'), + (rethinkdb.json, b'r.json(json_string) -> value\n\nParse a JSON string on the server.\n\n*Example* Send an array to the server\'\n\n r.json("[1,2,3]").run(conn)\n\n'), + (rethinkdb.range, b'r.range() -> stream\nr.range([start_value, ]end_value) -> stream\n\nGenerate a stream of sequential integers in a specified range. `range` takes 0, 1 or 2 arguments:\n\n* With no arguments, `range` returns an "infinite" stream from 0 up to and including the maximum integer value;\n* With one argument, `range` returns a stream from 0 up to but not including the end value;\n* With two arguments, `range` returns a stream from the start value up to but not including the end value.\n\nNote that the left bound (including the implied left bound of 0 in the 0- and 1-argument form) is always closed and the right bound is always open: the start value will always be included in the returned range and the end value will *not* be included in the returned range.\n\nAny specified arguments must be integers, or a `RqlRuntimeError` will be thrown. If the start value is equal or to higher than the end value, no error will be thrown but a zero-element stream will be returned.\n\n*Example* Return a four-element range of `[0, 1, 2, 3]`.\n\n > r.range(4).run(conn)\n \n [0, 1, 2, 3]\n\nYou can also use the [limit](http://rethinkdb.com/api/python/limit) command with the no-argument variant to achieve the same result in this case:\n\n > r.range().limit(4).run(conn)\n \n [0, 1, 2, 3]\n\n*Example* Return a range from -5 through 5.\n\n > r.range(-5, 6).run(conn)\n \n [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]\n'), + (rethinkdb.ast.RqlQuery.to_json_string, b'value.to_json_string() -> string\nvalue.to_json() -> string\n\nConvert a ReQL value or object to a JSON string. You may use either `to_json_string` or `to_json`.\n\n*Example* Get a ReQL document as a JSON string.\n\n > r.table(\'hero\').get(1).to_json()\n \n \'{"id": 1, "name": "Batman", "city": "Gotham", "powers": ["martial arts", "cinematic entrances"]}\'\n'), + (rethinkdb.ast.RqlQuery.to_json, b'value.to_json_string() -> string\nvalue.to_json() -> string\n\nConvert a ReQL value or object to a JSON string. You may use either `to_json_string` or `to_json`.\n\n*Example* Get a ReQL document as a JSON string.\n\n > r.table(\'hero\').get(1).to_json()\n \n \'{"id": 1, "name": "Batman", "city": "Gotham", "powers": ["martial arts", "cinematic entrances"]}\'\n'), + (rethinkdb.ast.RqlQuery.type_of, b'any.type_of() -> string\n\nGets the type of a value.\n\n*Example* Get the type of a string.\n\n r.expr("foo").type_of().run(conn)\n\n'), + (rethinkdb.uuid, b'r.uuid() -> string\n\nReturn a UUID (universally unique identifier), a string that can be used as a unique ID.\n\n*Example* Generate a UUID.\n\n > r.uuid().run(conn)\n \n 27961a0e-f4e8-4eb3-bf95-c5203e1d87b9\n'), + (rethinkdb.net.Cursor.close, b'cursor.close()\n\nClose a cursor. Closing a cursor cancels the corresponding query and frees the memory\nassociated with the open request.\n\n*Example* Close a cursor.\n\n cursor.close()\n'), + (rethinkdb.net.Cursor.next, b"cursor.next([wait=True])\n\nGet the next element in the cursor.\n\nThe optional `wait` argument specifies whether to wait for the next available element and how long to wait:\n\n* `True`: Wait indefinitely (the default).\n* `False`: Do not wait at all. If data is immediately available, it will be returned; if it is not available, a `RqlDriverError` will be raised.\n* number: Wait up the specified number of seconds for data to be available before raising `RqlDriverError`.\n\nThe behavior of `next` will be identical with `False`, `None` or the number `0`.\n\nCalling `next` the first time on a cursor provides the first element of the cursor. If the data set is exhausted (e.g., you have retrieved all the documents in a table), a `StopIteration` error will be raised when `next` is called.\n\n*Example* Retrieve the next element.\n\n cursor = r.table('superheroes').run(conn)\n doc = cursor.next()\n\n*Example* Retrieve the next element on a [changefeed](http://rethinkdb.com/docs/changefeeds/python), waiting up to five seconds.\n\n cursor = r.table('superheroes').changes().run(conn)\n doc = cursor.next(wait=5)\n\n__Note:__ RethinkDB sequences can be iterated through via the Python [Iterable][it] interface. The canonical way to retrieve all the results is to use a [for...in](../each/) loop or [list()](../to_array/).\n\n[it]: https://docs.python.org/3.4/library/stdtypes.html#iterator-types\n"), + (rethinkdb.ast.RqlQuery.date, b'time.date() -> time\n\nReturn a new time object only based on the day, month and year (ie. the same day at 00:00).\n\n*Example* Retrieve all the users whose birthday is today\n\n r.table("users").filter(lambda user:\n user["birthdate"].date() == r.now().date()\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.day, b'time.day() -> number\n\nReturn the day of a time object as a number between 1 and 31.\n\n*Example* Return the users born on the 24th of any month.\n\n r.table("users").filter(\n r.row["birthdate"].day() == 24\n )\n\n'), + (rethinkdb.ast.RqlQuery.day_of_week, b'time.day_of_week() -> number\n\nReturn the day of week of a time object as a number between 1 and 7 (following ISO 8601 standard). For your convenience, the terms r.monday, r.tuesday etc. are defined and map to the appropriate integer.\n\n*Example* Return today\'s day of week.\n\n r.now().day_of_week().run(conn)\n\n*Example* Retrieve all the users who were born on a Tuesday.\n\n r.table("users").filter( lambda user:\n user["birthdate"].day_of_week().eq(r.tuesday)\n )\n\n'), + (rethinkdb.ast.RqlQuery.day_of_year, b'time.day_of_year() -> number\n\nReturn the day of the year of a time object as a number between 1 and 366 (following ISO 8601 standard).\n\n*Example* Retrieve all the users who were born the first day of a year.\n\n r.table("users").filter(\n r.row["birthdate"].day_of_year() == 1\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.during, b'time.during(start_time, end_time[, left_bound="closed", right_bound="open"])\n -> bool\n\nReturn whether a time is between two other times. By default, this is inclusive of the start time and exclusive of the end time. Set `left_bound` and `right_bound` to explicitly include (`closed`) or exclude (`open`) that endpoint of the range.\n\n*Example* Retrieve all the posts that were posted between December 1st, 2013 (inclusive) and December 10th, 2013 (exclusive).\n\n r.table("posts").filter(\n r.row[\'date\'].during(r.time(2013, 12, 1, "Z"), r.time(2013, 12, 10, "Z"))\n ).run(conn)\n\n*Example* Retrieve all the posts that were posted between December 1st, 2013 (exclusive) and December 10th, 2013 (inclusive).\n\n r.table("posts").filter(\n r.row[\'date\'].during(r.time(2013, 12, 1, "Z"), r.time(2013, 12, 10, "Z"), left_bound="open", right_bound="closed")\n ).run(conn)\n\n'), + (rethinkdb.epoch_time, b'r.epoch_time(epoch_time) -> time\n\nCreate a time object based on seconds since epoch. The first argument is a double and\nwill be rounded to three decimal places (millisecond-precision).\n\n*Example* Update the birthdate of the user "John" to November 3rd, 1986.\n\n r.table("user").get("John").update({"birthdate": r.epoch_time(531360000)}).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.hours, b'time.hours() -> number\n\nReturn the hour in a time object as a number between 0 and 23.\n\n*Example* Return all the posts submitted after midnight and before 4am.\n\n r.table("posts").filter(lambda post:\n post["date"].hours() < 4\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.in_timezone, b"time.in_timezone(timezone) -> time\n\nReturn a new time object with a different timezone. While the time stays the same, the results returned by methods such as hours() will change since they take the timezone into account. The timezone argument has to be of the ISO 8601 format.\n\n*Example* Hour of the day in San Francisco (UTC/GMT -8, without daylight saving time).\n\n r.now().in_timezone('-08:00').hours().run(conn)\n"), + (rethinkdb.iso8601, b'r.iso8601(iso8601Date[, default_timezone=\'\']) -> time\n\nCreate a time object based on an ISO 8601 date-time string (e.g. \'2013-01-01T01:01:01+00:00\'). We support all valid ISO 8601 formats except for week dates. If you pass an ISO 8601 date-time without a time zone, you must specify the time zone with the `default_timezone` argument. Read more about the ISO 8601 format at [Wikipedia](http://en.wikipedia.org/wiki/ISO_8601).\n\n*Example* Update the time of John\'s birth.\n\n r.table("user").get("John").update({"birth": r.iso8601(\'1986-11-03T08:30:00-07:00\')}).run(conn)\n'), + (rethinkdb.ast.RqlQuery.minutes, b'time.minutes() -> number\n\nReturn the minute in a time object as a number between 0 and 59.\n\n*Example* Return all the posts submitted during the first 10 minutes of every hour.\n\n r.table("posts").filter(lambda post:\n post["date"].minutes() < 10\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.month, b'time.month() -> number\n\nReturn the month of a time object as a number between 1 and 12. For your convenience, the terms r.january, r.february etc. are defined and map to the appropriate integer.\n\n*Example* Retrieve all the users who were born in November.\n\n r.table("users").filter(\n r.row["birthdate"].month() == 11\n )\n\n*Example* Retrieve all the users who were born in November.\n\n r.table("users").filter(\n r.row["birthdate"].month() == r.november\n )\n\n'), + (rethinkdb.now, b'r.now() -> time\n\nReturn a time object representing the current time in UTC. The command now() is computed once when the server receives the query, so multiple instances of r.now() will always return the same time inside a query.\n\n*Example* Add a new user with the time at which he subscribed.\n\n r.table("users").insert({\n "name": "John",\n "subscription_date": r.now()\n }).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.seconds, b'time.seconds() -> number\n\nReturn the seconds in a time object as a number between 0 and 59.999 (double precision).\n\n*Example* Return the post submitted during the first 30 seconds of every minute.\n\n r.table("posts").filter(lambda post:\n post["date"].seconds() < 30\n ).run(conn)\n\n'), + (rethinkdb.time, b'r.time(year, month, day[, hour, minute, second], timezone)\n -> time\n\nCreate a time object for a specific time.\n\nA few restrictions exist on the arguments:\n\n- `year` is an integer between 1400 and 9,999.\n- `month` is an integer between 1 and 12.\n- `day` is an integer between 1 and 31.\n- `hour` is an integer.\n- `minutes` is an integer.\n- `seconds` is a double. Its value will be rounded to three decimal places\n(millisecond-precision).\n- `timezone` can be `\'Z\'` (for UTC) or a string with the format `\xc2\xb1[hh]:[mm]`.\n\n*Example* Update the birthdate of the user "John" to November 3rd, 1986 UTC.\n\n r.table("user").get("John").update({"birthdate": r.time(1986, 11, 3, \'Z\')}).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.time_of_day, b'time.time_of_day() -> number\n\nReturn the number of seconds elapsed since the beginning of the day stored in the time object.\n\n*Example* Retrieve posts that were submitted before noon.\n\n r.table("posts").filter(\n r.row["date"].time_of_day() <= 12*60*60\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.timezone, b'time.timezone() -> string\n\nReturn the timezone of the time object.\n\n*Example* Return all the users in the "-07:00" timezone.\n\n r.table("users").filter(lambda user:\n user["subscriptionDate"].timezone() == "-07:00"\n )\n\n'), + (rethinkdb.ast.RqlQuery.to_epoch_time, b'time.to_epoch_time() -> number\n\nConvert a time object to its epoch time.\n\n*Example* Return the current time in seconds since the Unix Epoch with millisecond-precision.\n\n r.now().to_epoch_time()\n\n'), + (rethinkdb.ast.RqlQuery.to_iso8601, b'time.to_iso8601() -> string\n\nConvert a time object to a string in ISO 8601 format.\n\n*Example* Return the current ISO 8601 time.\n\n > r.now().to_iso8601().run(conn)\n \n "2015-04-20T18:37:52.690+00:00"\n\n'), + (rethinkdb.ast.RqlQuery.year, b'time.year() -> number\n\nReturn the year of a time object.\n\n*Example* Retrieve all the users born in 1986.\n\n r.table("users").filter(lambda user:\n user["birthdate"].year() == 1986\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.append, b"array.append(value) -> array\n\nAppend a value to an array.\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].append('newBoots').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.__getitem__, b"sequence[attr] -> sequence\nsingleSelection[attr] -> value\nobject[attr] -> value\narray[index] -> value\n\nGet a single field from an object. If called on a sequence, gets that field from every object in the sequence, skipping objects that lack it.\n\n*Example* What was Iron Man's first appearance in a comic?\n\n r.table('marvel').get('IronMan')['firstAppearance'].run(conn)\n\nThe `[]` command also accepts integer arguments as array offsets, like the [nth](http://rethinkdb.com/api/python/nth) command.\n\n*Example* Get the fourth element in a sequence. (The first element is position `0`, so the fourth element is position `3`.)\n\n r.expr([10, 20, 30, 40, 50])[3]\n \n 40\n"), + (rethinkdb.ast.RqlQuery.change_at, b'array.change_at(index, value) -> array\n\nChange a value in an array at a given index. Returns the modified array.\n\n*Example* Bruce Banner hulks out.\n\n r.expr(["Iron Man", "Bruce", "Spider-Man"]).change_at(1, "Hulk").run(conn)\n'), + (rethinkdb.ast.RqlQuery.delete_at, b"array.delete_at(index [,endIndex]) -> array\n\nRemove one or more elements from an array at a given index. Returns the modified array. (Note: `delete_at` operates on arrays, not documents; to delete documents, see the [delete](http://rethinkdb.com/api/python/delete) command.)\n\nIf only `index` is specified, `delete_at` removes the element at that index. If both `index` and `end_index` are specified, `delete_at` removes the range of elements between `index` and `end_index`, inclusive of `index` but not inclusive of `end_index`.\n\nIf `end_index` is specified, it must not be less than `index`. Both `index` and `end_index` must be within the array's bounds (i.e., if the array has 10 elements, an `index` or `end_index` of 10 or higher is invalid).\n\nBy using a negative `index` you can delete from the end of the array. `-1` is the last element in the array, `-2` is the second-to-last element, and so on. You may specify a negative `end_index`, although just as with a positive value, this will not be inclusive. The range `(2,-1)` specifies the third element through the next-to-last element.\n\n*Example* Delete the second element of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(1).run(conn)\n \n ['a', 'c', 'd', 'e', 'f']\n\n*Example* Delete the second and third elements of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(1,3).run(conn)\n \n ['a', 'd', 'e', 'f']\n\n*Example* Delete the next-to-last element of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(-2).run(conn)\n \n ['a', 'b', 'c', 'd', 'f']\n\n*Example* Delete a comment on a post.\n\nGiven a post document such as:\n\n{\n id: '4cf47834-b6f9-438f-9dec-74087e84eb63',\n title: 'Post title',\n author: 'Bob',\n comments: [\n { author: 'Agatha', text: 'Comment 1' },\n { author: 'Fred', text: 'Comment 2' }\n ]\n}\n\nThe second comment can be deleted by using `update` and `delete_at` together.\n\n r.table('posts').get('4cf47834-b6f9-438f-9dec-74087e84eb63').update(\n lambda post: { 'comments': post['comments'].delete_at(1) }\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.difference, b"array.difference(array) -> array\n\nRemove the elements of one array from another array.\n\n*Example* Retrieve Iron Man's equipment list without boots.\n\n r.table('marvel').get('IronMan')['equipment'].difference(['Boots']).run(conn)\n\n*Example* Remove Iron Man's boots from his equipment.\n\n r.table('marvel').get('IronMan')[:equipment].update(lambda doc:\n {'equipment': doc['equipment'].difference(['Boots'])}\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.get_field, b"sequence.get_field(attr) -> sequence\nsingleSelection.get_field(attr) -> value\nobject.get_field(attr) -> value\n\nGet a single field from an object. If called on a sequence, gets that field from every\nobject in the sequence, skipping objects that lack it.\n\n*Example* What was Iron Man's first appearance in a comic?\n\n r.table('marvel').get('IronMan').get_field('firstAppearance').run(conn)\n"), + (rethinkdb.ast.RqlQuery.has_fields, b'sequence.has_fields([selector1, selector2...]) -> stream\narray.has_fields([selector1, selector2...]) -> array\nobject.has_fields([selector1, selector2...]) -> boolean\n\nTest if an object has one or more fields. An object has a field if it has that key and the key has a non-null value. For instance, the object `{\'a\': 1,\'b\': 2,\'c\': null}` has the fields `a` and `b`.\n\nWhen applied to a single object, `has_fields` returns `true` if the object has the fields and `false` if it does not. When applied to a sequence, it will return a new sequence (an array or stream) containing the elements that have the specified fields.\n\n*Example* Return the players who have won games.\n\n r.table(\'players\').has_fields(\'games_won\').run(conn)\n\n*Example* Return the players who have *not* won games. To do this, use `has_fields` with [not](http://rethinkdb.com/api/python/not), wrapped with [filter](http://rethinkdb.com/api/python/filter).\n\n r.table(\'players\').filter(~r.row.has_fields(\'games_won\')).run(conn)\n\n*Example* Test if a specific player has won any games.\n\n r.table(\'players\').get(\n \'b5ec9714-837e-400c-aa74-dbd35c9a7c4c\').has_fields(\'games_won\').run(conn)\n\n**Nested Fields**\n\n`has_fields` lets you test for nested fields in objects. If the value of a field is itself a set of key/value pairs, you can test for the presence of specific keys.\n\n*Example* In the `players` table, the `games_won` field contains one or more fields for kinds of games won:\n\n {\n \'games_won\': {\n \'playoffs\': 2,\n \'championships\': 1\n }\n }\n\nReturn players who have the "championships" field.\n\n r.table(\'players\').has_fields({\'games_won\': {\'championships\': true}}).run(conn)\n\nNote that `true` in the example above is testing for the existence of `championships` as a field, not testing to see if the value of the `championships` field is set to `true`. There\'s a more convenient shorthand form available. (See [pluck](http://rethinkdb.com/api/python/pluck) for more details on this.)\n\n r.table(\'players\').has_fields({\'games_won\': \'championships\'}).run(conn)\n'), + (rethinkdb.ast.RqlQuery.insert_at, b'array.insert_at(index, value) -> array\n\nInsert a value in to an array at a given index. Returns the modified array.\n\n*Example* Hulk decides to join the avengers.\n\n r.expr(["Iron Man", "Spider-Man"]).insert_at(1, "Hulk").run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.keys, b"singleSelection.keys() -> array\nobject.keys() -> array\n\nReturn an array containing all of the object's keys.\n\n*Example* Get all the keys of a row.\n\n r.table('marvel').get('ironman').keys().run(conn)\n\n"), + (rethinkdb.literal, b'r.literal(object) -> special\n\nReplace an object in a field instead of merging it with an existing object in a `merge` or `update` operation. = Using `literal` with no arguments in a `merge` or `update` operation will remove the corresponding field.\n\n*Example* Replace one nested document with another rather than merging the fields.\n\nAssume your users table has this structure:\n\n [\n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 18,\n "city": "Dallas"\n }\n } \n ...\n ]\n\nUsing `update` to modify the `data` field will normally merge the nested documents:\n\n r.table(\'users\').get(1).update({ \'data\': { \'age\': 19, \'job\': \'Engineer\' } }).run(conn)\n \n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 19,\n "city": "Dallas",\n "job": "Engineer"\n }\n } \n\nThat will preserve `city` and other existing fields. But to replace the entire `data` document with a new object, use `literal`:\n\n r.table(\'users\').get(1).update({ \'data\': r.literal({ \'age\': 19, \'job\': \'Engineer\' }) }).run(conn)\n \n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 19,\n "job": "Engineer"\n }\n } \n\n*Example* Use `literal` to remove a field from a document.\n\n r.table(\'users\').get(1).merge({ "data": r.literal() }).run(conn)\n \n {\n "id": 1,\n "name": "Alice"\n }\n'), + (rethinkdb.ast.RqlQuery.merge, b'singleSelection.merge(object|function[, object|function, ...]) -> object\nobject.merge(object|function[, object|function, ...]) -> object\nsequence.merge(object|function[, object|function, ...]) -> stream\narray.merge(object|function[, object|function, ...]) -> array\n\nMerge two or more objects together to construct a new object with properties from all. When there is a conflict between field names, preference is given to fields in the rightmost object in the argument list `merge` also accepts a subquery function that returns an object, which will be used similarly to a [map](http://rethinkdb.com/api/python/map/) function.\n\n*Example* Equip Thor for battle.\n\n r.table(\'marvel\').get(\'thor\').merge(\n r.table(\'equipment\').get(\'hammer\'),\n r.table(\'equipment\').get(\'pimento_sandwich\')\n ).run(conn)\n\n*Example* Equip every hero for battle, using a subquery function to retrieve their weapons.\n\n r.table(\'marvel\').merge(lambda hero:\n { \'weapons\': r.table(\'weapons\').get(hero[\'weapon_id\']) }\n ).run(conn)\n\n*Example* Use `merge` to join each blog post with its comments.\n\nNote that the sequence being merged—in this example, the comments—must be coerced from a selection to an array. Without `coerce_to` the operation will throw an error ("Expected type DATUM but found SELECTION").\n\n r.table(\'posts\').merge(lambda post:\n { \'comments\': r.table(\'comments\').get_all(post[\'id\'],\n index=\'post_id\').coerce_to(\'array\') }\n ).run(conn)\n\n*Example* Merge can be used recursively to modify object within objects.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10}}}\n ).run(conn)\n\n*Example* To replace a nested object with another object you can use the literal keyword.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : r.literal({\'repulsor rays\' : {\'dmg\' : 3, \'cooldown\' : 0}})}\n ).run(conn)\n\n*Example* Literal can be used to remove keys from an object as well.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : {\'spectacular graviton beam\' : r.literal()}}\n ).run(conn)\n\n'), + (rethinkdb.object, b'r.object([key, value,]...) -> object\n\nCreates an object from a list of key-value pairs, where the keys must\nbe strings. `r.object(A, B, C, D)` is equivalent to\n`r.expr([[A, B], [C, D]]).coerce_to(\'OBJECT\')`.\n\n*Example* Create a simple object.\n\n > r.object(\'id\', 5, \'data\', [\'foo\', \'bar\']).run(conn)\n {\'data\': ["foo", "bar"], \'id\': 5}\n'), + (rethinkdb.ast.RqlQuery.pluck, b"sequence.pluck([selector1, selector2...]) -> stream\narray.pluck([selector1, selector2...]) -> array\nobject.pluck([selector1, selector2...]) -> object\nsingleSelection.pluck([selector1, selector2...]) -> object\n\nPlucks out one or more attributes from either an object or a sequence of objects\n(projection).\n\n*Example* We just need information about IronMan's reactor and not the rest of the\ndocument.\n\n r.table('marvel').get('IronMan').pluck('reactorState', 'reactorPower').run(conn)\n\n*Example* For the hero beauty contest we only care about certain qualities.\n\n r.table('marvel').pluck('beauty', 'muscleTone', 'charm').run(conn)\n\n*Example* Pluck can also be used on nested objects.\n\n r.table('marvel').pluck({'abilities' : {'damage' : True, 'mana_cost' : True}, 'weapons' : True}).run(conn)\n\n*Example* The nested syntax can quickly become overly verbose so there's a shorthand\nfor it.\n\n r.table('marvel').pluck({'abilities' : ['damage', 'mana_cost']}, 'weapons').run(conn)\n\nFor more information read the [nested field documentation](http://rethinkdb.com/docs/nested-fields/).\n"), + (rethinkdb.ast.RqlQuery.prepend, b"array.prepend(value) -> array\n\nPrepend a value to an array.\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].prepend('newBoots').run(conn)\n"), + (rethinkdb.row, b"r.row -> value\n\nReturns the currently visited document. Note that `row` does not work within subqueries to access nested documents; you should use anonymous functions to access those documents instead. (See the last example.)\n\n*Example* Get all users whose age is greater than 5.\n\n r.table('users').filter(r.row['age'] > 5).run(conn)\n\n*Example* Access the attribute 'child' of an embedded document.\n\n r.table('users').filter(r.row['embedded_doc']['child'] > 5).run(conn)\n\n*Example* Add 1 to every element of an array.\n\n r.expr([1, 2, 3]).map(r.row + 1).run(conn)\n\n*Example* For nested queries, use functions instead of `row`.\n\n r.table('users').filter(\n lambda doc: doc['name'] == r.table('prizes').get('winner')\n ).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.set_difference, b"array.set_difference(array) -> array\n\nRemove the elements of one array from another and return them as a set (an array with\ndistinct values).\n\n*Example* Check which pieces of equipment Iron Man has, excluding a fixed list.\n\n r.table('marvel').get('IronMan')['equipment'].set_difference(['newBoots', 'arc_reactor']).run(conn)\n"), + (rethinkdb.ast.RqlQuery.set_insert, b"array.set_insert(value) -> array\n\nAdd a value to an array and return it as a set (an array with distinct values).\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].set_insert('newBoots').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.set_intersection, b"array.set_intersection(array) -> array\n\nIntersect two arrays returning values that occur in both of them as a set (an array with\ndistinct values).\n\n*Example* Check which pieces of equipment Iron Man has from a fixed list.\n\n r.table('marvel').get('IronMan')['equipment'].set_intersection(['newBoots', 'arc_reactor']).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.set_union, b"array.set_union(array) -> array\n\nAdd a several values to an array and return it as a set (an array with distinct values).\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots and an arc reactor.\n\n r.table('marvel').get('IronMan')['equipment'].set_union(['newBoots', 'arc_reactor']).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.splice_at, b'array.splice_at(index, array) -> array\n\nInsert several values in to an array at a given index. Returns the modified array.\n\n*Example* Hulk and Thor decide to join the avengers.\n\n r.expr(["Iron Man", "Spider-Man"]).splice_at(1, ["Hulk", "Thor"]).run(conn)\n'), + (rethinkdb.ast.RqlQuery.without, b"sequence.without([selector1, selector2...]) -> stream\narray.without([selector1, selector2...]) -> array\nsingleSelection.without([selector1, selector2...]) -> object\nobject.without([selector1, selector2...]) -> object\n\nThe opposite of pluck; takes an object or a sequence of objects, and returns them with\nthe specified paths removed.\n\n*Example* Since we don't need it for this computation we'll save bandwidth and leave\nout the list of IronMan's romantic conquests.\n\n r.table('marvel').get('IronMan').without('personalVictoriesList').run(conn)\n\n*Example* Without their prized weapons, our enemies will quickly be vanquished.\n\n r.table('enemies').without('weapons').run(conn)\n\n*Example* Nested objects can be used to remove the damage subfield from the weapons and abilities fields.\n\n r.table('marvel').without({'weapons' : {'damage' : True}, 'abilities' : {'damage' : True}}).run(conn)\n\n*Example* The nested syntax can quickly become overly verbose so there's a shorthand for it.\n\n r.table('marvel').without({'weapons' : 'damage', 'abilities' : 'damage'}).run(conn)\n\n"), + (rethinkdb.circle, b"r.circle([longitude, latitude], radius[, num_vertices=32, geo_system='WGS84', unit='m', fill=True]) -> geometry\nr.circle(point, radius[, {num_vertices=32, geo_system='WGS84', unit='m', fill=True]) -> geometry\n\nConstruct a circular line or polygon. A circle in RethinkDB is a polygon or line *approximating* a circle of a given radius around a given center, consisting of a specified number of vertices (default 32).\n\nThe center may be specified either by two floating point numbers, the latitude (−90 to 90) and longitude (−180 to 180) of the point on a perfect sphere (see [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system), or by a point object. The radius is a floating point number whose units are meters by default, although that may be changed with the `unit` argument.\n\nOptional arguments available with `circle` are:\n\n* `num_vertices`: the number of vertices in the polygon or line. Defaults to 32.\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n* `unit`: Unit for the radius distance. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n* `fill`: if `True` (the default) the circle is filled, creating a polygon; if `False` the circle is unfilled (creating a line).\n\n*Example* Define a circle.\n\n r.table('geo').insert({\n 'id': 300,\n 'name': 'Hayes Valley',\n 'neighborhood': r.circle([-122.423246,37.779388], 1000)\n }).run(conn)\n"), + (rethinkdb.ast.RqlQuery.distance, b"geometry.distance(geometry[, geo_system='WGS84', unit='m']) -> number\n\nCompute the distance between a point and another geometry object. At least one of the geometry objects specified must be a point.\n\nOptional arguments available with `distance` are:\n\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n* `unit`: Unit to return the distance in. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n\nIf one of the objects is a polygon or a line, the point will be projected onto the line or polygon assuming a perfect sphere model before the distance is computed (using the model specified with `geo_system`). As a consequence, if the polygon or line is extremely large compared to Earth's radius and the distance is being computed with the default WGS84 model, the results of `distance` should be considered approximate due to the deviation between the ellipsoid and spherical models.\n\n*Example* Compute the distance between two points on the Earth in kilometers.\n\n > point1 = r.point(-122.423246,37.779388)\n > point2 = r.point(-117.220406,32.719464)\n > r.distance(point1, point2, unit='km').run(conn)\n \n 734.1252496021841\n"), + (rethinkdb.ast.RqlQuery.fill, b"line.fill() -> polygon\n\nConvert a Line object into a Polygon object. If the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\nIf the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them. You cannot directly construct a polygon with holes in it using `polygon`, but you can use [polygon_sub](http://rethinkdb.com/api/python/polygon_sub) to use a second polygon within the interior of the first to define a hole.\n\n*Example* Create a line object and then convert it to a polygon.\n\n r.table('geo').insert({\n 'id': 201,\n 'rectangle': r.line(\n [-122.423246,37.779388],\n [-122.423246,37.329898],\n [-121.886420,37.329898],\n [-121.886420,37.779388]\n )\n }).run(conn)\n \n r.table('geo').get(201).update({\n 'rectangle': r.row('rectangle').fill()\n }).run(conn)\n"), + (rethinkdb.geojson, b"r.geojson(geojson) -> geometry\n\nConvert a [GeoJSON][] object to a ReQL geometry object.\n\n[GeoJSON]: http://geojson.org\n\nRethinkDB only allows conversion of GeoJSON objects which have ReQL equivalents: Point, LineString, and Polygon. MultiPoint, MultiLineString, and MultiPolygon are not supported. (You could, however, store multiple points, lines and polygons in an array and use a geospatial multi index with them.)\n\nOnly longitude/latitude coordinates are supported. GeoJSON objects that use Cartesian coordinates, specify an altitude, or specify their own coordinate reference system will be rejected.\n\n*Example* Convert a GeoJSON object to a ReQL geometry object.\n\n geo_json = {\n 'type': 'Point',\n 'coordinates': [ -122.423246, 37.779388 ]\n }\n r.table('geo').insert({\n 'id': 'sfo',\n 'name': 'San Francisco',\n 'location': r.geojson(geo_json)\n }).run(conn)\n"), + (rethinkdb.ast.Table.get_intersecting, b"table.get_intersecting(geometry, index='indexname') -> selection\n\nGet all documents where the given geometry object intersects the geometry object of the requested geospatial index.\n\nThe `index` argument is mandatory. This command returns the same results as `table.filter(r.row('index').intersects(geometry))`. The total number of results is limited to the array size limit which defaults to 100,000, but can be changed with the `array_limit` option to [run](http://rethinkdb.com/api/python/run).\n\n*Example* Which of the locations in a list of parks intersect `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks').get_intersecting(circle1, index='area').run(conn)\n"), + (rethinkdb.ast.Table.get_nearest, b"table.get_nearest(point, index='indexname'[, max_results=100, max_dist=100000, unit='m', geo_system='WGS84']) -> array\n\nGet all documents where the specified geospatial index is within a certain distance of the specified point (default 100 kilometers).\n\nThe `index` argument is mandatory. Optional arguments are:\n\n* `max_results`: the maximum number of results to return (default 100).\n* `unit`: Unit for the distance. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n* `max_dist`: the maximum distance from an object to the specified point (default 100 km).\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n\nThe return value will be an array of two-item objects with the keys `dist` and `doc`, set to the distance between the specified point and the document (in the units specified with `unit`, defaulting to meters) and the document itself, respectively.\n\n*Example* Return a list of enemy hideouts within 5000 meters of the secret base.\n\n secret_base = r.point(-122.422876,37.777128)\n r.table('hideouts').get_nearest(secret_base, index='location',\n max_dist=5000).run(conn)\n"), + (rethinkdb.ast.RqlQuery.includes, b"sequence.includes(geometry) -> sequence\ngeometry.includes(geometry) -> bool\n\nTests whether a geometry object is completely contained within another. When applied to a sequence of geometry objects, `includes` acts as a [filter](http://rethinkdb.com/api/python/filter), returning a sequence of objects from the sequence that include the argument.\n\n*Example* Is `point2` included within a 2000-meter circle around `point1`?\n\n > point1 = r.point(-117.220406,32.719464)\n > point2 = r.point(-117.206201,32.725186)\n > r.circle(point1, 2000).includes(point2).run(conn)\n \n True\n\n*Example* Which of the locations in a list of parks include `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks')['area'].includes(circle1).run(conn)\n"), + (rethinkdb.ast.RqlQuery.intersects, b"sequence.intersects(geometry) -> sequence\ngeometry.intersects(geometry) -> bool\n\nTests whether two geometry objects intersect with one another. When applied to a sequence of geometry objects, `intersects` acts as a [filter](http://rethinkdb.com/api/python/filter), returning a sequence of objects from the sequence that intersect with the argument.\n\n*Example* Is `point2` within a 2000-meter circle around `point1`?\n\n > point1 = r.point(-117.220406,32.719464)\n > point2 = r.point(-117.206201,32.725186)\n > r.circle(point1, 2000).intersects(point2).run(conn)\n \n True\n\n*Example* Which of the locations in a list of parks intersect `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks')('area').intersects(circle1).run(conn)\n"), + (rethinkdb.line, b"r.line([lon1, lat1], [lon2, lat2], ...) -> line\nr.line(point1, point2, ...) -> line\n\nConstruct a geometry object of type Line. The line can be specified in one of two ways:\n\n* Two or more two-item arrays, specifying latitude and longitude numbers of the line's vertices;\n* Two or more [Point](http://rethinkdb.com/api/python/point) objects specifying the line's vertices.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\n*Example* Define a line.\n\n r.table('geo').insert({\n 'id': 101,\n 'route': r.line([-122.423246,37.779388], [-121.886420,37.329898])\n }).run(conn)\n"), + (rethinkdb.point, b"r.point(longitude, latitude) -> point\n\nConstruct a geometry object of type Point. The point is specified by two floating point numbers, the longitude (−180 to 180) and latitude (−90 to 90) of the point on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\n*Example* Define a point.\n\n r.table('geo').insert({\n 'id': 1,\n 'name': 'San Francisco',\n 'location': r.point(-122.423246,37.779388)\n }).run(conn)\n"), + (rethinkdb.polygon, b"r.polygon([lon1, lat1], [lon2, lat2], ...) -> polygon\nr.polygon(point1, point2, ...) -> polygon\n\nConstruct a geometry object of type Polygon. The Polygon can be specified in one of two ways:\n\n* Three or more two-item arrays, specifying latitude and longitude numbers of the polygon's vertices;\n* Three or more [Point](http://rethinkdb.com/api/python/point) objects specifying the polygon's vertices.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\nIf the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them. You cannot directly construct a polygon with holes in it using `polygon`, but you can use [polygon_sub](http://rethinkdb.com/api/python/polygon_sub) to use a second polygon within the interior of the first to define a hole.\n\n*Example* Define a polygon.\n\n r.table('geo').insert({\n 'id': 101,\n 'rectangle': r.polygon(\n [-122.423246,37.779388],\n [-122.423246,37.329898],\n [-121.886420,37.329898],\n [-121.886420,37.779388]\n )\n }).run(conn)\n"), + (rethinkdb.ast.RqlQuery.polygon_sub, b'polygon1.polygon_sub(polygon2) -> polygon\n\nUse `polygon2` to "punch out" a hole in `polygon1`. `polygon2` must be completely contained within `polygon1` and must have no holes itself (it must not be the output of `polygon_sub` itself).\n\n*Example* Define a polygon with a hole punched in it.\n\n outer_polygon = r.polygon(\n [-122.4,37.7],\n [-122.4,37.3],\n [-121.8,37.3],\n [-121.8,37.7]\n )\n inner_polygon = r.polygon(\n [-122.3,37.4],\n [-122.3,37.6],\n [-122.0,37.6],\n [-122.0,37.4]\n )\n outer_polygon.polygon_sub(inner_polygon).run(conn)\n'), + (rethinkdb.ast.RqlQuery.to_geojson, b"geometry.to_geojson() -> object\n\nConvert a ReQL geometry object to a [GeoJSON][] object.\n\n[GeoJSON]: http://geojson.org\n\n*Example* Convert a ReQL geometry object to a GeoJSON object.\n\n > r.table(geo).get('sfo')['location'].to_geojson().run(conn)\n \n {\n 'type': 'Point',\n 'coordinates': [ -122.423246, 37.779388 ]\n }\n"), + (rethinkdb.ast.RqlQuery.eq_join, b'sequence.eq_join(left_field, right_table[, index=\'id\']) -> sequence\n\nJoin tables using a field on the left-hand sequence matching primary keys or secondary indexes on the right-hand table. `eq_join` is more efficient than other Re_qL join types, and operates much faster. Documents in the result set consist of pairs of left-hand and right-hand documents, matched when the field on the left-hand side exists and is non-null and an entry with that field\'s value exists in the specified index on the right-hand side.\n\nThe result set of `eq_join` is a stream or array of objects. Each object in the returned set will be an object of the form `{ left: , right: }`, where the values of `left` and `right` will be the joined documents. Use the zip command to merge the `left` and `right` fields together.\n\n**Example:** Match players with the games they\'ve played against one another.\n\nThe players table contains these documents:\n\n [\n { \'id\': 1, \'player\': \'George\', \'gameId\': 1 },\n { \'id\': 2, \'player\': \'Agatha\', \'gameId\': 3 },\n { \'id\': 3, \'player\': \'Fred\', \'gameId\': 2 },\n { \'id\': 4, \'player\': \'Marie\', \'gameId\': 2 },\n { \'id\': 5, \'player\': \'Earnest\', \'gameId\': 1 },\n { \'id\': 6, \'player\': \'Beth\', \'gameId\': 3 }\n ]\n\nThe games table contains these documents:\n\n [\n { \'id\': 1, \'field\': \'Little Delving\' },\n { \'id\': 2, \'field\': \'Rushock Bog\' },\n { \'id\': 3, \'field\': \'Bucklebury\' }\n ]\n\nJoin these tables using `game_id` on the player table and `id` on the games table:\n\n r.table(\'players\').eq_join(\'game_id\', r.table(\'games\')).run(conn)\n\nThis will return a result set such as the following:\n\n [\n {\n "left" : { "gameId" : 3, "id" : 2, "player" : "Agatha" },\n "right" : { "id" : 3, "field" : "Bucklebury" }\n },\n {\n "left" : { "gameId" : 2, "id" : 3, "player" : "Fred" },\n "right" : { "id" : 2, "field" : "Rushock Bog" }\n },\n ...\n ]\n\nWhat you likely want is the result of using `zip` with that. For clarity, we\'ll use `without` to drop the `id` field from the games table (it conflicts with the `id` field for the players and it\'s redundant anyway), and we\'ll order it by the games.\n\n r.table(\'players\').eq_join(\'game_id\', r.table(\'games\')).without({\'right\': "id"}).zip().order_by(\'game_id\').run(conn)\n \n [\n { "field": "Little Delving", "gameId": 1, "id": 5, "player": "Earnest" },\n { "field": "Little Delving", "gameId": 1, "id": 1, "player": "George" },\n { "field": "Rushock Bog", "gameId": 2, "id": 3, "player": "Fred" },\n { "field": "Rushock Bog", "gameId": 2, "id": 4, "player": "Marie" },\n { "field": "Bucklebury", "gameId": 3, "id": 6, "player": "Beth" },\n { "field": "Bucklebury", "gameId": 3, "id": 2, "player": "Agatha" }\n ]\n\nFor more information, see [Table joins in Rethink_dB](http://rethinkdb.com/docs/table-joins/).\n\n**Example:** Use a secondary index on the right table rather than the primary key. If players have a secondary index on their cities, we can get a list of arenas with players in the same area.\n\n r.table(\'arenas\').eq_join(\'city_id\', r.table(\'arenas\'), index=\'city_id\').run(conn)\n\n**Example:** Use a nested key as the join field. Suppose the documents in the players table were structured like this:\n\n { \'id\': 1, \'player\': \'George\', \'game\': {\'id\': 1} },\n { \'id\': 2, \'player\': \'Agatha\', \'game\': {\'id\': 3} },\n ...\n\nSimply specify the field using the `row` command instead of a string.\n\n r.table(\'players\').eq_join(r.row[\'game\'][\'id\'], r.table(\'games\')).without({\'right\': \'id\'}).zip().run(conn)\n \n [\n { "field": "Little Delving", "game": { "id": 1 }, "id": 5, "player": "Earnest" },\n { "field": "Little Delving", "game": { "id": 1 }, "id": 1, "player": "George" },\n ...\n ]\n\n**Example:** Use a function instead of a field to join on a more complicated expression. Suppose the players have lists of favorite games ranked in order in a field such as `"favorites": [3, 2, 1]`. Get a list of players and their top favorite:\n\n r.table(\'players3\').eq_join(\n lambda player: player[\'favorites\'].nth(0),\n r.table(\'games\')\n ).without([{\'left\': [\'favorites\', \'game_id\', \'id\']}, {\'right\': \'id\'}]).zip()\n\nResult:\n\n [\n \t{ "field": "Rushock Bog", "name": "Fred" },\n \t{ "field": "Little Delving", "name": "George" },\n \t...\n ]\n'), + (rethinkdb.ast.RqlQuery.inner_join, b"sequence.inner_join(other_sequence, predicate) -> stream\narray.inner_join(other_sequence, predicate) -> array\n\nReturns an inner join of two sequences. The returned sequence represents an intersection of the left-hand sequence and the right-hand sequence: each row of the left-hand sequence will be compared with each row of the right-hand sequence to find all pairs of rows which satisfy the predicate. Each matched pair of rows of both sequences are combined into a result row. In most cases, you will want to follow the join with [zip](http://rethinkdb.com/api/python/zip) to combine the left and right results.\n\nNote that `inner_join` is slower and much less efficient than using [eq_join](http://rethinkdb.com/api/python/eq_join/) or [concat_map](http://rethinkdb.com/api/python/concat_map/) with [get_all](http://rethinkdb.com/api/python/get_all/). You should avoid using `inner_join` in commands when possible.\n\n*Example* Return a list of all matchups between Marvel and DC heroes in which the DC hero could beat the Marvel hero in a fight.\n\n r.table('marvel').inner_join(r.table('dc'),\n lambda marvel_row, dc_row: marvel_row['strength'] < dc_row['strength']\n ).zip().run(conn)\n\n(Compare this to an [outer_join](http://rethinkdb.com/api/python/outer_join) with the same inputs and predicate, which would return a list of *all* Marvel heroes along with any DC heroes with a higher strength.)"), + (rethinkdb.ast.RqlQuery.outer_join, b"sequence.outer_join(other_sequence, predicate) -> stream\narray.outer_join(other_sequence, predicate) -> array\n\nReturns a left outer join of two sequences. The returned sequence represents a union of the left-hand sequence and the right-hand sequence: all documents in the left-hand sequence will be returned, each matched with a document in the right-hand sequence if one satisfies the predicate condition. In most cases, you will want to follow the join with [zip](http://rethinkdb.com/api/python/zip) to combine the left and right results.\n\nNote that `outer_join` is slower and much less efficient than using [concat_map](http://rethinkdb.com/api/python/concat_map/) with [get_all](http://rethinkdb.com/api/python/get_all). You should avoid using `outer_join` in commands when possible.\n\n*Example* Return a list of all Marvel heroes, paired with any DC heroes who could beat them in a fight.\n\n r.table('marvel').outer_join(r.table('dc'),\n lambda marvel_row, dc_row: marvel_row['strength'] < dc_row['strength']\n ).zip().run(conn)\n\n(Compare this to an [inner_join](http://rethinkdb.com/api/python/inner_join) with the same inputs and predicate, which would return a list only of the matchups in which the DC hero has the higher strength.)\n"), + (rethinkdb.ast.RqlQuery.zip, b"stream.zip() -> stream\narray.zip() -> array\n\nUsed to 'zip' up the result of a join by merging the 'right' fields into 'left' fields of each member of the sequence.\n\n*Example* 'zips up' the sequence by merging the left and right fields produced by a join.\n\n r.table('marvel').eq_join('main_dc_collaborator', r.table('dc')).zip().run(conn)\n"), + (rethinkdb.db_create, b'r.db_create(db_name) -> object\n\nCreate a database. A RethinkDB database is a collection of tables, similar to\nrelational databases.\n\nIf successful, the command returns an object with two fields:\n\n* `dbs_created`: always `1`.\n* `config_changes`: a list containing one object with two fields, `old_val` and `new_val`:\n * `old_val`: always `None`.\n * `new_val`: the database\'s new [config](http://rethinkdb.com/api/python/config) value.\n\nIf a database with the same name already exists, the command throws `RqlRuntimeError`.\n\nNote: Only alphanumeric characters and underscores are valid for the database name.\n\n*Example* Create a database named \'superheroes\'.\n\n r.db_create(\'superheroes\').run(conn)\n \n {\n "config_changes": [\n {\n "new_val": {\n "id": "e4689cfc-e903-4532-a0e6-2d6797a43f07",\n "name": "superheroes"\n },\n "old_val": None\n }\n ],\n "dbs_created": 1\n }\n\n'), + (rethinkdb.db_drop, b'r.db_drop(db_name) -> object\n\nDrop a database. The database, all its tables, and corresponding data will be deleted.\n\nIf successful, the command returns an object with two fields:\n\n* `dbs_dropped`: always `1`.\n* `tables_dropped`: the number of tables in the dropped database.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: the database\'s original [config](http://rethinkdb.com/api/python/config) value.\n * `new_val`: always `None`.\n\nIf the given database does not exist, the command throws `RqlRuntimeError`.\n\n*Example* Drop a database named \'superheroes\'.\n\n r.db_drop(\'superheroes\').run(conn)\n \n {\n "config_changes": [\n {\n "old_val": {\n "id": "e4689cfc-e903-4532-a0e6-2d6797a43f07",\n "name": "superheroes"\n },\n "new_val": None\n }\n ],\n "tables_dropped": 3,\n "dbs_dropped": 1\n }\n\n'), + (rethinkdb.db_list, b'r.db_list() -> array\n\nList all database names in the system. The result is a list of strings.\n\n*Example* List all databases.\n\n r.db_list().run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.changes, b'stream.changes(squash=True, include_states=False) -> stream\nsingleSelection.changes(squash=True, include_states=False) -> stream\n\nReturn an infinite stream of objects representing changes to a query.\n\nThe `squash` optional argument controls how `changes` batches change notifications:\n\n* `True`: When multiple changes to the same document occur before a batch of notifications is sent, the changes are "squashed" into one change. The client receives a notification that will bring it fully up to date with the server. This is the default.\n* `False`: All changes will be sent to the client verbatim.\n* `n`: A numeric value (floating point). Similar to `True`, but the server will wait `n` seconds to respond in order to squash as many changes together as possible, reducing network traffic.\n\nIf the `include_states` optional argument is `True`, the changefeed stream will include special status documents consisting of the field `state` and a string indicating a change in the feed\'s state. These documents can occur at any point in the feed between the notification documents described below. There are currently two states:\n\n* `{"state": "initializing"}` indicates the following documents represent initial values on the feed rather than changes. This will be the first document of a feed that returns initial values.\n* `{"state": "ready"}` indicates the following documents represent changes. This will be the first document of a feed that does *not* return initial values; otherwise, it will indicate the initial values have all been sent.\n\nPoint changefeeds will always return initial values and have an `initializing` state; feeds that return changes on unfiltered tables will never return initial values. Feeds that return changes on more complex queries may or may not return return initial values, depending on the kind of aggregation. Read the article on [Changefeeds in RethinkDB][cfr] for a more detailed discussion. If `include_states` is `True` on a changefeed that does not return initial values, the first document on the feed will be `{"state": "ready"}`.\n\n[cfr]: /docs/changefeeds/python/\n\nIf `include_states` is `False` (the default), the status documents will not be sent on the feed.\n\nIf the table becomes unavailable, the changefeed will be disconnected, and a runtime exception will be thrown by the driver.\n\nChangefeed notifications take the form of a two-field object:\n\n {\n "old_val": ,\n "new_val": \n }\n\nThe first notification object in the changefeed stream will contain the query\'s initial value in `new_val` and have no `old_val` field. When a document is deleted, `new_val` will be `None`; when a document is inserted, `old_val` will be `None`.\n\nCertain document transformation commands can be chained before changefeeds. For more information, read the [discussion of changefeeds][cfr] in the "Query language" documentation.\n\nThe server will buffer up to 100,000 elements. If the buffer limit is hit, early changes will be discarded, and the client will receive an object of the form `{"error": "Changefeed cache over array size limit, skipped X elements."}` where `X` is the number of elements skipped.\n\nCommands that operate on streams (such as `filter` or `map`) can usually be chained after `changes`. However, since the stream produced by `changes` has no ending, commands that need to consume the entire stream before returning (such as `reduce` or `count`) cannot.\n\nIt\'s a good idea to open changefeeds on their own connection. If you don\'t, other queries run on the same connection will experience unpredictable latency spikes while the connection blocks on more changes.\n\n*Example* Subscribe to the changes on a table.\n\nStart monitoring the changefeed in one client:\n\n for change in r.table(\'games\').changes().run(conn):\n print change\n\nAs these queries are performed in a second client, the first client would receive and print the following objects:\n\n > r.table(\'games\').insert({\'id\': 1}).run(conn)\n {\'old_val\': None, \'new_val\': {\'id\': 1}}\n \n > r.table(\'games\').get(1).update({\'player1\': \'Bob\'}).run(conn)\n {\'old_val\': {\'id\': 1}, \'new_val\': {\'id\': 1, \'player1\': \'Bob\'}}\n \n > r.table(\'games\').get(1).replace({\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}).run(conn)\n {\'old_val\': {\'id\': 1, \'player1\': \'Bob\'},\n \'new_val\': {\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}}\n \n > r.table(\'games\').get(1).delete().run(conn)\n {\'old_val\': {\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}, \'new_val\': None}\n \n > r.table_drop(\'games\').run(conn)\n RqlRuntimeError: Changefeed aborted (table unavailable)\n\n*Example* Return all the changes that increase a player\'s score.\n\n r.table(\'test\').changes().filter(\n r.row[\'new_val\'][\'score\'] > r.row[\'old_val\'][\'score\']\n ).run(conn)\n\n*Example* Return all the changes to Bob\'s score.\n\n # Note that this will have to look at and discard all the changes to\n # rows besides Bob\'s. This is currently no way to filter with an index\n # on changefeeds.\n r.table(\'test\').changes().filter(r.row[\'new_val\'][\'name\'].eq(\'Bob\')).run(conn)\n\n*Example* Return all the inserts on a table.\n\n r.table(\'test\').changes().filter(r.row[\'old_val\'].eq(None)).run(conn)\n\n*Example* Return all the changes to game 1, with state notifications.\n\n r.table(\'games\').get(1).changes(include_states=True).run(conn)\n \n # result returned on changefeed\n {"state": "initializing"}\n {"new_val": {"id": 1, "score": 12, "arena": "Hobbiton Field"}}\n {"state": "ready"}\n {\n \t"old_val": {"id": 1, "score": 12, "arena": "Hobbiton Field"},\n \t"new_val": {"id": 1, "score": 14, "arena": "Hobbiton Field"}\n }\n {\n \t"old_val": {"id": 1, "score": 14, "arena": "Hobbiton Field"},\n \t"new_val": {"id": 1, "score": 17, "arena": "Hobbiton Field", "winner": "Frodo"}\n }\n\n*Example* Return all the changes to the top 10 games. This assumes the presence of a `score` secondary index on the `games` table.\n\n r.table(\'games\').order_by(index=r.desc(\'score\')).limit(10).run(conn)\n'), + (rethinkdb.ast.Table.index_create, b'table.index_create(index_name[, index_function][, multi=False, geo=False]) -> object\n\nCreate a new secondary index on a table. Secondary indexes improve the speed of many read queries at the slight cost of increased storage space and decreased write performance. For more information about secondary indexes, read the article "[Using secondary indexes in RethinkDB](http://rethinkdb.com/docs/secondary-indexes/)."\n\nRethinkDB supports different types of secondary indexes:\n\n- *Simple indexes* based on the value of a single field.\n- *Compound indexes* based on multiple fields.\n- *Multi indexes* based on arrays of values.\n- *Geospatial indexes* based on indexes of geometry objects, created when the `geo` optional argument is true.\n- Indexes based on *arbitrary expressions*.\n\nThe `index_function` can be an anonymous function or a binary representation obtained from the `function` field of [index_status](http://rethinkdb.com/api/python/index_status).\n\nIf successful, `create_index` will return an object of the form `{"created": 1}`. If an index by that name already exists on the table, a `RqlRuntimeError` will be thrown.\n\n*Example* Create a simple index based on the field `post_id`.\n\n r.table(\'comments\').index_create(\'post_id\').run(conn)\n*Example* Create a simple index based on the nested field `author > name`.\n\n r.table(\'comments\').index_create(\'author_name\', r.row["author"]["name"]).run(conn)\n\n*Example* Create a geospatial index based on the field `location`.\n\n r.table(\'places\').index_create(\'location\', geo=True).run(conn)\n\nA geospatial index field should contain only geometry objects. It will work with geometry ReQL terms ([get_intersecting](http://rethinkdb.com/api/python/get_intersecting/) and [get_nearest](http://rethinkdb.com/api/python/get_nearest/)) as well as index-specific terms ([index_status](http://rethinkdb.com/api/python/index_status), [index_wait](http://rethinkdb.com/api/python/index_wait), [index_drop](http://rethinkdb.com/api/python/index_drop) and [index_list](http://rethinkdb.com/api/python/index_list)). Using terms that rely on non-geometric ordering such as [get_all](http://rethinkdb.com/api/python/get_all/), [order_by](http://rethinkdb.com/api/python/order_by/) and [between](http://rethinkdb.com/api/python/order_by/) will result in an error.\n\n*Example* Create a compound index based on the fields `post_id` and `date`.\n\n r.table(\'comments\').index_create(\'post_and_date\', [r.row["post_id"], r.row["date"]]).run(conn)\n\n*Example* Create a multi index based on the field `authors`.\n\n r.table(\'posts\').index_create(\'authors\', multi=True).run(conn)\n\n*Example* Create a geospatial multi index based on the field `towers`.\n\n r.table(\'networks\').index_create(\'towers\', geo=True, multi=True).run(conn)\n\n*Example* Create an index based on an arbitrary expression.\n\n r.table(\'posts\').index_create(\'authors\', lambda doc:\n r.branch(\n doc.has_fields("updated_at"),\n doc["updated_at"],\n doc["created_at"]\n )\n ).run(conn)\n\n*Example* Create a new secondary index based on an existing one.\n\n index = r.table(\'posts\').index_status(\'authors\').nth(0)[\'function\'].run(conn)\n r.table(\'new_posts\').index_create(\'authors\', index).run(conn)\n\n*Example* Rebuild an outdated secondary index on a table.\n\n old_index = r.table(\'posts\').index_status(\'old_index\').nth(0)[\'function\'].run(conn)\n r.table(\'posts\').index_create(\'new_index\', old_index).run(conn)\n r.table(\'posts\').index_wait(\'new_index\').run(conn)\n r.table(\'posts\').index_rename(\'new_index\', \'old_index\', overwrite=True).run(conn)\n'), + (rethinkdb.ast.Table.index_drop, b"table.index_drop(index_name) -> object\n\nDelete a previously created secondary index of this table.\n\n*Example* Drop a secondary index named 'code_name'.\n\n r.table('dc').index_drop('code_name').run(conn)\n\n"), + (rethinkdb.ast.Table.index_list, b"table.index_list() -> array\n\nList all the secondary indexes of this table.\n\n*Example* List the available secondary indexes for this table.\n\n r.table('marvel').index_list().run(conn)\n"), + (rethinkdb.ast.Table.index_rename, b"table.index_rename(old_index_name, new_index_name[, overwrite=False]) -> object\n\nRename an existing secondary index on a table. If the optional argument `overwrite` is specified as `True`, a previously existing index with the new name will be deleted and the index will be renamed. If `overwrite` is `False` (the default) an error will be raised if the new index name already exists.\n\nThe return value on success will be an object of the format `{'renamed': 1}`, or `{'renamed': 0}` if the old and new names are the same.\n\nAn error will be raised if the old index name does not exist, if the new index name is already in use and `overwrite` is `False`, or if either the old or new index name are the same as the primary key field name.\n\n*Example* Rename an index on the comments table.\n\n r.table('comments').index_rename('post_id', 'message_id').run(conn)\n"), + (rethinkdb.ast.Table.index_status, b'table.index_status([, index...]) -> array\n\nGet the status of the specified indexes on this table, or the status\nof all indexes on this table if no indexes are specified.\n\nThe result is an array where for each index, there will be an object like this one:\n\n {\n "index": ,\n "ready": True,\n "function": ,\n "multi": ,\n "outdated": \n }\n\nor this one:\n\n {\n "index": ,\n "ready": False,\n "blocks_processed": ,\n "blocks_total": ,\n "function": ,\n "multi": ,\n "outdated": \n }\n\nThe `multi` field will be `true` or `false` depending on whether this index was created as a multi index (see [index_create](http://rethinkdb.com/api/python/index_create/) for details). The `outdated` field will be true if the index is outdated in the current version of RethinkDB and needs to be rebuilt.\n\nThe `function` field is a binary object containing an opaque representation of the secondary index (including the `multi` argument if specified). It can be passed as the second argument to [index_create](http://rethinkdb.com/api/python/index_create/) to create a new index with the same function; see `index_create` for more information.\n\n*Example* Get the status of all the indexes on `test`:\n\n r.table(\'test\').index_status().run(conn)\n\n*Example* Get the status of the `timestamp` index:\n\n r.table(\'test\').index_status(\'timestamp\').run(conn)\n\n*Example* Save the binary representation of the index:\n\n func = r.table(\'test\').index_status(\'timestamp\').nth(0)[\'function\'].run(conn)\n'), + (rethinkdb.ast.Table.index_wait, b'table.index_wait([, index...]) -> array\n\nWait for the specified indexes on this table to be ready, or for all\nindexes on this table to be ready if no indexes are specified.\n\nThe result is an array containing one object for each table index:\n\n {\n "index": ,\n "ready": True,\n "function": ,\n "multi": ,\n "geo": ,\n "outdated": \n }\n\nSee the [index_status](http://rethinkdb.com/api/python/index_status) documentation for a description of the field values.\n\n*Example* Wait for all indexes on the table `test` to be ready:\n\n r.table(\'test\').index_wait().run(conn)\n\n*Example* Wait for the index `timestamp` to be ready:\n\n r.table(\'test\').index_wait(\'timestamp\').run(conn)\n'), + (rethinkdb.ast.DB.table_create, b'db.table_create(table_name[, options]) -> object\n\nCreate a table. A RethinkDB table is a collection of JSON documents.\n\nIf successful, the command returns an object with two fields:\n\n* `tables_created`: always `1`.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: always `None`.\n * `new_val`: the table\'s new [config](http://rethinkdb.com/api/python/config) value.\n\nIf a table with the same name already exists, the command throws `RqlRuntimeError`.\n\nNote: Only alphanumeric characters and underscores are valid for the table name.\n\nWhen creating a table you can specify the following options:\n\n* `primary_key`: the name of the primary key. The default primary key is `id`.\n* `durability`: if set to `soft`, writes will be acknowledged by the server immediately and flushed to disk in the background. The default is `hard`: acknowledgment of writes happens after data has been written to disk.\n* `shards`: the number of shards, an integer from 1-32. Defaults to `1`.\n* `replicas`: either an integer or a mapping object. Defaults to `1`.\n * If `replicas` is an integer, it specifies the number of replicas per shard. Specifying more replicas than there are servers will return an error.\n * If `replicas` is an object, it specifies key-value pairs of server tags and the number of replicas to assign to those servers: `{\'tag1\': 2, \'tag2\': 4, \'tag3\': 2, ...}`.\n* `primary_replica_tag`: the primary server specified by its server tag. Required if `replicas` is an object; the tag must be in the object. This must *not* be specified if `replicas` is an integer.\n\nThe [data type](http://rethinkdb.com/docs/data-types/) of a primary key is usually a string (like a UUID) or a number, but it can also be a time, binary object, boolean or an array. It cannot be an object.\n\n*Example* Create a table named \'dc_universe\' with the default settings.\n\n r.db(\'test\').table_create(\'dc_universe\').run(conn)\n \n {\n "config_changes": [\n {\n "new_val": {\n "db": "test",\n "durability": "hard",\n "id": "20ea60d4-3b76-4817-8828-98a236df0297",\n "name": "dc_universe",\n "primary_key": "id",\n "shards": [\n {\n "primary_replica": "rethinkdb_srv1",\n "replicas": [\n "rethinkdb_srv1",\n "rethinkdb_srv2"\n ]\n }\n ],\n "write_acks": "majority"\n },\n "old_val": None\n }\n ],\n "tables_created": 1\n }\n\n*Example* Create a table named \'dc_universe\' using the field \'name\' as primary key.\n\n r.db(\'test\').table_create(\'dc_universe\', primary_key=\'name\').run(conn)\n\n*Example* Create a table set up for two shards and three replicas per shard. This requires three available servers.\n\n r.db(\'test\').table_create(\'dc_universe\', shards=2, replicas=3).run(conn)\n\nRead [Sharding and replication](http://rethinkdb.com/docs/sharding-and-replication/) for a complete discussion of the subject, including advanced topics.\n'), + (rethinkdb.ast.DB.table_drop, b'db.table_drop(table_name) -> object\n\nDrop a table. The table and all its data will be deleted.\n\nIf successful, the command returns an object with two fields:\n\n* `tables_dropped`: always `1`.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: the dropped table\'s [config](http://rethinkdb.com/api/python/config) value.\n * `new_val`: always `None`.\n\nIf the given table does not exist in the database, the command throws `RqlRuntimeError`.\n\n*Example* Drop a table named \'dc_universe\'.\n\n r.db(\'test\').table_drop(\'dc_universe\').run(conn)\n \n {\n "config_changes": [\n {\n "old_val": {\n "db": "test",\n "durability": "hard",\n "id": "20ea60d4-3b76-4817-8828-98a236df0297",\n "name": "dc_universe",\n "primary_key": "id",\n "shards": [\n {\n "primary_replica": "rethinkdb_srv1",\n "replicas": [\n "rethinkdb_srv1",\n "rethinkdb_srv2"\n ]\n }\n ],\n "write_acks": "majority"\n },\n "new_val": None\n }\n ],\n "tables_dropped": 1\n }\n'), + (rethinkdb.ast.DB.table_list, b"db.table_list() -> array\n\nList all table names in a database. The result is a list of strings.\n\n*Example* List all tables of the 'test' database.\n\n r.db('test').table_list().run(conn)\n \n"), + (rethinkdb.ast.RqlQuery.__add__, b'number + number -> number\nstring + string -> string\narray + array -> array\ntime + number -> time\n\nSum two numbers, concatenate two strings, or concatenate 2 arrays.\n\n*Example* It\'s as easy as 2 + 2 = 4.\n\n > (r.expr(2) + 2).run(conn)\n \n 4\n\n*Example* Strings can be concatenated too.\n\n > (r.expr("foo") + "bar").run(conn)\n \n "foobar"\n\n*Example* Arrays can be concatenated too.\n\n > (r.expr(["foo", "bar"]) + ["buzz"]).run(conn)\n \n [\'foo\', \'bar\', \'buzz\']\n\n*Example* Create a date one year from now.\n\n r.now() + 365*24*60*60\n\n*Example* Use [args](http://rethinkdb.com/api/python/args) with `add` to sum multiple values.\n\n > r.add(r.args([10, 20, 30])).run(conn)\n \n 60\n\n*Example* Concatenate an array of strings with `args`.\n\n > r.add(r.args([\'foo\', \'bar\', \'buzz\'])).run(conn)\n \n "foobarbuzz"\n'), + (rethinkdb.add, b'number + number -> number\nstring + string -> string\narray + array -> array\ntime + number -> time\n\nSum two numbers, concatenate two strings, or concatenate 2 arrays.\n\n*Example* It\'s as easy as 2 + 2 = 4.\n\n > (r.expr(2) + 2).run(conn)\n \n 4\n\n*Example* Strings can be concatenated too.\n\n > (r.expr("foo") + "bar").run(conn)\n \n "foobar"\n\n*Example* Arrays can be concatenated too.\n\n > (r.expr(["foo", "bar"]) + ["buzz"]).run(conn)\n \n [\'foo\', \'bar\', \'buzz\']\n\n*Example* Create a date one year from now.\n\n r.now() + 365*24*60*60\n\n*Example* Use [args](http://rethinkdb.com/api/python/args) with `add` to sum multiple values.\n\n > r.add(r.args([10, 20, 30])).run(conn)\n \n 60\n\n*Example* Concatenate an array of strings with `args`.\n\n > r.add(r.args([\'foo\', \'bar\', \'buzz\'])).run(conn)\n \n "foobarbuzz"\n'), + (rethinkdb.ast.RqlQuery.__and__, b'bool & bool -> bool\nr.and_(bool, bool) -> bool\nbool.and_(bool) -> bool\n\nCompute the logical "and" of two or more values. The `and_` command can be used as an infix operator after its first argument (`r.expr(True).and_(False)`) or given all of its arguments as parameters (`r.and_(True, False)`). The standard Python and operator, `&`, may also be used with ReQL.\n\n*Example* Return whether both `a` and `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) & b).run(conn)\n \n False\n*Example* Return whether all of `x`, `y` and `z` evaluate to true.\n\n > x = True\n > y = True\n > z = True\n > r.and_(x, y, z).run(conn)\n \n True\n'), + (rethinkdb.and_, b'bool & bool -> bool\nr.and_(bool, bool) -> bool\nbool.and_(bool) -> bool\n\nCompute the logical "and" of two or more values. The `and_` command can be used as an infix operator after its first argument (`r.expr(True).and_(False)`) or given all of its arguments as parameters (`r.and_(True, False)`). The standard Python and operator, `&`, may also be used with ReQL.\n\n*Example* Return whether both `a` and `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) & b).run(conn)\n \n False\n*Example* Return whether all of `x`, `y` and `z` evaluate to true.\n\n > x = True\n > y = True\n > z = True\n > r.and_(x, y, z).run(conn)\n \n True\n'), + (rethinkdb.ast.RqlQuery.__div__, b"number / number -> number\n\nDivide two numbers.\n\n*Example* It's as easy as 2 / 2 = 1.\n\n (r.expr(2) / 2).run(conn)\n"), + (rethinkdb.div, b"number / number -> number\n\nDivide two numbers.\n\n*Example* It's as easy as 2 / 2 = 1.\n\n (r.expr(2) / 2).run(conn)\n"), + (rethinkdb.ast.RqlQuery.__eq__, b'value == value -> bool\nvalue.eq(value) -> bool\n\nTest if two values are equal.\n\n*Example* Does 2 equal 2?\n\n (r.expr(2) == 2).run(conn)\n r.expr(2).eq(2).run(conn)\n'), + (rethinkdb.ast.RqlQuery.eq, b'value == value -> bool\nvalue.eq(value) -> bool\n\nTest if two values are equal.\n\n*Example* Does 2 equal 2?\n\n (r.expr(2) == 2).run(conn)\n r.expr(2).eq(2).run(conn)\n'), + (rethinkdb.ast.RqlQuery.__ge__, b'value >= value -> bool\nvalue.ge(value) -> bool\n\nTest if the first value is greater than or equal to other.\n\n*Example* Is 2 greater than or equal to 2?\n\n (r.expr(2) >= 2).run(conn)\n r.expr(2).ge(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.ge, b'value >= value -> bool\nvalue.ge(value) -> bool\n\nTest if the first value is greater than or equal to other.\n\n*Example* Is 2 greater than or equal to 2?\n\n (r.expr(2) >= 2).run(conn)\n r.expr(2).ge(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__gt__, b'value > value -> bool\nvalue.gt(value) -> bool\n\nTest if the first value is greater than other.\n\n*Example* Is 2 greater than 2?\n\n (r.expr(2) > 2).run(conn)\n r.expr(2).gt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.gt, b'value > value -> bool\nvalue.gt(value) -> bool\n\nTest if the first value is greater than other.\n\n*Example* Is 2 greater than 2?\n\n (r.expr(2) > 2).run(conn)\n r.expr(2).gt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__le__, b'value <= value -> bool\nvalue.le(value) -> bool\n\nTest if the first value is less than or equal to other.\n\n*Example* Is 2 less than or equal to 2?\n\n (r.expr(2) <= 2).run(conn)\n r.expr(2).le(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.le, b'value <= value -> bool\nvalue.le(value) -> bool\n\nTest if the first value is less than or equal to other.\n\n*Example* Is 2 less than or equal to 2?\n\n (r.expr(2) <= 2).run(conn)\n r.expr(2).le(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__lt__, b'value < value -> bool\nvalue.lt(value) -> bool\n\nTest if the first value is less than other.\n\n*Example* Is 2 less than 2?\n\n (r.expr(2) < 2).run(conn)\n r.expr(2).lt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.lt, b'value < value -> bool\nvalue.lt(value) -> bool\n\nTest if the first value is less than other.\n\n*Example* Is 2 less than 2?\n\n (r.expr(2) < 2).run(conn)\n r.expr(2).lt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__mod__, b"number % number -> number\n\nFind the remainder when dividing two numbers.\n\n*Example* It's as easy as 2 % 2 = 0.\n\n (r.expr(2) % 2).run(conn)\n\n`\n"), + (rethinkdb.mod, b"number % number -> number\n\nFind the remainder when dividing two numbers.\n\n*Example* It's as easy as 2 % 2 = 0.\n\n (r.expr(2) % 2).run(conn)\n\n`\n"), + (rethinkdb.ast.RqlQuery.__mul__, b'number * number -> number\narray * number -> array\n\nMultiply two numbers, or make a periodic array.\n\n*Example* It\'s as easy as 2 * 2 = 4.\n\n (r.expr(2) * 2).run(conn)\n\n*Example* Arrays can be multiplied by numbers as well.\n\n (r.expr(["This", "is", "the", "song", "that", "never", "ends."]) * 100).run(conn)\n\n'), + (rethinkdb.mul, b'number * number -> number\narray * number -> array\n\nMultiply two numbers, or make a periodic array.\n\n*Example* It\'s as easy as 2 * 2 = 4.\n\n (r.expr(2) * 2).run(conn)\n\n*Example* Arrays can be multiplied by numbers as well.\n\n (r.expr(["This", "is", "the", "song", "that", "never", "ends."]) * 100).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__ne__, b'value != value -> bool\nvalue.ne(value) -> bool\n\nTest if two values are not equal.\n\n*Example* Does 2 not equal 2?\n\n (r.expr(2) != 2).run(conn)\n r.expr(2).ne(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.ne, b'value != value -> bool\nvalue.ne(value) -> bool\n\nTest if two values are not equal.\n\n*Example* Does 2 not equal 2?\n\n (r.expr(2) != 2).run(conn)\n r.expr(2).ne(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__invert__, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.not_, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), + (rethinkdb.not_, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.__or__, b'bool | bool -> bool\nbool.or_(bool[, bool, ...]) -> bool\nr.or_(bool, bool) -> bool\n\nCompute the logical "or" of two or more values. The `or_` command can be used as an infix operator after its first argument (`r.expr(True).or_(False)`) or given all of its arguments as parameters (`r.or_(True, False)`). The standard Python or operator, `|`, may also be used with ReQL.\n\n*Example* Return whether either `a` or `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) | b).run(conn)\n \n True\n\n*Example* Return whether any of `x`, `y` or `z` evaluate to true.\n\n > x = False\n > y = False\n > z = False\n > r.or_(x, y, z).run(conn)\n \n False\n\n__Note:__ When using `or` inside a `filter` predicate to test the values of fields that may not exist on the documents being tested, you should use the `default` command with those fields so they explicitly return `False`.\n\n r.table(\'posts\').filter(lambda post:\n post[\'category\'].default(\'foo\').eq(\'article\').or(\n post[\'genre\'].default(\'foo\').eq(\'mystery\'))\n ).run(conn)\n'), + (rethinkdb.or_, b'bool | bool -> bool\nbool.or_(bool[, bool, ...]) -> bool\nr.or_(bool, bool) -> bool\n\nCompute the logical "or" of two or more values. The `or_` command can be used as an infix operator after its first argument (`r.expr(True).or_(False)`) or given all of its arguments as parameters (`r.or_(True, False)`). The standard Python or operator, `|`, may also be used with ReQL.\n\n*Example* Return whether either `a` or `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) | b).run(conn)\n \n True\n\n*Example* Return whether any of `x`, `y` or `z` evaluate to true.\n\n > x = False\n > y = False\n > z = False\n > r.or_(x, y, z).run(conn)\n \n False\n\n__Note:__ When using `or` inside a `filter` predicate to test the values of fields that may not exist on the documents being tested, you should use the `default` command with those fields so they explicitly return `False`.\n\n r.table(\'posts\').filter(lambda post:\n post[\'category\'].default(\'foo\').eq(\'article\').or(\n post[\'genre\'].default(\'foo\').eq(\'mystery\'))\n ).run(conn)\n'), + (rethinkdb.random, b"r.random() -> number\nr.random(number[, number], float=True) -> number\nr.random(integer[, integer]) -> integer\n\nGenerate a random number between given (or implied) bounds. `random` takes zero, one or two arguments.\n\n- With __zero__ arguments, the result will be a floating-point number in the range `[0,1)` (from 0 up to but not including 1).\n- With __one__ argument _x,_ the result will be in the range `[0,x)`, and will be integer unless `float=True` is given as an option. Specifying a floating point number without the `float` option will raise an error.\n- With __two__ arguments _x_ and _y,_ the result will be in the range `[x,y)`, and will be integer unless `float=True` is given as an option. If _x_ and _y_ are equal an error will occur, unless the floating-point option has been specified, in which case _x_ will be returned. Specifying a floating point number without the `float` option will raise an error.\n\nNote: The last argument given will always be the 'open' side of the range, but when\ngenerating a floating-point number, the 'open' side may be less than the 'closed' side.\n\n*Example* Generate a random number in the range `[0,1)`\n\n r.random().run(conn)\n\n*Example* Generate a random integer in the range `[0,100)`\n\n r.random(100).run(conn)\n r.random(0, 100).run(conn)\n\n*Example* Generate a random number in the range `(-2.24,1.59]`\n\n r.random(1.59, -2.24, float=True).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.__sub__, b"number - number -> number\ntime - time -> number\ntime - number -> time\n\nSubtract two numbers.\n\n*Example* It's as easy as 2 - 2 = 0.\n\n (r.expr(2) - 2).run(conn)\n\n*Example* Create a date one year ago today.\n\n r.now() - 365*24*60*60\n\n*Example* Retrieve how many seconds elapsed between today and date\n\n r.now() - date\n\n"), + (rethinkdb.sub, b"number - number -> number\ntime - time -> number\ntime - number -> time\n\nSubtract two numbers.\n\n*Example* It's as easy as 2 - 2 = 0.\n\n (r.expr(2) - 2).run(conn)\n\n*Example* Create a date one year ago today.\n\n r.now() - 365*24*60*60\n\n*Example* Retrieve how many seconds elapsed between today and date\n\n r.now() - date\n\n"), + (rethinkdb.ast.Table.between, b'table.between(lower_key, upper_key[, index=\'id\', left_bound=\'closed\', right_bound=\'open\'])\n -> selection\n\nGet all documents between two keys. Accepts three optional arguments: `index`, `left_bound`, and `right_bound`. If `index` is set to the name of a secondary index, `between` will return all documents where that index\'s value is in the specified range (it uses the primary key by default). `left_bound` or `right_bound` may be set to `open` or `closed` to indicate whether or not to include that endpoint of the range (by default, `left_bound` is closed and `right_bound` is open).\n\nYou may also use the special constants `r.minval` and `r.maxval` for boundaries, which represent "less than any index key" and "more than any index key" respectively. For instance, if you use `r.minval` as the lower key, then `between` will return all documents whose primary keys (or indexes) are less than the specified upper key.\n\nNote that compound indexes are sorted using [lexicographical order][lo]. Take the following range as an example:\n\n\t[[1, "c"] ... [5, "e"]]\n\nThis range includes all compound keys:\n\n* whose first item is 1 and second item is equal or greater than "c";\n* whose first item is between 1 and 5, *regardless of the value of the second item*;\n* whose first item is 5 and second item is less than or equal to "e".\n\n[lo]: https://en.wikipedia.org/wiki/Lexicographical_order\n\n*Example* Find all users with primary key >= 10 and < 20 (a normal half-open interval).\n\n r.table(\'marvel\').between(10, 20).run(conn)\n\n*Example* Find all users with primary key >= 10 and <= 20 (an interval closed on both sides).\n\n r.table(\'marvel\').between(10, 20, right_bound=\'closed\').run(conn)\n\n*Example* Find all users with primary key < 20.\n\n r.table(\'marvel\').between(r.minval, 20).run(conn)\n\n*Example* Find all users with primary key > 10.\n\n r.table(\'marvel\').between(10, r.maxval, left_bound=\'open\').run(conn)\n\n*Example* Between can be used on secondary indexes too. Just pass an optional index argument giving the secondary index to query.\n\n r.table(\'dc\').between(\'dark_knight\', \'man_of_steel\', index=\'code_name\').run(conn)\n\n*Example* Get all users whose full name is between "John Smith" and "Wade Welles."\n\n r.table("users").between(["Smith", "John"], ["Welles", "Wade"],\n index="full_name").run(conn)\n\n*Example* Subscribe to a [changefeed](http://rethinkdb.com/docs/changefeeds/javascript) of teams ranked in the top 10.\n\n changes = r.table("teams").between(1, 11, index="rank").changes().run(conn)\n\n__Note:__ Between works with secondary indexes on date fields, but will not work with unindexed date fields. To test whether a date value is between two other dates, use the [during](http://rethinkdb.com/api/python/during) command, not `between`.\n\nSecondary indexes can be used in extremely powerful ways with `between` and other commands; read the full article on [secondary indexes](http://rethinkdb.com/docs/secondary-indexes) for examples using boolean operations, `contains` and more.\n\n__Note:__ RethinkDB uses byte-wise ordering for `between` and does not support Unicode collations; non-ASCII characters will be sorted by UTF-8 codepoint.\n\n__Note:__ If you chain `between` after [order_by](http://rethinkdb.com/api/python/order_by), the `between` command must use the index specified in `order_by`, and will default to that index. Trying to specify another index will result in a `RqlRuntimeError`.\n'), + (rethinkdb.db, b"r.db(db_name) -> db\n\nReference a database.\n\n*Example* Before we can query a table we have to select the correct database.\n\n r.db('heroes').table('marvel').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.filter, b'selection.filter(predicate[, default=False]) -> selection\nstream.filter(predicate[, default=False]) -> stream\narray.filter(predicate[, default=False]) -> array\n\nReturn all the elements in a sequence for which the given predicate is true. The return value of `filter` will be the same as the input (sequence, stream, or array). Documents can be filtered in a variety of ways—ranges, nested values, boolean conditions, and the results of anonymous functions.\n\nBy default, `filter` will silently skip documents with missing fields: if the predicate tries to access a field that doesn\'t exist (for instance, the predicate `{\'age\': 30}` applied to a document with no `age` field), that document will not be returned in the result set, and no error will be generated. This behavior can be changed with the `default` optional argument.\n\n* If `default` is set to `True`, documents with missing fields will be returned rather than skipped.\n* If `default` is set to `r.error()`, an `RqlRuntimeError` will be thrown when a document with a missing field is tested.\n* If `default` is set to `False` (the default), documents with missing fields will be skipped.\n\n*Example* Get all users who are 30 years old.\n\n r.table(\'users\').filter({\'age\': 30}).run(conn)\n\nThe predicate `{\'age\': 30}` selects documents in the `users` table with an `age` field whose value is `30`. Documents with an `age` field set to any other value *or* with no `age` field present are skipped.\n\nWhile the `{\'field\': value}` style of predicate is useful for exact matches, a more general way to write a predicate is to use the [row](http://rethinkdb.com/api/python/row) command with a comparison operator such as [eq](http://rethinkdb.com/api/python/eq) (`==`) or [gt](http://rethinkdb.com/api/python/gt) (`>`), or to use a lambda function that returns `True` or `False`.\n\n r.table(\'users\').filter(r.row["age"] == 30).run(conn)\n\nIn this case, the predicate `r.row["age"] == 30` returns `True` if the field `age` is equal to 30. You can write this predicate as a lambda function instead:\n\n r.table(\'users\').filter(lambda user:\n user["age"] == 30\n ).run(conn)\n\nPredicates to `filter` are evaluated on the server, and must use ReQL expressions. Some Python comparison operators are overloaded by the RethinkDB driver and will be translated to ReQL, such as `==`, `<`/`>` and `|`/`&` (note the single character form, rather than `||`/`&&`).\n\nAlso, predicates must evaluate document fields. They cannot evaluate [secondary indexes](http://rethinkdb.com/docs/secondary-indexes/).\n\n*Example* Get all users who are more than 18 years old.\n\n r.table("users").filter(r.row["age"] > 18).run(conn)\n\n*Example* Get all users who are less than 18 years old and more than 13 years old.\n\n r.table("users").filter((r.row["age"] < 18) & (r.row["age"] > 13)).run(conn)\n\n*Example* Get all users who are more than 18 years old or have their parental consent.\n\n r.table("users").filter(\n (r.row["age"] >= 18) | (r.row["hasParentalConsent"])).run(conn)\n\n*Example* Retrieve all users who subscribed between January 1st, 2012\n(included) and January 1st, 2013 (excluded).\n\n r.table("users").filter(\n lambda user: user["subscription_date"].during(\n r.time(2012, 1, 1, \'Z\'), r.time(2013, 1, 1, \'Z\'))\n ).run(conn)\n\n*Example* Retrieve all users who have a gmail account (whose field `email` ends with `@gmail.com`).\n\n r.table("users").filter(\n lambda user: user["email"].match("@gmail.com$")\n ).run(conn)\n\n*Example* Filter based on the presence of a value in an array.\n\nGiven this schema for the `users` table:\n\n {\n "name": \n "places_visited": []\n }\n\nRetrieve all users whose field `places_visited` contains `France`.\n\n r.table("users").filter(lambda user:\n user["places_visited"].contains("France")\n ).run(conn)\n\n*Example* Filter based on nested fields.\n\nGiven this schema for the `users` table:\n\n {\n "id": \n "name": {\n "first": ,\n "middle": ,\n "last": \n }\n }\n\nRetrieve all users named "William Adama" (first name "William", last name\n"Adama"), with any middle name.\n\n r.table("users").filter({\n "name": {\n "first": "William",\n "last": "Adama"\n }\n }).run(conn)\n\nIf you want an exact match for a field that is an object, you will have to use `r.literal`.\n\nRetrieve all users named "William Adama" (first name "William", last name\n"Adama"), and who do not have a middle name.\n\n r.table("users").filter(r.literal({\n "name": {\n "first": "William",\n "last": "Adama"\n }\n })).run(conn)\n\nYou may rewrite these with lambda functions.\n\n r.table("users").filter(\n lambda user:\n (user["name"]["first"] == "William")\n & (user["name"]["last"] == "Adama")\n ).run(conn)\n\n r.table("users").filter(lambda user:\n user["name"] == {\n "first": "William",\n "last": "Adama"\n }\n ).run(conn)\n\nBy default, documents missing fields tested by the `filter` predicate are skipped. In the previous examples, users without an `age` field are not returned. By passing the optional `default` argument to `filter`, you can change this behavior.\n\n*Example* Get all users less than 18 years old or whose `age` field is missing.\n\n r.table("users").filter(r.row["age"] < 18, default=True).run(conn)\n\n*Example* Get all users more than 18 years old. Throw an error if a\ndocument is missing the field `age`.\n\n r.table("users").filter(r.row["age"] > 18, default=r.error()).run(conn)\n\n*Example* Get all users who have given their phone number (all the documents whose field `phone_number` exists and is not `None`).\n\n r.table(\'users\').filter(\n lambda user: user.has_fields(\'phone_number\')\n ).run(conn)\n\n*Example* Get all users with an "editor" role or an "admin" privilege.\n\n r.table(\'users\').filter(\n lambda user: (user[\'role\'] == \'editor\').default(False) |\n (user[\'privilege\'] == \'admin\').default(False)\n ).run(conn)\n\nInstead of using the `default` optional argument to `filter`, we have to use default values on the fields within the `or` clause. Why? If the field on the left side of the `or` clause is missing from a document—in this case, if the user doesn\'t have a `role` field—the predicate will generate an error, and will return `False` (or the value the `default` argument is set to) without evaluating the right side of the `or`. By using `.default(False)` on the fields, each side of the `or` will evaluate to either the field\'s value or `False` if the field doesn\'t exist.\n'), + (rethinkdb.ast.Table.get, b"table.get(key) -> singleRowSelection\n\nGet a document by primary key.\n\nIf no document exists with that primary key, `get` will return `None`.\n\n*Example* Find a document by UUID.\n\n r.table('posts').get('a9849eef-7176-4411-935b-79a6e3c56a74').run(conn)\n\n*Example* Find a document and merge another document with it.\n\n r.table('heroes').get(3).merge(\n { 'powers': ['invisibility', 'speed'] }\n ).run(conn)\n\n_*Example* Subscribe to a document's [changefeed](http://rethinkdb.com/docs/changefeeds/python).\n\n changes = r.table('heroes').get(3).changes().run(conn)\n"), + (rethinkdb.ast.Table.get_all, b"table.get_all(key1[, key2...], [, index='id']) -> selection\n\nGet all documents where the given value matches the value of the requested index.\n\n*Example* Secondary index keys are not guaranteed to be unique so we cannot query via [get](http://rethinkdb.com/api/python/get/) when using a secondary index.\n\n r.table('marvel').get_all('man_of_steel', index='code_name').run(conn)\n\n*Example* Without an index argument, we default to the primary index. While `get` will either return the document or `None` when no document with such a primary key value exists, this will return either a one or zero length stream.\n\n r.table('dc').get_all('superman').run(conn)\n\n*Example* You can get multiple documents in a single call to `get_all`.\n\n r.table('dc').get_all('superman', 'ant man').run(conn)\n\n*Example* You can use [args](http://rethinkdb.com/api/python/args/) with `get_all` to retrieve multiple documents whose keys are in a list. This uses `get_all` to get a list of female superheroes, coerces that to an array, and then gets a list of villains who have those superheroes as enemies.\n\n r.do(\n r.table('heroes').get_all('f', {'index': 'gender'})['id'].coerce_to('array'), \n lamdba heroines: r.table('villains').get_all(r.args(heroines))\n ).run(conn)\n\nSecondary indexes can be used in extremely powerful ways with `get_all` and other commands; read the full article on [secondary indexes](http://rethinkdb.com/docs/secondary-indexes) for examples using boolean operations, `contains` and more.\n"), + (rethinkdb.ast.DB.table, b"db.table(name[, use_outdated=False, identifier_format='name']) -> table\n\nReturn all documents in a table. Other commands may be chained after `table` to return a subset of documents (such as `get` and `filter`) or perform further processing.\n\n*Example* Return all documents in the table 'marvel' of the default database.\n\n r.table('marvel').run(conn)\n\n*Example* Return all documents in the table 'marvel' of the database 'heroes'.\n\n r.db('heroes').table('marvel').run(conn)\n\nThere are two optional arguments.\n\n* `use_outdated`: if `True`, this allows potentially out-of-date data to be returned, with potentially faster reads. It also allows you to perform reads from a secondary replica if a primary has failed. Default `False`.\n* `identifier_format`: possible values are `name` and `uuid`, with a default of `name`. If set to `uuid`, then [system tables](http://rethinkdb.com/docs/system-tables/) will refer to servers, databases and tables by UUID rather than name. (This only has an effect when used with system tables.)\n\n*Example* Allow potentially out-of-date data in exchange for faster reads.\n\n r.db('heroes').table('marvel', use_outdated=True).run(conn)\n"), + (rethinkdb.ast.RqlQuery.downcase, b'string.downcase() -> string\n\nLowercases a string.\n\n*Example*\n\n > r.expr("Sentence about LaTeX.").downcase().run(conn)\n "sentence about latex."\n\n__Note:__ `upcase` and `downcase` only affect ASCII characters.\n'), + (rethinkdb.ast.RqlQuery.match, b'string.match(regexp) -> None/object\n\nMatches against a regular expression. If there is a match, returns an object with the fields:\n\n- `str`: The matched string\n- `start`: The matched string\'s start\n- `end`: The matched string\'s end\n- `groups`: The capture groups defined with parentheses\n\nIf no match is found, returns `None`.\n\nAccepts RE2 syntax\n([https://code.google.com/p/re2/wiki/Syntax](https://code.google.com/p/re2/wiki/Syntax)).\nYou can enable case-insensitive matching by prefixing the regular expression with\n`(?i)`. See the linked RE2 documentation for more flags.\n\nThe `match` command does not support backreferences.\n\n*Example* Get all users whose name starts with "A". Because `None` evaluates to `false` in\n`filter`, you can just use the result of `match` for the predicate.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("^A")\n ).run(conn)\n\n*Example* Get all users whose name ends with "n".\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("n$")\n ).run(conn)\n\n*Example* Get all users whose name has "li" in it\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("li")\n ).run(conn)\n\n*Example* Get all users whose name is "John" with a case-insensitive search.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("(?i)^john$")\n ).run(conn)\n\n*Example* Get all users whose name is composed of only characters between "a" and "z".\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("(?i)^[a-z]+$")\n ).run(conn)\n\n*Example* Get all users where the zipcode is a string of 5 digits.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'zipcode\'].match("\\d{5}")\n ).run(conn)\n\n*Example* Retrieve the domain of a basic email\n\n r.expr("name@domain.com").match(".*@(.*)").run(conn)\n\nResult:\n\n {\n "start": 0,\n "end": 20,\n "str": "name@domain.com",\n "groups":[\n {\n "end": 17,\n "start": 7,\n "str": "domain.com"\n }\n ]\n }\n\nYou can then retrieve only the domain with the [\\[\\]](http://rethinkdb.com/api/python/get_field) selector.\n\n r.expr("name@domain.com").match(".*@(.*)")["groups"][0]["str"].run(conn)\n\nReturns `\'domain.com\'`\n\n*Example* Fail to parse out the domain and returns `None`.\n\n r.expr("name[at]domain.com").match(".*@(.*)").run(conn)\n'), + (rethinkdb.ast.RqlQuery.split, b'string.split([separator, [max_splits]]) -> array\n\nSplits a string into substrings. Splits on whitespace when called\nwith no arguments. When called with a separator, splits on that\nseparator. When called with a separator and a maximum number of\nsplits, splits on that separator at most `max_splits` times. (Can be\ncalled with `None` as the separator if you want to split on whitespace\nwhile still specifying `max_splits`.)\n\nMimics the behavior of Python\'s `string.split` in edge cases, except\nfor splitting on the empty string, which instead produces an array of\nsingle-character strings.\n\n*Example* Split on whitespace.\n\n > r.expr("foo bar bax").split().run(conn)\n ["foo", "bar", "bax"]\n\n*Example* Split the entries in a CSV file.\n\n > r.expr("12,37,,22,").split(",").run(conn)\n ["12", "37", "", "22", ""]\n\n*Example* Split a string into characters.\n\n > r.expr("mlucy").split("").run(conn)\n ["m", "l", "u", "c", "y"]\n\n*Example* Split the entries in a CSV file, but only at most 3\ntimes.\n\n > r.expr("12,37,,22,").split(",", 3).run(conn)\n ["12", "37", "", "22,"]\n\n*Example* Split on whitespace at most once (i.e. get the first word).\n\n > r.expr("foo bar bax").split(None, 1).run(conn)\n ["foo", "bar bax"]\n'), + (rethinkdb.ast.RqlQuery.upcase, b'string.upcase() -> string\n\nUppercases a string.\n\n*Example*\n\n > r.expr("Sentence about LaTeX.").upcase().run(conn)\n "SENTENCE ABOUT LATEX."\n\n__Note:__ `upcase` and `downcase` only affect ASCII characters.\n'), + (rethinkdb.ast.RqlQuery.concat_map, b'stream.concat_map(mapping_function) -> stream\narray.concat_map(mapping_function) -> array\n\nConcatenate one or more elements into a single sequence using a mapping function.\n\n`concat_map` works in a similar fashion to `map`, applying the given function to each element in a sequence, but it will always return a single sequence. If the mapping function returns a sequence, `map` would produce a sequence of sequences:\n\n r.expr([1, 2, 3]).map(lambda x: [x, x.mul(2)]).run(conn)\n\nResult:\n\n [[1, 2], [2, 4], [3, 6]]\n\nWhereas `concat_map` with the same mapping function would merge those sequences into one:\n\n r.expr([1, 2, 3]).concat_map(lambda x: [x, x.mul(2)]).run(conn)\n\nResult:\n\n [1, 2, 2, 4, 3, 6]\n\nThe return value, array or stream, will be the same type as the input.\n\n*Example* Construct a sequence of all monsters defeated by Marvel heroes. The field "defeatedMonsters" is an array of one or more monster names.\n\n r.table(\'marvel\').concat_map(lambda hero: hero[\'defeatedMonsters\']).run(conn)\n\n*Example* Simulate an [eq_join](http://rethinkdb.com/api/python/eq_join/) using `concat_map`. (This is how ReQL joins are implemented internally.)\n\n r.table(\'posts\').concat_map(\n lambda post: r.table(\'comments\').get_all(\n post[\'id\'], index=\'post_id\'\n ).map(\n lambda comment: { \'left\': post, \'right\': comment}\n )\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.is_empty, b"sequence.is_empty() -> bool\n\nTest if a sequence is empty.\n\n*Example* Are there any documents in the marvel table?\n\n r.table('marvel').is_empty().run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.limit, b"sequence.limit(n) -> stream\narray.limit(n) -> array\n\nEnd the sequence after the given number of elements.\n\n*Example* Only so many can fit in our Pantheon of heroes.\n\n r.table('marvel').order_by('belovedness').limit(10).run(conn)\n"), + (rethinkdb.ast.RqlQuery.map, b"sequence1.map([sequence2, ...], mapping_function) -> stream\narray1.map([sequence2, ...], mapping_function) -> array\nr.map(sequence1[, sequence2, ...], mapping_function) -> stream\nr.map(array1[, array2, ...], mapping_function) -> array\n\nTransform each element of one or more sequences by applying a mapping function to them. If `map` is run with two or more sequences, it will iterate for as many items as there are in the shortest sequence.\n\nNote that `map` can only be applied to sequences, not single values. If you wish to apply a function to a single value/selection (including an array), use the [do](http://rethinkdb.com/api/python/do) command.\n\n*Example* Return the first five squares.\n\n > r.expr([1, 2, 3, 4, 5]).map(lambda val: (val * val)).run(conn)\n \n [1, 4, 9, 16, 25]\n\n*Example* Sum the elements of three sequences.\n\n > sequence1 = [100, 200, 300, 400]\n > sequence2 = [10, 20, 30, 40]\n > sequence3 = [1, 2, 3, 4]\n > r.map(sequence1, sequence2, sequence3,\n lambda val1, val2, val3: (val1 + val2 + val3)).run(conn)\n \n [111, 222, 333, 444]\n\n*Example* Rename a field when retrieving documents using `map` and `merge`.\n\nThis example renames the field `id` to `user_id` when retrieving documents from the table `users`.\n\n r.table('users').map(\n lambda doc: doc.merge({'user_id': doc['id']}).without('id')).run(conn)\n\nNote that in this case, [row](http://rethinkdb.com/api/python/row) may be used as an alternative to writing an anonymous function, as it returns the same value as the function parameter receives:\n\n r.table('users').map(\n r.row.merge({'user_id': r.row['id']}).without('id')).run(conn)\n\n*Example* Assign every superhero an archenemy.\n\n r.table('heroes').map(r.table('villains'),\n lambda hero, villain: hero.merge({'villain': villain})).run(conn)\n"), + (rethinkdb.ast.RqlQuery.nth, b"sequence.nth(index) -> object\nselection.nth(index) -> selection<object>\n\nGet the *nth* element of a sequence, counting from zero. If the argument is negative, count from the last element.\n\nIn Python, you can use `[]` with an integer as a shorthand for `nth`.\n\n*Example* Select the second element in the array.\n\n r.expr([1,2,3]).nth(1).run(conn)\n r.expr([1,2,3])[1].run(conn)\n\n*Example* Select the bronze medalist from the competitors.\n\n r.table('players').order_by(index=r.desc('score')).nth(3).run(conn)\n\n*Example* Select the last place competitor.\n\n r.table('players').order_by(index=r.desc('score')).nth(-1).run(conn)\n"), + (rethinkdb.ast.RqlQuery.offsets_of, b"sequence.offsets_of(datum | predicate) -> array\n\nGet the indexes of an element in a sequence. If the argument is a predicate, get the indexes of all elements matching it.\n\n*Example* Find the position of the letter 'c'.\n\n r.expr(['a','b','c']).offsets_of('c').run(conn)\n\n*Example* Find the popularity ranking of invisible heroes.\n\n r.table('marvel').union(r.table('dc')).order_by('popularity').offsets_of(\n r.row['superpowers'].contains('invisibility')\n ).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.order_by, b'table.order_by([key1...], index=index_name) -> selection\nselection.order_by(key1, [key2...]) -> selection\nsequence.order_by(key1, [key2...]) -> array\n\nSort the sequence by document values of the given key(s). To specify\nthe ordering, wrap the attribute with either `r.asc` or `r.desc`\n(defaults to ascending).\n\n__Note:__ RethinkDB uses byte-wise ordering for `orderBy` and does not support Unicode collations; non-ASCII characters will be sorted by UTF-8 codepoint. For more information on RethinkDB\'s sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nSorting without an index requires the server to hold the sequence in\nmemory, and is limited to 100,000 documents (or the setting of the `arrayLimit` option for [run](http://rethinkdb.com/api/python/run)). Sorting with an index can\nbe done on arbitrarily large tables, or after a `between` command\nusing the same index.\n\n*Example* Order all the posts using the index `date`. \n\n r.table(\'posts\').order_by(index=\'date\').run(conn)\n\nThe index must either be the primary key or have been previously created with [index_create](http://rethinkdb.com/api/python/index_create/).\n\n r.table(\'posts\').index_create(\'date\').run(conn)\n\nYou can also select a descending ordering:\n\n r.table(\'posts\').order_by(index=r.desc(\'date\')).run(conn, callback)\n\n*Example* Order a sequence without an index.\n\n r.table(\'posts\').get(1)[\'comments\'].order_by(\'date\')\n\nYou can also select a descending ordering:\n\n r.table(\'posts\').get(1)[\'comments\'].order_by(r.desc(\'date\'))\n\nIf you\'re doing ad-hoc analysis and know your table won\'t have more then 100,000\nelements (or you\'ve changed the setting of the `arrayLimit` option for [run](http://rethinkdb.com/api/python/run)) you can run `order_by` without an index:\n\n r.table(\'small_table\').order_by(\'date\')\n\n*Example* You can efficiently order using multiple fields by using a\n[compound index](http://www.rethinkdb.com/docs/secondary-indexes/python/).\n\nOrder by date and title.\n\n r.table(\'posts\').order_by(index=\'date_and_title\').run(conn)\n\nThe index must have been previously created with [index_create](http://rethinkdb.com/api/python/index_create/).\n\n r.table(\'posts\').index_create(\'date_and_title\', lambda post:\n [post["date"], post["title"]]).run(conn)\n\n_Note_: You cannot specify multiple orders in a compound index. See [issue #2306](https://github.com/rethinkdb/rethinkdb/issues/2306)\nto track progress.\n\n*Example* If you have a sequence with fewer documents than the `array_limit`, you can order it\nby multiple fields without an index.\n\n r.table(\'small_table\').order_by(\'date\', r.desc(\'title\'))\n\n*Example* Notice that an index ordering always has highest\nprecedence. The following query orders posts by date, and if multiple\nposts were published on the same date, they will be ordered by title.\n\n r.table(\'post\').order_by(\'title\', index=\'date\').run(conn)\n*Example* You can use [nested field](http://rethinkdb.com/docs/cookbook/python/#filtering-based-on-nested-fields) syntax to sort on fields from subdocuments. (You can also create indexes on nested fields using this syntax with `index_create`.)\n\n r.table(\'user\').order_by(lambda user: user[\'group\'][\'id\']).run(conn)\n\n*Example* You can efficiently order data on arbitrary expressions using indexes.\n\n r.table(\'posts\').order_by(index=\'votes\').run(conn)\n\nThe index must have been previously created with [index_create](http://rethinkdb.com/api/ruby/index_create/).\n\n r.table(\'posts\').index_create(\'votes\', lambda post:\n post["upvotes"]-post["downvotes"]\n ).run(conn)\n\n*Example* If you have a sequence with fewer documents than the `array_limit`, you can order it with an arbitrary function directly.\n\n r.table(\'small_table\').order_by(lambda doc:\n doc[\'upvotes\']-doc[\'downvotes\']\n );\n\nYou can also select a descending ordering:\n\n r.table(\'small_table\').order_by(r.desc(lambda doc:\n doc[\'upvotes\']-doc[\'downvotes\']\n ));\n\n*Example* Ordering after a `between` command can be done as long as the same index is being used.\n\n r.table("posts").between(r.time(2013, 1, 1, \'+00:00\'), r.time(2013, 1, 1, \'+00:00\'), index=\'date\')\n .order_by(index=\'date\').run(conn);\n\n'), + (rethinkdb.ast.RqlQuery.sample, b"sequence.sample(number) -> selection\nstream.sample(number) -> array\narray.sample(number) -> array\n\nSelect a given number of elements from a sequence with uniform random distribution. Selection is done without replacement.\n\nIf the sequence has less than the requested number of elements (i.e., calling `sample(10)` on a sequence with only five elements), `sample` will return the entire sequence in a random order.\n\n*Example* Select 3 random heroes.\n\n r.table('marvel').sample(3).run(conn)\n"), + (rethinkdb.ast.RqlQuery.skip, b"sequence.skip(n) -> stream\narray.skip(n) -> array\n\nSkip a number of elements from the head of the sequence.\n\n*Example* Here in conjunction with `order_by` we choose to ignore the most successful heroes.\n\n r.table('marvel').order_by('successMetric').skip(10).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.slice, b"selection.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> selection\nstream.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> stream\narray.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> array\nbinary.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> binary\n\nReturn the elements of a sequence within the specified range.\n\n`slice` returns the range between `start_index` and `end_index`. If only `start_index` is specified, `slice` returns the range from that index to the end of the sequence. Specify `left_bound` or `right_bound` as `open` or `closed` to indicate whether to include that endpoint of the range by default: `closed` returns that endpoint, while `open` does not. By default, `left_bound` is closed and `right_bound` is open, so the range `(10,13)` will return the tenth, eleventh and twelfth elements in the sequence.\n\nIf `end_index` is past the end of the sequence, all elements from `start_index` to the end of the sequence will be returned. If `start_index` is past the end of the sequence or `end_index` is less than `start_index`, a zero-element sequence will be returned (although see below for negative `end_index` values). An error will be raised on a negative `start_index`.\n\nA negative `end_index` is allowed with arrays; in that case, the returned range counts backward from the array's end. That is, the range of `(2,-1)` returns the second element through the next-to-last element of the range. A negative `end_index` is not allowed with a stream. (An `end_index` of −1 *is* allowed with a stream if `right_bound` is closed; this behaves as if no `end_index` was specified.)\n\nIf `slice` is used with a [binary](http://rethinkdb.com/api/python/binary) object, the indexes refer to byte positions within the object. That is, the range `(10,20)` will refer to the 10th byte through the 19th byte.\n\nIf you are only specifying the indexes and not the bounding options, you may use Python's slice operator as a shorthand: `[start_index:end_index]`.\n\n**Example:** Return the fourth, fifth and sixth youngest players. (The youngest player is at index 0, so those are elements 3–5.)\n\n r.table('players').order_by(index='age').slice(3,6).run(conn)\n\nOr, using Python's slice operator:\n\n r.table('players').filter({'class': 'amateur'})[10:20].run(conn)\n\n**Example:** Return all but the top three players who have a red flag.\n\n r.table('players').filter({'flag': 'red'}).order_by(index=r.desc('score')).slice(3).run(conn)\n\n**Example:** Return holders of tickets `X` through `Y`, assuming tickets are numbered sequentially. We want to include ticket `Y`.\n\n r.table('users').order_by(index='ticket').slice(x, y, right_bound='closed').run(conn)\n\n**Example:** Return the elements of an array from the second through two from the end (that is, not including the last two).\n\n r.expr([0,1,2,3,4,5]).slice(2,-2).run(conn)\n\nResult:\n\n [2,3]\n"), + (rethinkdb.ast.RqlQuery.union, b"stream.union(sequence[, sequence, ...]) -> stream\narray.union(sequence[, sequence, ...]) -> array\n\nConcatenate two or more sequences.\n\n*Example* Construct a stream of all heroes.\n\n r.table('marvel').union(r.table('dc')).run(conn)\n\n*Example* Combine four arrays into one.\n\n r.expr([1, 2]).union([3, 4], [5, 6], [7, 8, 9]).run(conn)\n \n [1, 2, 3, 4, 5, 6, 7, 8, 9]\n"), + (rethinkdb.ast.RqlQuery.with_fields, b"sequence.with_fields([selector1, selector2...]) -> stream\narray.with_fields([selector1, selector2...]) -> array\n\nPlucks one or more attributes from a sequence of objects, filtering out any objects in the sequence that do not have the specified fields. Functionally, this is identical to `has_fields` followed by `pluck` on a sequence.\n\n*Example* Get a list of users and their posts, excluding any users who have not made any posts.\n\nExisting table structure:\n\n [\n { 'id': 1, 'user': 'bob', 'email': 'bob@foo.com', 'posts': [ 1, 4, 5 ] },\n { 'id': 2, 'user': 'george', 'email': 'george@foo.com' },\n { 'id': 3, 'user': 'jane', 'email': 'jane@foo.com', 'posts': [ 2, 3, 6 ] }\n ]\n\nCommand and output:\n\n r.table('users').with_fields('id', 'user', 'posts').run(conn)\n \n [\n { 'id': 1, 'user': 'bob', 'posts': [ 1, 4, 5 ] },\n { 'id': 3, 'user': 'jane', 'posts': [ 2, 3, 6 ] }\n ]\n\n*Example* Use the [nested field syntax](http://rethinkdb.com/docs/nested-fields/) to get a list of users with cell phone numbers in their contacts.\n\n r.table('users').with_fields('id', 'user', {contact: {'phone': 'work'}).run(conn)\n"), + (rethinkdb.ast.Table.delete, b'table.delete([durability="hard", return_changes=False])\n -> object\nselection.delete([durability="hard", return_changes=False])\n -> object\nsingleSelection.delete([durability="hard", return_changes=False])\n -> object\n\nDelete one or more documents from a table.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the\ntable or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). \nIn soft durability mode RethinkDB will acknowledge the write immediately after\nreceiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n\nDelete returns an object that contains the following attributes:\n\n- `deleted`: the number of documents that were deleted.\n- `skipped`: the number of documents that were skipped. \nFor example, if you attempt to delete a batch of documents, and another concurrent query\ndeletes some of those documents first, they will be counted as skipped.\n- `errors`: the number of errors encountered while performing the delete.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `inserted`, `replaced`, and `unchanged`: all 0 for a delete operation.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `delete` operation. Each object will have two keys: `{"new_val": None, "old_val": }`.\n\n*Example* Delete a single document from the table `comments`.\n\n r.table("comments").get("7eab9e63-73f1-4f33-8ce4-95cbea626f59").delete().run(conn)\n\n*Example* Delete all documents from the table `comments`.\n\n r.table("comments").delete().run(conn)\n\n*Example* Delete all comments where the field `id_post` is `3`.\n\n r.table("comments").filter({"id_post": 3}).delete().run(conn)\n\n*Example* Delete a single document from the table `comments` and return its value.\n\n r.table("comments").get("7eab9e63-73f1-4f33-8ce4-95cbea626f59").delete(return_changes=True).run(conn)\n\nThe result will look like:\n\n {\n "deleted": 1,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": None,\n "old_val": {\n "id": "7eab9e63-73f1-4f33-8ce4-95cbea626f59",\n "author": "William",\n "comment": "Great post",\n "id_post": 3\n }\n }\n ],\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\n*Example* Delete all documents from the table `comments` without waiting for the\noperation to be flushed to disk.\n\n r.table("comments").delete(durability="soft"}).run(conn)\n'), + (rethinkdb.ast.Table.insert, b'table.insert(object | [object1, object2, ...][, durability="hard", return_changes=False, conflict="error"])\n -> object\n\nInsert documents into a table. Accepts a single document or an array of\ndocuments.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the table or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). In soft durability mode RethinkDB will acknowledge the write immediately after receiving and caching it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `conflict`: Determine handling of inserting documents with the same primary key as existing entries. Possible values are `"error"`, `"replace"` or `"update"`.\n - `"error"`: Do not insert the new document and record the conflict as an error. This is the default.\n - `"replace"`: [Replace](http://rethinkdb.com/api/python/replace/) the old document in its entirety with the new one.\n - `"update"`: [Update](http://rethinkdb.com/api/python/update/) fields of the old document with fields from the new one.\n\nInsert returns an object that contains the following attributes:\n\n- `inserted`: the number of documents successfully inserted.\n- `replaced`: the number of documents updated when `conflict` is set to `"replace"` or `"update"`.\n- `unchanged`: the number of documents whose fields are identical to existing documents with the same primary key when `conflict` is set to `"replace"` or `"update"`.\n- `errors`: the number of errors encountered while performing the insert.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `deleted` and `skipped`: 0 for an insert operation.\n- `generated_keys`: a list of generated primary keys for inserted documents whose primary keys were not specified (capped to 100,000).\n- `warnings`: if the field `generated_keys` is truncated, you will get the warning _"Too many generated keys (<X>), array truncated to 100000."_.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `insert` operation. Each object will have two keys: `{"new_val": , "old_val": None}`.\n\n*Example* Insert a document into the table `posts`.\n\n r.table("posts").insert({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }).run(conn)\n\nThe result will be:\n\n {\n "deleted": 0,\n "errors": 0,\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\n*Example* Insert a document without a defined primary key into the table `posts` where the\nprimary key is `id`.\n\n r.table("posts").insert({\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }).run(conn)\n\nRethinkDB will generate a primary key and return it in `generated_keys`.\n\n {\n "deleted": 0,\n "errors": 0,\n "generated_keys": [\n "dd782b64-70a7-43e4-b65e-dd14ae61d947"\n ],\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\nRetrieve the document you just inserted with:\n\n r.table("posts").get("dd782b64-70a7-43e4-b65e-dd14ae61d947").run(conn)\n\nAnd you will get back:\n\n {\n "id": "dd782b64-70a7-43e4-b65e-dd14ae61d947",\n "title": "Lorem ipsum",\n "content": "Dolor sit amet",\n }\n\n*Example* Insert multiple documents into the table `users`.\n\n r.table("users").insert([\n {"id": "william", "email": "william@rethinkdb.com"},\n {"id": "lara", "email": "lara@rethinkdb.com"}\n ]).run(conn)\n\n*Example* Insert a document into the table `users`, replacing the document if the document\nalready exists. \n\n r.table("users").insert(\n {"id": "william", "email": "william@rethinkdb.com"},\n conflict="error"\n ).run(conn)\n\n*Example* Copy the documents from `posts` to `posts_backup`.\n\n r.table("posts_backup").insert( r.table("posts") ).run(conn)\n\n*Example* Get back a copy of the inserted document (with its generated primary key).\n\n r.table("posts").insert(\n {"title": "Lorem ipsum", "content": "Dolor sit amet"},\n return_changes=True\n ).run(conn)\n\nThe result will be\n\n {\n "deleted": 0,\n "errors": 0,\n "generated_keys": [\n "dd782b64-70a7-43e4-b65e-dd14ae61d947"\n ],\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0,\n "changes": [\n {\n "old_val": None,\n "new_val": {\n "id": "dd782b64-70a7-43e4-b65e-dd14ae61d947",\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }\n }\n ]\n }\n'), + (rethinkdb.ast.Table.replace, b'table.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nselection.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nsingleSelection.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\n\nReplace documents in a table. Accepts a JSON document or a ReQL expression, and replaces\nthe original document with the new one. The new document must have the same primary key\nas the original document.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the\ntable or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). \nIn soft durability mode RethinkDB will acknowledge the write immediately after\nreceiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `non_atomic`: if set to `True`, executes the replacement and distributes the result to replicas in a non-atomic fashion. This flag is required to perform non-deterministic updates, such as those that require reading data from another table.\n\nReplace returns an object that contains the following attributes:\n\n- `replaced`: the number of documents that were replaced\n- `unchanged`: the number of documents that would have been modified, except that the\nnew value was the same as the old value\n- `inserted`: the number of new documents added. You can have new documents inserted if\nyou do a point-replace on a key that isn\'t in the table or you do a replace on a\nselection and one of the documents you are replacing has been deleted\n- `deleted`: the number of deleted documents when doing a replace with `None`\n- `errors`: the number of errors encountered while performing the replace.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `skipped`: 0 for a replace operation\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `replace` operation. Each object will have two keys: `{"new_val": , "old_val": }`.\n\n*Example* Replace the document with the primary key `1`.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "draft"\n }).run(conn)\n\n*Example* Remove the field `status` from all posts.\n\n r.table("posts").replace(lambda post:\n post.without("status")\n ).run(conn)\n\n*Example* Remove all the fields that are not `id`, `title` or `content`.\n\n r.table("posts").replace(lambda post:\n post.pluck("id", "title", "content")\n ).run(conn)\n\n*Example* Replace the document with the primary key `1` using soft durability.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "draft"\n }, durability="soft").run(conn)\n\n*Example* Replace the document with the primary key `1` and return the values of the document before\nand after the replace operation.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "published"\n }, return_changes=True).run(conn)\n\nThe result will have a `changes` field:\n\n {\n "deleted": 0,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": {\n "id":1,\n "title": "Lorem ipsum"\n "content": "Aleas jacta est",\n "status": "published",\n },\n "old_val": {\n "id":1,\n "title": "Lorem ipsum"\n "content": "TODO",\n "status": "draft",\n "author": "William",\n }\n }\n ], \n "replaced": 1,\n "skipped": 0,\n "unchanged": 0\n }\n'), + (rethinkdb.ast.Table.sync, b'table.sync() -> object\n\n`sync` ensures that writes on a given table are written to permanent storage. Queries\nthat specify soft durability (`durability=\'soft\'`) do not give such guarantees, so\n`sync` can be used to ensure the state of these queries. A call to `sync` does not return\nuntil all previous writes to the table are persisted.\n\nIf successful, the operation returns an object: `{"synced": 1}`.\n\n*Example* After having updated multiple heroes with soft durability, we now want to wait\nuntil these changes are persisted.\n\n r.table(\'marvel\').sync().run(conn)\n\n'), + (rethinkdb.ast.Table.update, b'table.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nselection.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nsingleSelection.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\n\nUpdate JSON documents in a table. Accepts a JSON document, a ReQL expression, or a combination of the two.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the table or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). In soft durability mode RethinkDB will acknowledge the write immediately after receiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `non_atomic`: if set to `True`, executes the update and distributes the result to replicas in a non-atomic fashion. This flag is required to perform non-deterministic updates, such as those that require reading data from another table.\n\nUpdate returns an object that contains the following attributes:\n\n- `replaced`: the number of documents that were updated.\n- `unchanged`: the number of documents that would have been modified except the new value was the same as the old value.\n- `skipped`: the number of documents that were skipped because the document didn\'t exist.\n- `errors`: the number of errors encountered while performing the update.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `deleted` and `inserted`: 0 for an update operation.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `update` operation. Each object will have two keys: `{"new_val": , "old_val": }`.\n\n*Example* Update the status of the post with `id` of `1` to `published`.\n\n r.table("posts").get(1).update({"status": "published"}).run(conn)\n\n*Example* Update the status of all posts to `published`.\n\n r.table("posts").update({"status": "published"}).run(conn)\n\n*Example* Update the status of all the posts written by William.\n\n r.table("posts").filter({"author": "William"}).update({"status": "published"}).run(conn)\n\n*Example* Increment the field `view` with `id` of `1`.\nThis query will throw an error if the field `views` doesn\'t exist.\n\n r.table("posts").get(1).update({\n "views": r.row["views"]+1\n }).run(conn)\n\n*Example* Increment the field `view` of the post with `id` of `1`.\nIf the field `views` does not exist, it will be set to `0`.\n\n r.table("posts").update({\n "views": (r.row["views"]+1).default(0)\n }).run(conn)\n\n*Example* Perform a conditional update. \nIf the post has more than 100 views, set the `type` of a post to `hot`, else set it to `normal`.\n\n r.table("posts").get(1).update(lambda post:\n r.branch(\n post["views"] > 100,\n {"type": "hot"},\n {"type": "normal"}\n )\n ).run(conn)\n\n*Example* Update the field `num_comments` with the result of a sub-query. Because this update is not atomic, you must pass the `non_atomic` flag.\n\n r.table("posts").get(1).update({\n "num_comments": r.table("comments").filter({"id_post": 1}).count()\n }, non_atomic=True).run(conn)\n\nIf you forget to specify the `non_atomic` flag, you will get a `RqlRuntimeError`:\n\nRqlRuntimeError: Could not prove function deterministic. Maybe you want to use the non_atomic flag? \n\n*Example* Update the field `num_comments` with a random value between 0 and 100. This update cannot be proven deterministic because of `r.js` (and in fact is not), so you must pass the `non_atomic` flag.\n\n r.table("posts").get(1).update({\n "num_comments": r.js("Math.floor(Math.random()*100)")\n }, non_atomic=True).run(conn)\n\n*Example* Update the status of the post with `id` of `1` using soft durability.\n\n r.table("posts").get(1).update({status: "published"}, durability="soft").run(conn)\n\n*Example* Increment the field `views` and return the values of the document before and after the update operation.\n\n r.table("posts").get(1).update({\n "views": r.row["views"]+1\n }, return_changes=True).run(conn)\n\nThe result will now include a `changes` field:\n\n {\n "deleted": 1,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": {\n "id": 1,\n "author": "Julius_Caesar",\n "title": "Commentarii de Bello Gallico",\n "content": "Aleas jacta est",\n "views": 207\n },\n "old_val": {\n "id": 1,\n "author": "Julius_Caesar",\n "title": "Commentarii de Bello Gallico",\n "content": "Aleas jacta est",\n "views": 206\n }\n }\n ],\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\nThe `update` command supports RethinkDB\'s [nested field][nf] syntax to update subdocuments. Consider a user table with contact information in this format:\n\n[nf]: /docs/nested-fields/python\n\n {\n "id": 10001,\n "name": "Bob Smith",\n "contact": {\n "phone": {\n "work": "408-555-1212",\n "home": "408-555-1213",\n "cell": "408-555-1214"\n },\n "email": {\n "work": "bob@smith.com",\n "home": "bobsmith@example.com",\n "other": "bobbys@moosecall.net"\n },\n "im": {\n "skype": "Bob Smith",\n "aim": "bobmoose",\n "icq": "nobodyremembersicqnumbers"\n }\n },\n "notes": [\n {\n "date": r.time(2014,1,1,\'Z\'),\n "from": "John Doe",\n "subject": "My name is even more boring than Bob\'s"\n },\n {\n "date": r.time(2014,2,2,\'Z\'),\n "from": "Bob Smith Sr",\n "subject": "Happy Second of February"\n }\n ]\n }\n\n*Example* Update Bob Smith\'s cell phone number.\n\n r.table("users").get(10001).update(\n {"contact": {"phone": {"cell": "408-555-4242"}}}\n ).run(conn)\n\n*Example* Add another note to Bob Smith\'s record.\n\n new_note = {\n "date": r.now(),\n "from": "Inigo Montoya",\n "subject": "You killed my father"\n }\n r.table("users").get(10001).update(\n {"notes": r.row["notes"].append(new_note)}\n ).run(conn)\n\n*Example* Send a note to every user with an ICQ number.\n\n icq_note = {\n "date": r.now(),\n "from": "Admin",\n "subject": "Welcome to the future"\n }\n r.table("users").filter(\n r.row.has_fields({"contact": {"im": "icq"}})\n ).update(\n {"notes": r.row["notes"].append(icq_note)}\n ).run(conn)\n\n*Example* Replace all of Bob\'s IM records. Normally, `update` will merge nested documents together; to replace the entire `"im"` document, use the [literal][] command.\n\n[literal]: /api/python/literal/\n\n r.table(\'users\').get(10001).update(\n {"contact": {"im": r.literal({"aim": "themoosemeister"})}}\n ).run(conn)\n'), +] + +for function, text in docsSource: + try: + text = str(text.decode('utf-8')) + except UnicodeEncodeError: + pass + if hasattr(function, "__func__"): + function.__func__.__doc__ = text + else: + function.__doc__ = text diff --git a/ext/librethinkdbxx/reql/ql2.proto b/ext/librethinkdbxx/reql/ql2.proto new file mode 100644 index 00000000..e40c5be5 --- /dev/null +++ b/ext/librethinkdbxx/reql/ql2.proto @@ -0,0 +1,843 @@ +//////////////////////////////////////////////////////////////////////////////// +// THE HIGH-LEVEL VIEW // +//////////////////////////////////////////////////////////////////////////////// + +// Process: When you first open a connection, send the magic number +// for the version of the protobuf you're targeting (in the [Version] +// enum). This should **NOT** be sent as a protobuf; just send the +// little-endian 32-bit integer over the wire raw. This number should +// only be sent once per connection. + +// The magic number shall be followed by an authorization key. The +// first 4 bytes are the length of the key to be sent as a little-endian +// 32-bit integer, followed by the key string. Even if there is no key, +// an empty string should be sent (length 0 and no data). + +// Following the authorization key, the client shall send a magic number +// for the communication protocol they want to use (in the [Protocol] +// enum). This shall be a little-endian 32-bit integer. + +// The server will then respond with a NULL-terminated string response. +// "SUCCESS" indicates that the connection has been accepted. Any other +// response indicates an error, and the response string should describe +// the error. + +// Next, for each query you want to send, construct a [Query] protobuf +// and serialize it to a binary blob. Send the blob's size to the +// server encoded as a little-endian 32-bit integer, followed by the +// blob itself. You will recieve a [Response] protobuf back preceded +// by its own size, once again encoded as a little-endian 32-bit +// integer. You can see an example exchange below in **EXAMPLE**. + +// A query consists of a [Term] to evaluate and a unique-per-connection +// [token]. + +// Tokens are used for two things: +// * Keeping track of which responses correspond to which queries. +// * Batched queries. Some queries return lots of results, so we send back +// batches of <1000, and you need to send a [CONTINUE] query with the same +// token to get more results from the original query. +//////////////////////////////////////////////////////////////////////////////// + +message VersionDummy { // We need to wrap it like this for some + // non-conforming protobuf libraries + // This enum contains the magic numbers for your version. See **THE HIGH-LEVEL + // VIEW** for what to do with it. + enum Version { + V0_1 = 0x3f61ba36; + V0_2 = 0x723081e1; // Authorization key during handshake + V0_3 = 0x5f75e83e; // Authorization key and protocol during handshake + V0_4 = 0x400c2d20; // Queries execute in parallel + V1_0 = 0x34c2bdc3; // Users and permissions + } + + // The protocol to use after the handshake, specified in V0_3 + enum Protocol { + PROTOBUF = 0x271ffc41; + JSON = 0x7e6970c7; + } +} + +// You send one of: +// * A [START] query with a [Term] to evaluate and a unique-per-connection token. +// * A [CONTINUE] query with the same token as a [START] query that returned +// [SUCCESS_PARTIAL] in its [Response]. +// * A [STOP] query with the same token as a [START] query that you want to stop. +// * A [NOREPLY_WAIT] query with a unique per-connection token. The server answers +// with a [WAIT_COMPLETE] [Response]. +// * A [SERVER_INFO] query. The server answers with a [SERVER_INFO] [Response]. +message Query { + enum QueryType { + START = 1; // Start a new query. + CONTINUE = 2; // Continue a query that returned [SUCCESS_PARTIAL] + // (see [Response]). + STOP = 3; // Stop a query partway through executing. + NOREPLY_WAIT = 4; // Wait for noreply operations to finish. + SERVER_INFO = 5; // Get server information. + } + optional QueryType type = 1; + // A [Term] is how we represent the operations we want a query to perform. + optional Term query = 2; // only present when [type] = [START] + optional int64 token = 3; + // This flag is ignored on the server. `noreply` should be added + // to `global_optargs` instead (the key "noreply" should map to + // either true or false). + optional bool OBSOLETE_noreply = 4 [default = false]; + + // If this is set to [true], then [Datum] values will sometimes be + // of [DatumType] [R_JSON] (see below). This can provide enormous + // speedups in languages with poor protobuf libraries. + optional bool accepts_r_json = 5 [default = false]; + + message AssocPair { + optional string key = 1; + optional Term val = 2; + } + repeated AssocPair global_optargs = 6; +} + +// A backtrace frame (see `backtrace` in Response below) +message Frame { + enum FrameType { + POS = 1; // Error occurred in a positional argument. + OPT = 2; // Error occurred in an optional argument. + } + optional FrameType type = 1; + optional int64 pos = 2; // The index of the positional argument. + optional string opt = 3; // The name of the optional argument. +} +message Backtrace { + repeated Frame frames = 1; +} + +// You get back a response with the same [token] as your query. +message Response { + enum ResponseType { + // These response types indicate success. + SUCCESS_ATOM = 1; // Query returned a single RQL datatype. + SUCCESS_SEQUENCE = 2; // Query returned a sequence of RQL datatypes. + SUCCESS_PARTIAL = 3; // Query returned a partial sequence of RQL + // datatypes. If you send a [CONTINUE] query with + // the same token as this response, you will get + // more of the sequence. Keep sending [CONTINUE] + // queries until you get back [SUCCESS_SEQUENCE]. + WAIT_COMPLETE = 4; // A [NOREPLY_WAIT] query completed. + SERVER_INFO = 5; // The data for a [SERVER_INFO] request. This is + // the same as `SUCCESS_ATOM` except that there will + // never be profiling data. + + // These response types indicate failure. + CLIENT_ERROR = 16; // Means the client is buggy. An example is if the + // client sends a malformed protobuf, or tries to + // send [CONTINUE] for an unknown token. + COMPILE_ERROR = 17; // Means the query failed during parsing or type + // checking. For example, if you pass too many + // arguments to a function. + RUNTIME_ERROR = 18; // Means the query failed at runtime. An example is + // if you add together two values from a table, but + // they turn out at runtime to be booleans rather + // than numbers. + } + optional ResponseType type = 1; + + // If `ResponseType` is `RUNTIME_ERROR`, this may be filled in with more + // information about the error. + enum ErrorType { + INTERNAL = 1000000; + RESOURCE_LIMIT = 2000000; + QUERY_LOGIC = 3000000; + NON_EXISTENCE = 3100000; + OP_FAILED = 4100000; + OP_INDETERMINATE = 4200000; + USER = 5000000; + PERMISSION_ERROR = 6000000; + } + optional ErrorType error_type = 7; + + // ResponseNotes are used to provide information about the query + // response that may be useful for people writing drivers or ORMs. + // Currently all the notes we send indicate that a stream has certain + // special properties. + enum ResponseNote { + // The stream is a changefeed stream (e.g. `r.table('test').changes()`). + SEQUENCE_FEED = 1; + // The stream is a point changefeed stream + // (e.g. `r.table('test').get(0).changes()`). + ATOM_FEED = 2; + // The stream is an order_by_limit changefeed stream + // (e.g. `r.table('test').order_by(index: 'id').limit(5).changes()`). + ORDER_BY_LIMIT_FEED = 3; + // The stream is a union of multiple changefeed types that can't be + // collapsed to a single type + // (e.g. `r.table('test').changes().union(r.table('test').get(0).changes())`). + UNIONED_FEED = 4; + // The stream is a changefeed stream and includes notes on what state + // the changefeed stream is in (e.g. objects of the form `{state: + // 'initializing'}`). + INCLUDES_STATES = 5; + } + repeated ResponseNote notes = 6; + + optional int64 token = 2; // Indicates what [Query] this response corresponds to. + + // [response] contains 1 RQL datum if [type] is [SUCCESS_ATOM] or + // [SERVER_INFO]. [response] contains many RQL data if [type] is + // [SUCCESS_SEQUENCE] or [SUCCESS_PARTIAL]. [response] contains 1 + // error message (of type [R_STR]) in all other cases. + repeated Datum response = 3; + + // If [type] is [CLIENT_ERROR], [TYPE_ERROR], or [RUNTIME_ERROR], then a + // backtrace will be provided. The backtrace says where in the query the + // error occurred. Ideally this information will be presented to the user as + // a pretty-printed version of their query with the erroneous section + // underlined. A backtrace is a series of 0 or more [Frame]s, each of which + // specifies either the index of a positional argument or the name of an + // optional argument. (Those words will make more sense if you look at the + // [Term] message below.) + optional Backtrace backtrace = 4; // Contains n [Frame]s when you get back an error. + + // If the [global_optargs] in the [Query] that this [Response] is a + // response to contains a key "profile" which maps to a static value of + // true then [profile] will contain a [Datum] which provides profiling + // information about the execution of the query. This field should be + // returned to the user along with the result that would normally be + // returned (a datum or a cursor). In official drivers this is accomplished + // by putting them inside of an object with "value" mapping to the return + // value and "profile" mapping to the profile object. + optional Datum profile = 5; +} + +// A [Datum] is a chunk of data that can be serialized to disk or returned to +// the user in a Response. Currently we only support JSON types, but we may +// support other types in the future (e.g., a date type or an integer type). +message Datum { + enum DatumType { + R_NULL = 1; + R_BOOL = 2; + R_NUM = 3; // a double + R_STR = 4; + R_ARRAY = 5; + R_OBJECT = 6; + // This [DatumType] will only be used if [accepts_r_json] is + // set to [true] in [Query]. [r_str] will be filled with a + // JSON encoding of the [Datum]. + R_JSON = 7; // uses r_str + } + optional DatumType type = 1; + optional bool r_bool = 2; + optional double r_num = 3; + optional string r_str = 4; + + repeated Datum r_array = 5; + message AssocPair { + optional string key = 1; + optional Datum val = 2; + } + repeated AssocPair r_object = 6; +} + +// A [Term] is either a piece of data (see **Datum** above), or an operator and +// its operands. If you have a [Datum], it's stored in the member [datum]. If +// you have an operator, its positional arguments are stored in [args] and its +// optional arguments are stored in [optargs]. +// +// A note about type signatures: +// We use the following notation to denote types: +// arg1_type, arg2_type, argrest_type... -> result_type +// So, for example, if we have a function `avg` that takes any number of +// arguments and averages them, we might write: +// NUMBER... -> NUMBER +// Or if we had a function that took one number modulo another: +// NUMBER, NUMBER -> NUMBER +// Or a function that takes a table and a primary key of any Datum type, then +// retrieves the entry with that primary key: +// Table, DATUM -> OBJECT +// Some arguments must be provided as literal values (and not the results of sub +// terms). These are marked with a `!`. +// Optional arguments are specified within curly braces as argname `:` value +// type (e.x `{noreply:BOOL}`) +// Many RQL operations are polymorphic. For these, alterantive type signatures +// are separated by `|`. +// +// The RQL type hierarchy is as follows: +// Top +// DATUM +// NULL +// BOOL +// NUMBER +// STRING +// OBJECT +// SingleSelection +// ARRAY +// Sequence +// ARRAY +// Stream +// StreamSelection +// Table +// Database +// Function +// Ordering - used only by ORDER_BY +// Pathspec -- an object, string, or array that specifies a path +// Error +message Term { + enum TermType { + // A RQL datum, stored in `datum` below. + DATUM = 1; + + MAKE_ARRAY = 2; // DATUM... -> ARRAY + // Evaluate the terms in [optargs] and make an object + MAKE_OBJ = 3; // {...} -> OBJECT + + // * Compound types + + // Takes an integer representing a variable and returns the value stored + // in that variable. It's the responsibility of the client to translate + // from their local representation of a variable to a unique _non-negative_ + // integer for that variable. (We do it this way instead of letting + // clients provide variable names as strings to discourage + // variable-capturing client libraries, and because it's more efficient + // on the wire.) + VAR = 10; // !NUMBER -> DATUM + // Takes some javascript code and executes it. + JAVASCRIPT = 11; // STRING {timeout: !NUMBER} -> DATUM | + // STRING {timeout: !NUMBER} -> Function(*) + UUID = 169; // () -> DATUM + + // Takes an HTTP URL and gets it. If the get succeeds and + // returns valid JSON, it is converted into a DATUM + HTTP = 153; // STRING {data: OBJECT | STRING, + // timeout: !NUMBER, + // method: STRING, + // params: OBJECT, + // header: OBJECT | ARRAY, + // attempts: NUMBER, + // redirects: NUMBER, + // verify: BOOL, + // page: FUNC | STRING, + // page_limit: NUMBER, + // auth: OBJECT, + // result_format: STRING, + // } -> STRING | STREAM + + // Takes a string and throws an error with that message. + // Inside of a `default` block, you can omit the first + // argument to rethrow whatever error you catch (this is most + // useful as an argument to the `default` filter optarg). + ERROR = 12; // STRING -> Error | -> Error + // Takes nothing and returns a reference to the implicit variable. + IMPLICIT_VAR = 13; // -> DATUM + + // * Data Operators + // Returns a reference to a database. + DB = 14; // STRING -> Database + // Returns a reference to a table. + TABLE = 15; // Database, STRING, {read_mode:STRING, identifier_format:STRING} -> Table + // STRING, {read_mode:STRING, identifier_format:STRING} -> Table + // Gets a single element from a table by its primary or a secondary key. + GET = 16; // Table, STRING -> SingleSelection | Table, NUMBER -> SingleSelection | + // Table, STRING -> NULL | Table, NUMBER -> NULL | + GET_ALL = 78; // Table, DATUM..., {index:!STRING} => ARRAY + + // Simple DATUM Ops + EQ = 17; // DATUM... -> BOOL + NE = 18; // DATUM... -> BOOL + LT = 19; // DATUM... -> BOOL + LE = 20; // DATUM... -> BOOL + GT = 21; // DATUM... -> BOOL + GE = 22; // DATUM... -> BOOL + NOT = 23; // BOOL -> BOOL + // ADD can either add two numbers or concatenate two arrays. + ADD = 24; // NUMBER... -> NUMBER | STRING... -> STRING + SUB = 25; // NUMBER... -> NUMBER + MUL = 26; // NUMBER... -> NUMBER + DIV = 27; // NUMBER... -> NUMBER + MOD = 28; // NUMBER, NUMBER -> NUMBER + + FLOOR = 183; // NUMBER -> NUMBER + CEIL = 184; // NUMBER -> NUMBER + ROUND = 185; // NUMBER -> NUMBER + + // DATUM Array Ops + // Append a single element to the end of an array (like `snoc`). + APPEND = 29; // ARRAY, DATUM -> ARRAY + // Prepend a single element to the end of an array (like `cons`). + PREPEND = 80; // ARRAY, DATUM -> ARRAY + //Remove the elements of one array from another array. + DIFFERENCE = 95; // ARRAY, ARRAY -> ARRAY + + // DATUM Set Ops + // Set ops work on arrays. They don't use actual sets and thus have + // performance characteristics you would expect from arrays rather than + // from sets. All set operations have the post condition that they + // array they return contains no duplicate values. + SET_INSERT = 88; // ARRAY, DATUM -> ARRAY + SET_INTERSECTION = 89; // ARRAY, ARRAY -> ARRAY + SET_UNION = 90; // ARRAY, ARRAY -> ARRAY + SET_DIFFERENCE = 91; // ARRAY, ARRAY -> ARRAY + + SLICE = 30; // Sequence, NUMBER, NUMBER -> Sequence + SKIP = 70; // Sequence, NUMBER -> Sequence + LIMIT = 71; // Sequence, NUMBER -> Sequence + OFFSETS_OF = 87; // Sequence, DATUM -> Sequence | Sequence, Function(1) -> Sequence + CONTAINS = 93; // Sequence, (DATUM | Function(1))... -> BOOL + + // Stream/Object Ops + // Get a particular field from an object, or map that over a + // sequence. + GET_FIELD = 31; // OBJECT, STRING -> DATUM + // | Sequence, STRING -> Sequence + // Return an array containing the keys of the object. + KEYS = 94; // OBJECT -> ARRAY + // Return an array containing the values of the object. + VALUES = 186; // OBJECT -> ARRAY + // Creates an object + OBJECT = 143; // STRING, DATUM, ... -> OBJECT + // Check whether an object contains all the specified fields, + // or filters a sequence so that all objects inside of it + // contain all the specified fields. + HAS_FIELDS = 32; // OBJECT, Pathspec... -> BOOL + // x.with_fields(...) <=> x.has_fields(...).pluck(...) + WITH_FIELDS = 96; // Sequence, Pathspec... -> Sequence + // Get a subset of an object by selecting some attributes to preserve, + // or map that over a sequence. (Both pick and pluck, polymorphic.) + PLUCK = 33; // Sequence, Pathspec... -> Sequence | OBJECT, Pathspec... -> OBJECT + // Get a subset of an object by selecting some attributes to discard, or + // map that over a sequence. (Both unpick and without, polymorphic.) + WITHOUT = 34; // Sequence, Pathspec... -> Sequence | OBJECT, Pathspec... -> OBJECT + // Merge objects (right-preferential) + MERGE = 35; // OBJECT... -> OBJECT | Sequence -> Sequence + + // Sequence Ops + // Get all elements of a sequence between two values. + // Half-open by default, but the openness of either side can be + // changed by passing 'closed' or 'open for `right_bound` or + // `left_bound`. + BETWEEN_DEPRECATED = 36; // Deprecated version of between, which allows `null` to specify unboundedness + // With the newer version, clients should use `r.minval` and `r.maxval` for unboundedness + BETWEEN = 182; // StreamSelection, DATUM, DATUM, {index:!STRING, right_bound:STRING, left_bound:STRING} -> StreamSelection + REDUCE = 37; // Sequence, Function(2) -> DATUM + MAP = 38; // Sequence, Function(1) -> Sequence + // The arity of the function should be + // Sequence..., Function(sizeof...(Sequence)) -> Sequence + + FOLD = 187; // Sequence, Datum, Function(2), {Function(3), Function(1) + + // Filter a sequence with either a function or a shortcut + // object (see API docs for details). The body of FILTER is + // wrapped in an implicit `.default(false)`, and you can + // change the default value by specifying the `default` + // optarg. If you make the default `r.error`, all errors + // caught by `default` will be rethrown as if the `default` + // did not exist. + FILTER = 39; // Sequence, Function(1), {default:DATUM} -> Sequence | + // Sequence, OBJECT, {default:DATUM} -> Sequence + // Map a function over a sequence and then concatenate the results together. + CONCAT_MAP = 40; // Sequence, Function(1) -> Sequence + // Order a sequence based on one or more attributes. + ORDER_BY = 41; // Sequence, (!STRING | Ordering)..., {index: (!STRING | Ordering)} -> Sequence + // Get all distinct elements of a sequence (like `uniq`). + DISTINCT = 42; // Sequence -> Sequence + // Count the number of elements in a sequence, or only the elements that match + // a given filter. + COUNT = 43; // Sequence -> NUMBER | Sequence, DATUM -> NUMBER | Sequence, Function(1) -> NUMBER + IS_EMPTY = 86; // Sequence -> BOOL + // Take the union of multiple sequences (preserves duplicate elements! (use distinct)). + UNION = 44; // Sequence... -> Sequence + // Get the Nth element of a sequence. + NTH = 45; // Sequence, NUMBER -> DATUM + // do NTH or GET_FIELD depending on target object + BRACKET = 170; // Sequence | OBJECT, NUMBER | STRING -> DATUM + // OBSOLETE_GROUPED_MAPREDUCE = 46; + // OBSOLETE_GROUPBY = 47; + + INNER_JOIN = 48; // Sequence, Sequence, Function(2) -> Sequence + OUTER_JOIN = 49; // Sequence, Sequence, Function(2) -> Sequence + // An inner-join that does an equality comparison on two attributes. + EQ_JOIN = 50; // Sequence, !STRING, Sequence, {index:!STRING} -> Sequence + ZIP = 72; // Sequence -> Sequence + RANGE = 173; // -> Sequence [0, +inf) + // NUMBER -> Sequence [0, a) + // NUMBER, NUMBER -> Sequence [a, b) + + // Array Ops + // Insert an element in to an array at a given index. + INSERT_AT = 82; // ARRAY, NUMBER, DATUM -> ARRAY + // Remove an element at a given index from an array. + DELETE_AT = 83; // ARRAY, NUMBER -> ARRAY | + // ARRAY, NUMBER, NUMBER -> ARRAY + // Change the element at a given index of an array. + CHANGE_AT = 84; // ARRAY, NUMBER, DATUM -> ARRAY + // Splice one array in to another array. + SPLICE_AT = 85; // ARRAY, NUMBER, ARRAY -> ARRAY + + // * Type Ops + // Coerces a datum to a named type (e.g. "bool"). + // If you previously used `stream_to_array`, you should use this instead + // with the type "array". + COERCE_TO = 51; // Top, STRING -> Top + // Returns the named type of a datum (e.g. TYPE_OF(true) = "BOOL") + TYPE_OF = 52; // Top -> STRING + + // * Write Ops (the OBJECTs contain data about number of errors etc.) + // Updates all the rows in a selection. Calls its Function with the row + // to be updated, and then merges the result of that call. + UPDATE = 53; // StreamSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | + // SingleSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | + // StreamSelection, OBJECT, {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | + // SingleSelection, OBJECT, {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT + // Deletes all the rows in a selection. + DELETE = 54; // StreamSelection, {durability:STRING, return_changes:BOOL} -> OBJECT | SingleSelection -> OBJECT + // Replaces all the rows in a selection. Calls its Function with the row + // to be replaced, and then discards it and stores the result of that + // call. + REPLACE = 55; // StreamSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | SingleSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT + // Inserts into a table. If `conflict` is replace, overwrites + // entries with the same primary key. If `conflict` is + // update, does an update on the entry. If `conflict` is + // error, or is omitted, conflicts will trigger an error. + INSERT = 56; // Table, OBJECT, {conflict:STRING, durability:STRING, return_changes:BOOL} -> OBJECT | Table, Sequence, {conflict:STRING, durability:STRING, return_changes:BOOL} -> OBJECT + + // * Administrative OPs + // Creates a database with a particular name. + DB_CREATE = 57; // STRING -> OBJECT + // Drops a database with a particular name. + DB_DROP = 58; // STRING -> OBJECT + // Lists all the databases by name. (Takes no arguments) + DB_LIST = 59; // -> ARRAY + // Creates a table with a particular name in a particular + // database. (You may omit the first argument to use the + // default database.) + TABLE_CREATE = 60; // Database, STRING, {primary_key:STRING, shards:NUMBER, replicas:NUMBER, primary_replica_tag:STRING} -> OBJECT + // Database, STRING, {primary_key:STRING, shards:NUMBER, replicas:OBJECT, primary_replica_tag:STRING} -> OBJECT + // STRING, {primary_key:STRING, shards:NUMBER, replicas:NUMBER, primary_replica_tag:STRING} -> OBJECT + // STRING, {primary_key:STRING, shards:NUMBER, replicas:OBJECT, primary_replica_tag:STRING} -> OBJECT + // Drops a table with a particular name from a particular + // database. (You may omit the first argument to use the + // default database.) + TABLE_DROP = 61; // Database, STRING -> OBJECT + // STRING -> OBJECT + // Lists all the tables in a particular database. (You may + // omit the first argument to use the default database.) + TABLE_LIST = 62; // Database -> ARRAY + // -> ARRAY + // Returns the row in the `rethinkdb.table_config` or `rethinkdb.db_config` table + // that corresponds to the given database or table. + CONFIG = 174; // Database -> SingleSelection + // Table -> SingleSelection + // Returns the row in the `rethinkdb.table_status` table that corresponds to the + // given table. + STATUS = 175; // Table -> SingleSelection + // Called on a table, waits for that table to be ready for read/write operations. + // Called on a database, waits for all of the tables in the database to be ready. + // Returns the corresponding row or rows from the `rethinkdb.table_status` table. + WAIT = 177; // Table -> OBJECT + // Database -> OBJECT + // Generates a new config for the given table, or all tables in the given database + // The `shards` and `replicas` arguments are required. If `emergency_repair` is + // specified, it will enter a completely different mode of repairing a table + // which has lost half or more of its replicas. + RECONFIGURE = 176; // Database|Table, {shards:NUMBER, replicas:NUMBER [, + // dry_run:BOOLEAN] + // } -> OBJECT + // Database|Table, {shards:NUMBER, replicas:OBJECT [, + // primary_replica_tag:STRING, + // nonvoting_replica_tags:ARRAY, + // dry_run:BOOLEAN] + // } -> OBJECT + // Table, {emergency_repair:STRING, dry_run:BOOLEAN} -> OBJECT + // Balances the table's shards but leaves everything else the same. Can also be + // applied to an entire database at once. + REBALANCE = 179; // Table -> OBJECT + // Database -> OBJECT + + // Ensures that previously issued soft-durability writes are complete and + // written to disk. + SYNC = 138; // Table -> OBJECT + + // Set global, database, or table-specific permissions + GRANT = 188; // -> OBJECT + // Database -> OBJECT + // Table -> OBJECT + + // * Secondary indexes OPs + // Creates a new secondary index with a particular name and definition. + INDEX_CREATE = 75; // Table, STRING, Function(1), {multi:BOOL} -> OBJECT + // Drops a secondary index with a particular name from the specified table. + INDEX_DROP = 76; // Table, STRING -> OBJECT + // Lists all secondary indexes on a particular table. + INDEX_LIST = 77; // Table -> ARRAY + // Gets information about whether or not a set of indexes are ready to + // be accessed. Returns a list of objects that look like this: + // {index:STRING, ready:BOOL[, progress:NUMBER]} + INDEX_STATUS = 139; // Table, STRING... -> ARRAY + // Blocks until a set of indexes are ready to be accessed. Returns the + // same values INDEX_STATUS. + INDEX_WAIT = 140; // Table, STRING... -> ARRAY + // Renames the given index to a new name + INDEX_RENAME = 156; // Table, STRING, STRING, {overwrite:BOOL} -> OBJECT + + // * Control Operators + // Calls a function on data + FUNCALL = 64; // Function(*), DATUM... -> DATUM + // Executes its first argument, and returns its second argument if it + // got [true] or its third argument if it got [false] (like an `if` + // statement). + BRANCH = 65; // BOOL, Top, Top -> Top + // Returns true if any of its arguments returns true (short-circuits). + OR = 66; // BOOL... -> BOOL + // Returns true if all of its arguments return true (short-circuits). + AND = 67; // BOOL... -> BOOL + // Calls its Function with each entry in the sequence + // and executes the array of terms that Function returns. + FOR_EACH = 68; // Sequence, Function(1) -> OBJECT + +//////////////////////////////////////////////////////////////////////////////// +////////// Special Terms +//////////////////////////////////////////////////////////////////////////////// + + // An anonymous function. Takes an array of numbers representing + // variables (see [VAR] above), and a [Term] to execute with those in + // scope. Returns a function that may be passed an array of arguments, + // then executes the Term with those bound to the variable names. The + // user will never construct this directly. We use it internally for + // things like `map` which take a function. The "arity" of a [Function] is + // the number of arguments it takes. + // For example, here's what `_X_.map{|x| x+2}` turns into: + // Term { + // type = MAP; + // args = [_X_, + // Term { + // type = Function; + // args = [Term { + // type = DATUM; + // datum = Datum { + // type = R_ARRAY; + // r_array = [Datum { type = R_NUM; r_num = 1; }]; + // }; + // }, + // Term { + // type = ADD; + // args = [Term { + // type = VAR; + // args = [Term { + // type = DATUM; + // datum = Datum { type = R_NUM; + // r_num = 1}; + // }]; + // }, + // Term { + // type = DATUM; + // datum = Datum { type = R_NUM; r_num = 2; }; + // }]; + // }]; + // }]; + FUNC = 69; // ARRAY, Top -> ARRAY -> Top + + // Indicates to ORDER_BY that this attribute is to be sorted in ascending order. + ASC = 73; // !STRING -> Ordering + // Indicates to ORDER_BY that this attribute is to be sorted in descending order. + DESC = 74; // !STRING -> Ordering + + // Gets info about anything. INFO is most commonly called on tables. + INFO = 79; // Top -> OBJECT + + // `a.match(b)` returns a match object if the string `a` + // matches the regular expression `b`. + MATCH = 97; // STRING, STRING -> DATUM + + // Change the case of a string. + UPCASE = 141; // STRING -> STRING + DOWNCASE = 142; // STRING -> STRING + + // Select a number of elements from sequence with uniform distribution. + SAMPLE = 81; // Sequence, NUMBER -> Sequence + + // Evaluates its first argument. If that argument returns + // NULL or throws an error related to the absence of an + // expected value (for instance, accessing a non-existent + // field or adding NULL to an integer), DEFAULT will either + // return its second argument or execute it if it's a + // function. If the second argument is a function, it will be + // passed either the text of the error or NULL as its + // argument. + DEFAULT = 92; // Top, Top -> Top + + // Parses its first argument as a json string and returns it as a + // datum. + JSON = 98; // STRING -> DATUM + // Returns the datum as a JSON string. + // N.B.: we would really prefer this be named TO_JSON and that exists as + // an alias in Python and JavaScript drivers; however it conflicts with the + // standard `to_json` method defined by Ruby's standard json library. + TO_JSON_STRING = 172; // DATUM -> STRING + + // Parses its first arguments as an ISO 8601 time and returns it as a + // datum. + ISO8601 = 99; // STRING -> PSEUDOTYPE(TIME) + // Prints a time as an ISO 8601 time. + TO_ISO8601 = 100; // PSEUDOTYPE(TIME) -> STRING + + // Returns a time given seconds since epoch in UTC. + EPOCH_TIME = 101; // NUMBER -> PSEUDOTYPE(TIME) + // Returns seconds since epoch in UTC given a time. + TO_EPOCH_TIME = 102; // PSEUDOTYPE(TIME) -> NUMBER + + // The time the query was received by the server. + NOW = 103; // -> PSEUDOTYPE(TIME) + // Puts a time into an ISO 8601 timezone. + IN_TIMEZONE = 104; // PSEUDOTYPE(TIME), STRING -> PSEUDOTYPE(TIME) + // a.during(b, c) returns whether a is in the range [b, c) + DURING = 105; // PSEUDOTYPE(TIME), PSEUDOTYPE(TIME), PSEUDOTYPE(TIME) -> BOOL + // Retrieves the date portion of a time. + DATE = 106; // PSEUDOTYPE(TIME) -> PSEUDOTYPE(TIME) + // x.time_of_day == x.date - x + TIME_OF_DAY = 126; // PSEUDOTYPE(TIME) -> NUMBER + // Returns the timezone of a time. + TIMEZONE = 127; // PSEUDOTYPE(TIME) -> STRING + + // These access the various components of a time. + YEAR = 128; // PSEUDOTYPE(TIME) -> NUMBER + MONTH = 129; // PSEUDOTYPE(TIME) -> NUMBER + DAY = 130; // PSEUDOTYPE(TIME) -> NUMBER + DAY_OF_WEEK = 131; // PSEUDOTYPE(TIME) -> NUMBER + DAY_OF_YEAR = 132; // PSEUDOTYPE(TIME) -> NUMBER + HOURS = 133; // PSEUDOTYPE(TIME) -> NUMBER + MINUTES = 134; // PSEUDOTYPE(TIME) -> NUMBER + SECONDS = 135; // PSEUDOTYPE(TIME) -> NUMBER + + // Construct a time from a date and optional timezone or a + // date+time and optional timezone. + TIME = 136; // NUMBER, NUMBER, NUMBER, STRING -> PSEUDOTYPE(TIME) | + // NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, STRING -> PSEUDOTYPE(TIME) | + + // Constants for ISO 8601 days of the week. + MONDAY = 107; // -> 1 + TUESDAY = 108; // -> 2 + WEDNESDAY = 109; // -> 3 + THURSDAY = 110; // -> 4 + FRIDAY = 111; // -> 5 + SATURDAY = 112; // -> 6 + SUNDAY = 113; // -> 7 + + // Constants for ISO 8601 months. + JANUARY = 114; // -> 1 + FEBRUARY = 115; // -> 2 + MARCH = 116; // -> 3 + APRIL = 117; // -> 4 + MAY = 118; // -> 5 + JUNE = 119; // -> 6 + JULY = 120; // -> 7 + AUGUST = 121; // -> 8 + SEPTEMBER = 122; // -> 9 + OCTOBER = 123; // -> 10 + NOVEMBER = 124; // -> 11 + DECEMBER = 125; // -> 12 + + // Indicates to MERGE to replace, or remove in case of an empty literal, the + // other object rather than merge it. + LITERAL = 137; // -> Merging + // JSON -> Merging + + // SEQUENCE, STRING -> GROUPED_SEQUENCE | SEQUENCE, FUNCTION -> GROUPED_SEQUENCE + GROUP = 144; + SUM = 145; + AVG = 146; + MIN = 147; + MAX = 148; + + // `str.split()` splits on whitespace + // `str.split(" ")` splits on spaces only + // `str.split(" ", 5)` splits on spaces with at most 5 results + // `str.split(nil, 5)` splits on whitespace with at most 5 results + SPLIT = 149; // STRING -> ARRAY | STRING, STRING -> ARRAY | STRING, STRING, NUMBER -> ARRAY | STRING, NULL, NUMBER -> ARRAY + + UNGROUP = 150; // GROUPED_DATA -> ARRAY + + // Takes a range of numbers and returns a random number within the range + RANDOM = 151; // NUMBER, NUMBER {float:BOOL} -> DATUM + + CHANGES = 152; // TABLE -> STREAM + ARGS = 154; // ARRAY -> SPECIAL (used to splice arguments) + + // BINARY is client-only at the moment, it is not supported on the server + BINARY = 155; // STRING -> PSEUDOTYPE(BINARY) + + GEOJSON = 157; // OBJECT -> PSEUDOTYPE(GEOMETRY) + TO_GEOJSON = 158; // PSEUDOTYPE(GEOMETRY) -> OBJECT + POINT = 159; // NUMBER, NUMBER -> PSEUDOTYPE(GEOMETRY) + LINE = 160; // (ARRAY | PSEUDOTYPE(GEOMETRY))... -> PSEUDOTYPE(GEOMETRY) + POLYGON = 161; // (ARRAY | PSEUDOTYPE(GEOMETRY))... -> PSEUDOTYPE(GEOMETRY) + DISTANCE = 162; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) {geo_system:STRING, unit:STRING} -> NUMBER + INTERSECTS = 163; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> BOOL + INCLUDES = 164; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> BOOL + CIRCLE = 165; // PSEUDOTYPE(GEOMETRY), NUMBER {num_vertices:NUMBER, geo_system:STRING, unit:STRING, fill:BOOL} -> PSEUDOTYPE(GEOMETRY) + GET_INTERSECTING = 166; // TABLE, PSEUDOTYPE(GEOMETRY) {index:!STRING} -> StreamSelection + FILL = 167; // PSEUDOTYPE(GEOMETRY) -> PSEUDOTYPE(GEOMETRY) + GET_NEAREST = 168; // TABLE, PSEUDOTYPE(GEOMETRY) {index:!STRING, max_results:NUM, max_dist:NUM, geo_system:STRING, unit:STRING} -> ARRAY + POLYGON_SUB = 171; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> PSEUDOTYPE(GEOMETRY) + + // Constants for specifying key ranges + MINVAL = 180; + MAXVAL = 181; + } + optional TermType type = 1; + + // This is only used when type is DATUM. + optional Datum datum = 2; + + repeated Term args = 3; // Holds the positional arguments of the query. + message AssocPair { + optional string key = 1; + optional Term val = 2; + } + repeated AssocPair optargs = 4; // Holds the optional arguments of the query. + // (Note that the order of the optional arguments doesn't matter; think of a + // Hash.) +} + +//////////////////////////////////////////////////////////////////////////////// +// EXAMPLE // +//////////////////////////////////////////////////////////////////////////////// +// ```ruby +// r.table('tbl', {:read_mode => 'outdated'}).insert([{:id => 0}, {:id => 1}]) +// ``` +// Would turn into: +// Term { +// type = INSERT; +// args = [Term { +// type = TABLE; +// args = [Term { +// type = DATUM; +// datum = Datum { type = R_STR; r_str = "tbl"; }; +// }]; +// optargs = [["read_mode", +// Term { +// type = DATUM; +// datum = Datum { type = R_STR; r_bool = "outdated"; }; +// }]]; +// }, +// Term { +// type = MAKE_ARRAY; +// args = [Term { +// type = DATUM; +// datum = Datum { type = R_OBJECT; r_object = [["id", 0]]; }; +// }, +// Term { +// type = DATUM; +// datum = Datum { type = R_OBJECT; r_object = [["id", 1]]; }; +// }]; +// }] +// } +// And the server would reply: +// Response { +// type = SUCCESS_ATOM; +// token = 1; +// response = [Datum { type = R_OBJECT; r_object = [["inserted", 2]]; }]; +// } +// Or, if there were an error: +// Response { +// type = RUNTIME_ERROR; +// token = 1; +// response = [Datum { type = R_STR; r_str = "The table `tbl` doesn't exist!"; }]; +// backtrace = [Frame { type = POS; pos = 0; }, Frame { type = POS; pos = 0; }]; +// } diff --git a/ext/librethinkdbxx/src/connection.cc b/ext/librethinkdbxx/src/connection.cc new file mode 100644 index 00000000..62f6efee --- /dev/null +++ b/ext/librethinkdbxx/src/connection.cc @@ -0,0 +1,431 @@ +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include "connection.h" +#include "connection_p.h" +#include "json_p.h" +#include "exceptions.h" +#include "term.h" +#include "cursor_p.h" + +#include "rapidjson-config.h" +#include "rapidjson/rapidjson.h" +#include "rapidjson/encodedstream.h" +#include "rapidjson/document.h" + +namespace RethinkDB { + +using QueryType = Protocol::Query::QueryType; + +// constants +const int debug_net = 0; +const uint32_t version_magic = + static_cast(Protocol::VersionDummy::Version::V0_4); +const uint32_t json_magic = + static_cast(Protocol::VersionDummy::Protocol::JSON); + +std::unique_ptr connect(std::string host, int port, std::string auth_key) { + struct addrinfo hints; + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + char port_str[16]; + snprintf(port_str, 16, "%d", port); + struct addrinfo *servinfo; + int ret = getaddrinfo(host.c_str(), port_str, &hints, &servinfo); + if (ret) throw Error("getaddrinfo: %s\n", gai_strerror(ret)); + + struct addrinfo *p; + Error error; + int sockfd; + for (p = servinfo; p != NULL; p = p->ai_next) { + sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); + if (sockfd == -1) { + error = Error::from_errno("socket"); + continue; + } + + if (connect(sockfd, p->ai_addr, p->ai_addrlen) == -1) { + ::close(sockfd); + error = Error::from_errno("connect"); + continue; + } + + break; + } + + if (p == NULL) { + throw error; + } + + freeaddrinfo(servinfo); + + std::unique_ptr conn_private(new ConnectionPrivate(sockfd)); + WriteLock writer(conn_private.get()); + { + size_t size = auth_key.size(); + char buf[12 + size]; + memcpy(buf, &version_magic, 4); + uint32_t n = size; + memcpy(buf + 4, &n, 4); + memcpy(buf + 8, auth_key.data(), size); + memcpy(buf + 8 + size, &json_magic, 4); + writer.send(buf, sizeof buf); + } + + ReadLock reader(conn_private.get()); + { + const size_t max_response_length = 1024; + char buf[max_response_length + 1]; + size_t len = reader.recv_cstring(buf, max_response_length); + if (len == max_response_length || strcmp(buf, "SUCCESS")) { + buf[len] = 0; + ::close(sockfd); + throw Error("Server rejected connection with message: %s", buf); + } + } + + return std::unique_ptr(new Connection(conn_private.release())); +} + +Connection::Connection(ConnectionPrivate *dd) : d(dd) { } +Connection::~Connection() { + // close(); +} + +size_t ReadLock::recv_some(char* buf, size_t size, double wait) { + if (wait != FOREVER) { + while (true) { + fd_set readfds; + struct timeval tv; + + FD_ZERO(&readfds); + FD_SET(conn->guarded_sockfd, &readfds); + + tv.tv_sec = (int)wait; + tv.tv_usec = (int)((wait - (int)wait) / MICROSECOND); + int rv = select(conn->guarded_sockfd + 1, &readfds, NULL, NULL, &tv); + if (rv == -1) { + throw Error::from_errno("select"); + } else if (rv == 0) { + throw TimeoutException(); + } + + if (FD_ISSET(conn->guarded_sockfd, &readfds)) { + break; + } + } + } + + ssize_t numbytes = ::recv(conn->guarded_sockfd, buf, size, 0); + if (numbytes == -1) throw Error::from_errno("recv"); + if (debug_net > 1) { + fprintf(stderr, "<< %s\n", write_datum(std::string(buf, numbytes)).c_str()); + } + + return numbytes; +} + +void ReadLock::recv(char* buf, size_t size, double wait) { + while (size) { + size_t numbytes = recv_some(buf, size, wait); + + buf += numbytes; + size -= numbytes; + } +} + +size_t ReadLock::recv_cstring(char* buf, size_t max_size){ + size_t size = 0; + for (; size < max_size; size++) { + recv(buf, 1, FOREVER); + if (*buf == 0) { + break; + } + buf++; + } + return size; +} + +void WriteLock::send(const char* buf, size_t size) { + while (size) { + ssize_t numbytes = ::write(conn->guarded_sockfd, buf, size); + if (numbytes == -1) throw Error::from_errno("write"); + if (debug_net > 1) { + fprintf(stderr, ">> %s\n", write_datum(std::string(buf, numbytes)).c_str()); + } + + buf += numbytes; + size -= numbytes; + } +} + +void WriteLock::send(const std::string data) { + send(data.data(), data.size()); +} + +std::string ReadLock::recv(size_t size) { + char buf[size]; + recv(buf, size, FOREVER); + return buf; +} + +void Connection::close() { + CacheLock guard(d.get()); + for (auto& it : d->guarded_cache) { + stop_query(it.first); + } + + int ret = ::close(d->guarded_sockfd); + if (ret == -1) { + throw Error::from_errno("close"); + } +} + +Response ConnectionPrivate::wait_for_response(uint64_t token_want, double wait) { + CacheLock guard(this); + ConnectionPrivate::TokenCache& cache = guarded_cache[token_want]; + + while (true) { + if (!cache.responses.empty()) { + Response response(std::move(cache.responses.front())); + cache.responses.pop(); + if (cache.closed && cache.responses.empty()) { + guarded_cache.erase(token_want); + } + + return response; + } + + if (cache.closed) { + throw Error("Trying to read from a closed token"); + } + + if (guarded_loop_active) { + cache.cond.wait(guard.inner_lock); + } else { + break; + } + } + + ReadLock reader(this); + return reader.read_loop(token_want, std::move(guard), wait); +} + +Response ReadLock::read_loop(uint64_t token_want, CacheLock&& guard, double wait) { + if (!guard.inner_lock) { + guard.lock(); + } + if (conn->guarded_loop_active) { + throw Error("Cannot run more than one read loop on the same connection"); + } + conn->guarded_loop_active = true; + guard.unlock(); + + try { + while (true) { + char buf[12]; + bzero(buf, sizeof(buf)); + recv(buf, 12, wait); + uint64_t token_got; + memcpy(&token_got, buf, 8); + uint32_t length; + memcpy(&length, buf + 8, 4); + + std::unique_ptr bufmem(new char[length + 1]); + char *buffer = bufmem.get(); + bzero(buffer, length + 1); + recv(buffer, length, wait); + buffer[length] = '\0'; + + rapidjson::Document json; + json.ParseInsitu(buffer); + if (json.HasParseError()) { + fprintf(stderr, "json parse error, code: %d, position: %d\n", + (int)json.GetParseError(), (int)json.GetErrorOffset()); + } else if (json.IsNull()) { + fprintf(stderr, "null value, read: %s\n", buffer); + } + + Datum datum = read_datum(json); + if (debug_net > 0) { + fprintf(stderr, "[%" PRIu64 "] << %s\n", token_got, write_datum(datum).c_str()); + } + + Response response(std::move(datum)); + + if (token_got == token_want) { + guard.lock(); + if (response.type != Protocol::Response::ResponseType::SUCCESS_PARTIAL) { + auto it = conn->guarded_cache.find(token_got); + if (it != conn->guarded_cache.end()) { + it->second.closed = true; + it->second.cond.notify_all(); + } + conn->guarded_cache.erase(it); + } + conn->guarded_loop_active = false; + for (auto& it : conn->guarded_cache) { + it.second.cond.notify_all(); + } + return response; + } else { + guard.lock(); + auto it = conn->guarded_cache.find(token_got); + if (it == conn->guarded_cache.end()) { + // drop the response + } else if (!it->second.closed) { + it->second.responses.emplace(std::move(response)); + if (response.type != Protocol::Response::ResponseType::SUCCESS_PARTIAL) { + it->second.closed = true; + } + } + it->second.cond.notify_all(); + guard.unlock(); + } + } + } catch (const TimeoutException &e) { + if (!guard.inner_lock){ + guard.lock(); + } + conn->guarded_loop_active = false; + throw e; + } +} + +void ConnectionPrivate::run_query(Query query, bool no_reply) { + WriteLock writer(this); + writer.send(query.serialize()); +} + +Cursor Connection::start_query(Term *term, OptArgs&& opts) { + bool no_reply = false; + auto it = opts.find("noreply"); + if (it != opts.end()) { + no_reply = *(it->second.datum.get_boolean()); + } + + uint64_t token = d->new_token(); + { + CacheLock guard(d.get()); + d->guarded_cache[token]; + } + + d->run_query(Query{QueryType::START, token, term->datum, std::move(opts)}); + if (no_reply) { + return Cursor(new CursorPrivate(token, this, Nil())); + } + + Cursor cursor(new CursorPrivate(token, this)); + Response response = d->wait_for_response(token, FOREVER); + cursor.d->add_response(std::move(response)); + return cursor; +} + +void Connection::stop_query(uint64_t token) { + const auto& it = d->guarded_cache.find(token); + if (it != d->guarded_cache.end() && !it->second.closed) { + d->run_query(Query{QueryType::STOP, token}, true); + } +} + +void Connection::continue_query(uint64_t token) { + d->run_query(Query{QueryType::CONTINUE, token}, true); +} + +Error Response::as_error() { + std::string repr; + if (result.size() == 1) { + std::string* string = result[0].get_string(); + if (string) { + repr = *string; + } else { + repr = write_datum(result[0]); + } + } else { + repr = write_datum(Datum(result)); + } + std::string err; + using RT = Protocol::Response::ResponseType; + using ET = Protocol::Response::ErrorType; + switch (type) { + case RT::SUCCESS_SEQUENCE: err = "unexpected response: SUCCESS_SEQUENCE"; break; + case RT::SUCCESS_PARTIAL: err = "unexpected response: SUCCESS_PARTIAL"; break; + case RT::SUCCESS_ATOM: err = "unexpected response: SUCCESS_ATOM"; break; + case RT::WAIT_COMPLETE: err = "unexpected response: WAIT_COMPLETE"; break; + case RT::SERVER_INFO: err = "unexpected response: SERVER_INFO"; break; + case RT::CLIENT_ERROR: err = "ReqlDriverError"; break; + case RT::COMPILE_ERROR: err = "ReqlCompileError"; break; + case RT::RUNTIME_ERROR: + switch (error_type) { + case ET::INTERNAL: err = "ReqlInternalError"; break; + case ET::RESOURCE_LIMIT: err = "ReqlResourceLimitError"; break; + case ET::QUERY_LOGIC: err = "ReqlQueryLogicError"; break; + case ET::NON_EXISTENCE: err = "ReqlNonExistenceError"; break; + case ET::OP_FAILED: err = "ReqlOpFailedError"; break; + case ET::OP_INDETERMINATE: err = "ReqlOpIndeterminateError"; break; + case ET::USER: err = "ReqlUserError"; break; + case ET::PERMISSION_ERROR: err = "ReqlPermissionError"; break; + default: err = "ReqlRuntimeError"; break; + } + } + throw Error("%s: %s", err.c_str(), repr.c_str()); +} + +Protocol::Response::ResponseType response_type(double t) { + int n = static_cast(t); + using RT = Protocol::Response::ResponseType; + switch (n) { + case static_cast(RT::SUCCESS_ATOM): + return RT::SUCCESS_ATOM; + case static_cast(RT::SUCCESS_SEQUENCE): + return RT::SUCCESS_SEQUENCE; + case static_cast(RT::SUCCESS_PARTIAL): + return RT::SUCCESS_PARTIAL; + case static_cast(RT::WAIT_COMPLETE): + return RT::WAIT_COMPLETE; + case static_cast(RT::CLIENT_ERROR): + return RT::CLIENT_ERROR; + case static_cast(RT::COMPILE_ERROR): + return RT::COMPILE_ERROR; + case static_cast(RT::RUNTIME_ERROR): + return RT::RUNTIME_ERROR; + default: + throw Error("Unknown response type"); + } +} + +Protocol::Response::ErrorType runtime_error_type(double t) { + int n = static_cast(t); + using ET = Protocol::Response::ErrorType; + switch (n) { + case static_cast(ET::INTERNAL): + return ET::INTERNAL; + case static_cast(ET::RESOURCE_LIMIT): + return ET::RESOURCE_LIMIT; + case static_cast(ET::QUERY_LOGIC): + return ET::QUERY_LOGIC; + case static_cast(ET::NON_EXISTENCE): + return ET::NON_EXISTENCE; + case static_cast(ET::OP_FAILED): + return ET::OP_FAILED; + case static_cast(ET::OP_INDETERMINATE): + return ET::OP_INDETERMINATE; + case static_cast(ET::USER): + return ET::USER; + default: + throw Error("Unknown error type"); + } +} + +} diff --git a/ext/librethinkdbxx/src/connection.h b/ext/librethinkdbxx/src/connection.h new file mode 100644 index 00000000..d3882857 --- /dev/null +++ b/ext/librethinkdbxx/src/connection.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "protocol_defs.h" +#include "datum.h" +#include "error.h" + +#define FOREVER (-1) +#define SECOND 1 +#define MICROSECOND 0.000001 + +namespace RethinkDB { + +class Term; +using OptArgs = std::map; + +// A connection to a RethinkDB server +// It contains: +// * A socket +// * Read and write locks +// * A cache of responses that have not been read by the corresponding Cursor +class ConnectionPrivate; +class Connection { +public: + Connection() = delete; + Connection(const Connection&) noexcept = delete; + Connection(Connection&&) noexcept = delete; + Connection& operator=(Connection&&) noexcept = delete; + Connection& operator=(const Connection&) noexcept = delete; + ~Connection(); + + void close(); + +private: + explicit Connection(ConnectionPrivate *dd); + std::unique_ptr d; + + Cursor start_query(Term *term, OptArgs&& args); + void stop_query(uint64_t); + void continue_query(uint64_t); + + friend class Cursor; + friend class CursorPrivate; + friend class Token; + friend class Term; + friend std::unique_ptr + connect(std::string host, int port, std::string auth_key); + +}; + +// $doc(connect) +std::unique_ptr connect(std::string host = "localhost", int port = 28015, std::string auth_key = ""); + +} diff --git a/ext/librethinkdbxx/src/connection_p.h b/ext/librethinkdbxx/src/connection_p.h new file mode 100644 index 00000000..d8a95e3c --- /dev/null +++ b/ext/librethinkdbxx/src/connection_p.h @@ -0,0 +1,133 @@ +#ifndef CONNECTION_P_H +#define CONNECTION_P_H + +#include + +#include "connection.h" +#include "term.h" +#include "json_p.h" + +namespace RethinkDB { + +extern const int debug_net; + +struct Query { + Protocol::Query::QueryType type; + uint64_t token; + Datum term; + OptArgs optArgs; + + std::string serialize() { + Array query_arr{static_cast(type)}; + if (term.is_valid()) query_arr.emplace_back(term); + if (!optArgs.empty()) + query_arr.emplace_back(Term(std::move(optArgs)).datum); + + std::string query_str = write_datum(query_arr); + if (debug_net > 0) { + fprintf(stderr, "[%" PRIu64 "] >> %s\n", token, query_str.c_str()); + } + + char header[12]; + memcpy(header, &token, 8); + uint32_t size = query_str.size(); + memcpy(header + 8, &size, 4); + query_str.insert(0, header, 12); + return query_str; + } +}; + +// Used internally to convert a raw response type into an enum +Protocol::Response::ResponseType response_type(double t); +Protocol::Response::ErrorType runtime_error_type(double t); + +// Contains a response from the server. Use the Cursor class to interact with these responses +class Response { +public: + Response() = delete; + explicit Response(Datum&& datum) : + type(response_type(std::move(datum).extract_field("t").extract_number())), + error_type(datum.get_field("e") ? + runtime_error_type(std::move(datum).extract_field("e").extract_number()) : + Protocol::Response::ErrorType(0)), + result(std::move(datum).extract_field("r").extract_array()) { } + Error as_error(); + Protocol::Response::ResponseType type; + Protocol::Response::ErrorType error_type; + Array result; +}; + +class Token; +class ConnectionPrivate { +public: + ConnectionPrivate(int sockfd) + : guarded_next_token(1), guarded_sockfd(sockfd), guarded_loop_active(false) + { } + + void run_query(Query query, bool no_reply = false); + + Response wait_for_response(uint64_t, double); + uint64_t new_token() { + return guarded_next_token++; + } + + std::mutex read_lock; + std::mutex write_lock; + std::mutex cache_lock; + + struct TokenCache { + bool closed = false; + std::condition_variable cond; + std::queue responses; + }; + + std::map guarded_cache; + uint64_t guarded_next_token; + int guarded_sockfd; + bool guarded_loop_active; +}; + +class CacheLock { +public: + CacheLock(ConnectionPrivate* conn) : inner_lock(conn->cache_lock) { } + + void lock() { + inner_lock.lock(); + } + + void unlock() { + inner_lock.unlock(); + } + + std::unique_lock inner_lock; +}; + +class ReadLock { +public: + ReadLock(ConnectionPrivate* conn_) : lock(conn_->read_lock), conn(conn_) { } + + size_t recv_some(char*, size_t, double wait); + void recv(char*, size_t, double wait); + std::string recv(size_t); + size_t recv_cstring(char*, size_t); + + Response read_loop(uint64_t, CacheLock&&, double); + + std::lock_guard lock; + ConnectionPrivate* conn; +}; + +class WriteLock { +public: + WriteLock(ConnectionPrivate* conn_) : lock(conn_->write_lock), conn(conn_) { } + + void send(const char*, size_t); + void send(std::string); + + std::lock_guard lock; + ConnectionPrivate* conn; +}; + +} // namespace RethinkDB + +#endif // CONNECTION_P_H diff --git a/ext/librethinkdbxx/src/cursor.cc b/ext/librethinkdbxx/src/cursor.cc new file mode 100644 index 00000000..987c4dba --- /dev/null +++ b/ext/librethinkdbxx/src/cursor.cc @@ -0,0 +1,221 @@ +#include "cursor.h" +#include "cursor_p.h" +#include "exceptions.h" + +namespace RethinkDB { + +// for type completion, in order to forward declare with unique_ptr +Cursor::Cursor(Cursor&&) = default; +Cursor& Cursor::operator=(Cursor&&) = default; + +CursorPrivate::CursorPrivate(uint64_t token_, Connection *conn_) + : single(false), no_more(false), index(0), + token(token_), conn(conn_) +{ } + +CursorPrivate::CursorPrivate(uint64_t token_, Connection *conn_, Datum&& datum) + : single(true), no_more(true), index(0), buffer(Array{std::move(datum)}), + token(token_), conn(conn_) +{ } + +Cursor::Cursor(CursorPrivate *dd) : d(dd) {} + +Cursor::~Cursor() { + if (d && d->conn) { + close(); + } +} + +Datum& Cursor::next(double wait) const { + if (!has_next(wait)) { + throw Error("next: No more data"); + } + + return d->buffer[d->index++]; +} + +Datum& Cursor::peek(double wait) const { + if (!has_next(wait)) { + throw Error("next: No more data"); + } + + return d->buffer[d->index]; +} + +void Cursor::each(std::function f, double wait) const { + while (has_next(wait)) { + f(std::move(d->buffer[d->index++])); + } +} + +void CursorPrivate::convert_single() const { + if (index != 0) { + throw Error("Cursor: already consumed"); + } + + if (buffer.size() != 1) { + throw Error("Cursor: invalid response from server"); + } + + if (!buffer[0].is_array()) { + throw Error("Cursor: not an array"); + } + + buffer.swap(buffer[0].extract_array()); + single = false; +} + +void CursorPrivate::clear_and_read_all() const { + if (single) { + convert_single(); + } + if (index != 0) { + buffer.erase(buffer.begin(), buffer.begin() + index); + index = 0; + } + while (!no_more) { + add_response(conn->d->wait_for_response(token, FOREVER)); + } +} + +Array&& Cursor::to_array() && { + d->clear_and_read_all(); + return std::move(d->buffer); +} + +Array Cursor::to_array() const & { + d->clear_and_read_all(); + return d->buffer; +} + +Datum Cursor::to_datum() const & { + if (d->single) { + if (d->index != 0) { + throw Error("to_datum: already consumed"); + } + return d->buffer[0]; + } + + d->clear_and_read_all(); + return d->buffer; +} + +Datum Cursor::to_datum() && { + Datum ret((Nil())); + if (d->single) { + if (d->index != 0) { + throw Error("to_datum: already consumed"); + } + ret = std::move(d->buffer[0]); + } else { + d->clear_and_read_all(); + ret = std::move(d->buffer); + } + + return ret; +} + +void Cursor::close() const { + d->conn->stop_query(d->token); + d->no_more = true; +} + +bool Cursor::has_next(double wait) const { + if (d->single) { + d->convert_single(); + } + + while (true) { + if (d->index >= d->buffer.size()) { + if (d->no_more) { + return false; + } + d->add_response(d->conn->d->wait_for_response(d->token, wait)); + } else { + return true; + } + } +} + +bool Cursor::is_single() const { + return d->single; +} + +void CursorPrivate::add_results(Array&& results) const { + if (index >= buffer.size()) { + buffer = std::move(results); + index = 0; + } else { + for (auto& it : results) { + buffer.emplace_back(std::move(it)); + } + } +} + +void CursorPrivate::add_response(Response&& response) const { + using RT = Protocol::Response::ResponseType; + switch (response.type) { + case RT::SUCCESS_SEQUENCE: + add_results(std::move(response.result)); + no_more = true; + break; + case RT::SUCCESS_PARTIAL: + conn->continue_query(token); + add_results(std::move(response.result)); + break; + case RT::SUCCESS_ATOM: + add_results(std::move(response.result)); + single = true; + no_more = true; + break; + case RT::SERVER_INFO: + add_results(std::move(response.result)); + single = true; + no_more = true; + break; + case RT::WAIT_COMPLETE: + case RT::CLIENT_ERROR: + case RT::COMPILE_ERROR: + case RT::RUNTIME_ERROR: + no_more = true; + throw response.as_error(); + } +} + +Cursor::iterator Cursor::begin() { + return iterator(this); +} + +Cursor::iterator Cursor::end() { + return iterator(nullptr); +} + +Cursor::iterator::iterator(Cursor* cursor_) : cursor(cursor_) {} + +Cursor::iterator& Cursor::iterator::operator++ () { + if (cursor == nullptr) { + throw Error("incrementing an exhausted Cursor iterator"); + } + + cursor->next(); + return *this; +} + +Datum& Cursor::iterator::operator* () { + if (cursor == nullptr) { + throw Error("reading from empty Cursor iterator"); + } + + return cursor->peek(); +} + +bool Cursor::iterator::operator!= (const Cursor::iterator& other) const { + if (cursor == other.cursor) { + return false; + } + + return !((cursor == nullptr && !other.cursor->has_next()) || + (other.cursor == nullptr && !cursor->has_next())); +} + +} diff --git a/ext/librethinkdbxx/src/cursor.h b/ext/librethinkdbxx/src/cursor.h new file mode 100644 index 00000000..60ae1817 --- /dev/null +++ b/ext/librethinkdbxx/src/cursor.h @@ -0,0 +1,76 @@ +#pragma once + +#include "connection.h" + +namespace RethinkDB { + +// The response from the server, as returned by run. +// The response is either a single datum or a stream: +// * If it is a stream, the cursor represents each element of the stream. +// - Batches are fetched from the server as needed. +// * If it is a single datum, is_single() returns true. +// - If it is an array, the cursor represents each element of that array +// - Otherwise, to_datum() returns the datum and iteration throws an exception. +// The cursor can only be iterated over once, it discards data that has already been read. +class CursorPrivate; +class Cursor { +public: + Cursor() = delete; + ~Cursor(); + + Cursor(Cursor&&); // movable + Cursor& operator=(Cursor&&); + Cursor(const Cursor&) = delete; // not copyable + Cursor& operator=(const Cursor&) = delete; + + // Returned by begin() and end() + class iterator { + public: + iterator(Cursor*); + iterator& operator++ (); + Datum& operator* (); + bool operator!= (const iterator&) const; + + private: + Cursor *cursor; + }; + + // Consume the next element + Datum& next(double wait = FOREVER) const; + + // Peek at the next element + Datum& peek(double wait = FOREVER) const; + + // Call f on every element of the Cursor + void each(std::function f, double wait = FOREVER) const; + + // Consume and return all elements + Array&& to_array() &&; + + // If is_single(), returns the single datum. Otherwise returns to_array(). + Datum to_datum() &&; + Datum to_datum() const &; + + // Efficiently consume and return all elements + Array to_array() const &; + + // Close the cursor + void close() const; + + // Returns false if there are no more elements + bool has_next(double wait = FOREVER) const; + + // Returns false if the cursor is a stream + bool is_single() const; + + iterator begin(); + iterator end(); + +private: + explicit Cursor(CursorPrivate *dd); + std::unique_ptr d; + + friend class Connection; +}; + +} diff --git a/ext/librethinkdbxx/src/cursor_p.h b/ext/librethinkdbxx/src/cursor_p.h new file mode 100644 index 00000000..ce584cd7 --- /dev/null +++ b/ext/librethinkdbxx/src/cursor_p.h @@ -0,0 +1,29 @@ +#ifndef CURSOR_P_H +#define CURSOR_P_H + +#include "connection_p.h" + +namespace RethinkDB { + +class CursorPrivate { +public: + CursorPrivate(uint64_t token, Connection *conn); + CursorPrivate(uint64_t token, Connection *conn, Datum&&); + + void add_response(Response&&) const; + void add_results(Array&&) const; + void clear_and_read_all() const; + void convert_single() const; + + mutable bool single = false; + mutable bool no_more = false; + mutable size_t index = 0; + mutable Array buffer; + + uint64_t token; + Connection *conn; +}; + +} // namespace RethinkDB + +#endif // CURSOR_P_H \ No newline at end of file diff --git a/ext/librethinkdbxx/src/datum.cc b/ext/librethinkdbxx/src/datum.cc new file mode 100644 index 00000000..e4dbc8dc --- /dev/null +++ b/ext/librethinkdbxx/src/datum.cc @@ -0,0 +1,449 @@ +#include +#include + +#include "datum.h" +#include "json_p.h" +#include "utils.h" +#include "cursor.h" + +#include "rapidjson-config.h" +#include "rapidjson/prettywriter.h" +#include "rapidjson/stringbuffer.h" + +namespace RethinkDB { + +using TT = Protocol::Term::TermType; + +bool Datum::is_nil() const { + return type == Type::NIL; +} + +bool Datum::is_boolean() const { + return type == Type::BOOLEAN; +} + +bool Datum::is_number() const { + return type == Type::NUMBER; +} + +bool Datum::is_string() const { + return type == Type::STRING; +} + +bool Datum::is_object() const { + return type == Type::OBJECT; +} + +bool Datum::is_array() const { + return type == Type::ARRAY; +} + +bool Datum::is_binary() const { + return type == Type::BINARY; +} + +bool Datum::is_time() const { + return type == Type::TIME; +} + +bool* Datum::get_boolean() { + if (type == Type::BOOLEAN) { + return &value.boolean; + } else { + return NULL; + } +} + +const bool* Datum::get_boolean() const { + if (type == Type::BOOLEAN) { + return &value.boolean; + } else { + return NULL; + } +} + +double* Datum::get_number() { + if (type == Type::NUMBER) { + return &value.number; + } else { + return NULL; + } +} + +const double* Datum::get_number() const { + if (type == Type::NUMBER) { + return &value.number; + } else { + return NULL; + } +} + +std::string* Datum::get_string() { + if (type == Type::STRING) { + return &value.string; + } else { + return NULL; + } +} + +const std::string* Datum::get_string() const { + if (type == Type::STRING) { + return &value.string; + } else { + return NULL; + } +} + +Datum* Datum::get_field(std::string key) { + if (type != Type::OBJECT) { + return NULL; + } + auto it = value.object.find(key); + if (it == value.object.end()) { + return NULL; + } + return &it->second; +} + +const Datum* Datum::get_field(std::string key) const { + if (type != Type::OBJECT) { + return NULL; + } + auto it = value.object.find(key); + if (it == value.object.end()) { + return NULL; + } + return &it->second; +} + +Datum* Datum::get_nth(size_t i) { + if (type != Type::ARRAY) { + return NULL; + } + if (i >= value.array.size()) { + return NULL; + } + return &value.array[i]; +} + +const Datum* Datum::get_nth(size_t i) const { + if (type != Type::ARRAY) { + return NULL; + } + if (i >= value.array.size()) { + return NULL; + } + return &value.array[i]; +} + +Object* Datum::get_object() { + if (type == Type::OBJECT) { + return &value.object; + } else { + return NULL; + } +} + +const Object* Datum::get_object() const { + if (type == Type::OBJECT) { + return &value.object; + } else { + return NULL; + } +} + +Array* Datum::get_array() { + if (type == Type::ARRAY) { + return &value.array; + } else { + return NULL; + } +} + +const Array* Datum::get_array() const { + if (type == Type::ARRAY) { + return &value.array; + } else { + return NULL; + } +} + +Binary* Datum::get_binary() { + if (type == Type::BINARY) { + return &value.binary; + } else { + return NULL; + } +} + +const Binary* Datum::get_binary() const { + if (type == Type::BINARY) { + return &value.binary; + } else { + return NULL; + } +} + +Time* Datum::get_time() { + if (type == Type::TIME) { + return &value.time; + } else { + return NULL; + } +} + +const Time* Datum::get_time() const { + if (type == Type::TIME) { + return &value.time; + } else { + return NULL; + } +} + +bool& Datum::extract_boolean() { + if (type != Type::BOOLEAN) { + throw Error("extract_bool: Not a boolean"); + } + return value.boolean; +} + +double& Datum::extract_number() { + if (type != Type::NUMBER) { + throw Error("extract_number: Not a number: %s", write_datum(*this).c_str()); + } + return value.number; +} + +std::string& Datum::extract_string() { + if (type != Type::STRING) { + throw Error("extract_string: Not a string"); + } + return value.string; +} + +Object& Datum::extract_object() { + if (type != Type::OBJECT) { + throw Error("extract_object: Not an object"); + } + return value.object; +} + +Datum& Datum::extract_field(std::string key) { + if (type != Type::OBJECT) { + throw Error("extract_field: Not an object"); + } + auto it = value.object.find(key); + if (it == value.object.end()) { + throw Error("extract_field: No such key in object"); + } + return it->second; +} + +Datum& Datum::extract_nth(size_t i) { + if (type != Type::ARRAY) { + throw Error("extract_nth: Not an array"); + } + if (i >= value.array.size()) { + throw Error("extract_nth: index too large"); + } + return value.array[i]; +} + +Array& Datum::extract_array() { + if (type != Type::ARRAY) { + throw Error("get_array: Not an array"); + } + return value.array; +} + +Binary& Datum::extract_binary() { + if (type != Type::BINARY) { + throw Error("get_binary: Not a binary"); + } + return value.binary; +} + +Time& Datum::extract_time() { + if (type != Type::TIME) { + throw Error("get_time: Not a time"); + } + return value.time; +} + +int Datum::compare(const Datum& other) const { +#define COMPARE(a, b) do { \ + if (a < b) { return -1; } \ + if (a > b) { return 1; } } while(0) +#define COMPARE_OTHER(x) COMPARE(x, other.x) + + COMPARE_OTHER(type); + int c; + switch (type) { + case Type::NIL: case Type::INVALID: break; + case Type::BOOLEAN: COMPARE_OTHER(value.boolean); break; + case Type::NUMBER: COMPARE_OTHER(value.number); break; + case Type::STRING: + c = value.string.compare(other.value.string); + COMPARE(c, 0); + break; + case Type::BINARY: + c = value.binary.data.compare(other.value.binary.data); + COMPARE(c, 0); + break; + case Type::TIME: + COMPARE(value.time.epoch_time, other.value.time.epoch_time); + COMPARE(value.time.utc_offset, other.value.time.utc_offset); + break; + case Type::ARRAY: + COMPARE_OTHER(value.array.size()); + for (size_t i = 0; i < value.array.size(); i++) { + c = value.array[i].compare(other.value.array[i]); + COMPARE(c, 0); + } + break; + case Type::OBJECT: + COMPARE_OTHER(value.object.size()); + for (Object::const_iterator l = value.object.begin(), + r = other.value.object.begin(); + l != value.object.end(); + ++l, ++r) { + COMPARE(l->first, r->first); + c = l->second.compare(r->second); + COMPARE(c, 0); + } + break; + default: + throw Error("cannot compare invalid datum"); + } + return 0; +#undef COMPARE_OTHER +#undef COMPARE +} + +bool Datum::operator== (const Datum& other) const { + return compare(other) == 0; +} + +Datum Datum::from_raw() const { + do { + const Datum* type_field = get_field("$reql_type$"); + if (!type_field) break; + const std::string* type = type_field->get_string(); + if (!type) break;; + if (!strcmp(type->c_str(), "BINARY")) { + const Datum* data_field = get_field("data"); + if (!data_field) break; + const std::string* encoded_data = data_field->get_string(); + if (!encoded_data) break; + Binary binary(""); + if (base64_decode(*encoded_data, binary.data)) { + return binary; + } + } else if (!strcmp(type->c_str(), "TIME")) { + const Datum* epoch_field = get_field("epoch_time"); + if (!epoch_field) break; + const Datum* tz_field = get_field("timezone"); + if (!tz_field) break; + const double* epoch_time = epoch_field->get_number(); + if (!epoch_time) break; + const std::string* tz = tz_field->get_string(); + if (!tz) break; + double offset; + if (!Time::parse_utc_offset(*tz, &offset)) break; + return Time(*epoch_time, offset); + } + } while (0); + return *this; +} + +Datum Datum::to_raw() const { + if (type == Type::BINARY) { + return Object{ + {"$reql_type$", "BINARY"}, + {"data", base64_encode(value.binary.data)}}; + } else if (type == Type::TIME) { + return Object{ + {"$reql_type$", "TIME"}, + {"epoch_time", value.time.epoch_time}, + {"timezone", Time::utc_offset_string(value.time.utc_offset)}}; + } + return *this; +} + +Datum::Datum(Cursor&& cursor) : Datum(cursor.to_datum()) { } +Datum::Datum(const Cursor& cursor) : Datum(cursor.to_datum()) { } + +static const double max_dbl_int = 0x1LL << DBL_MANT_DIG; +static const double min_dbl_int = max_dbl_int * -1; +bool number_as_integer(double d, int64_t *i_out) { + static_assert(DBL_MANT_DIG == 53, "Doubles are wrong size."); + + if (min_dbl_int <= d && d <= max_dbl_int) { + int64_t i = d; + if (static_cast(i) == d) { + *i_out = i; + return true; + } + } + return false; +} + +template void Datum::write_json( + rapidjson::Writer *writer) const; +template void Datum::write_json( + rapidjson::PrettyWriter *writer) const; + +template +void Datum::write_json(json_writer_t *writer) const { + switch (type) { + case Type::NIL: writer->Null(); break; + case Type::BOOLEAN: writer->Bool(value.boolean); break; + case Type::NUMBER: { + const double d = value.number; + // Always print -0.0 as a double since integers cannot represent -0. + // Otherwise check if the number is an integer and print it as such. + int64_t i; + if (!(d == 0.0 && std::signbit(d)) && number_as_integer(d, &i)) { + writer->Int64(i); + } else { + writer->Double(d); + } + } break; + case Type::STRING: writer->String(value.string.data(), value.string.size()); break; + case Type::ARRAY: { + writer->StartArray(); + for (auto it : value.array) { + it.write_json(writer); + } + writer->EndArray(); + } break; + case Type::OBJECT: { + writer->StartObject(); + for (auto it : value.object) { + writer->Key(it.first.data(), it.first.size()); + it.second.write_json(writer); + } + writer->EndObject(); + } break; + + case Type::BINARY: + case Type::TIME: + to_raw().write_json(writer); + break; + default: + throw Error("cannot write invalid datum"); + } +} + +std::string Datum::as_json() const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + write_json(&writer); + return std::string(buffer.GetString(), buffer.GetSize()); +} + +Datum Datum::from_json(const std::string& json) { + return read_datum(json); +} + +} // namespace RethinkDB diff --git a/ext/librethinkdbxx/src/datum.h b/ext/librethinkdbxx/src/datum.h new file mode 100644 index 00000000..051e2ca2 --- /dev/null +++ b/ext/librethinkdbxx/src/datum.h @@ -0,0 +1,287 @@ +#pragma once + +#include +#include +#include +#include + +#include "protocol_defs.h" +#include "error.h" +#include "types.h" + +namespace RethinkDB { + +class Cursor; + +// The type of data stored in a RethinkDB database. +// The following JSON types are represented in a Datum as +// * null -> Nil +// * boolean -> bool +// * number -> double +// * unicode strings -> std::string +// * array -> Array (aka std::vector +// * object -> Object (aka std::map> +// Datums can also contain one of the following extra types +// * binary strings -> Binary +// * timestamps -> Time +// * points. lines and polygons -> not implemented +class Datum { +public: + Datum() : type(Type::INVALID), value() {} + Datum(Nil) : type(Type::NIL), value() { } + Datum(bool boolean_) : type(Type::BOOLEAN), value(boolean_) { } + Datum(double number_) : type(Type::NUMBER), value(number_) { } + Datum(const std::string& string_) : type(Type::STRING), value(string_) { } + Datum(std::string&& string_) : type(Type::STRING), value(std::move(string_)) { } + Datum(const Array& array_) : type(Type::ARRAY), value(array_) { } + Datum(Array&& array_) : type(Type::ARRAY), value(std::move(array_)) { } + Datum(const Binary& binary) : type(Type::BINARY), value(binary) { } + Datum(Binary&& binary) : type(Type::BINARY), value(std::move(binary)) { } + Datum(const Time time) : type(Type::TIME), value(time) { } + Datum(const Object& object_) : type(Type::OBJECT), value(object_) { } + Datum(Object&& object_) : type(Type::OBJECT), value(std::move(object_)) { } + Datum(const Datum& other) : type(other.type), value(other.type, other.value) { } + Datum(Datum&& other) : type(other.type), value(other.type, std::move(other.value)) { } + + Datum& operator=(const Datum& other) { + value.destroy(type); + type = other.type; + value.set(type, other.value); + return *this; + } + + Datum& operator=(Datum&& other) { + value.destroy(type); + type = other.type; + value.set(type, std::move(other.value)); + return *this; + } + + Datum(unsigned short number_) : Datum(static_cast(number_)) { } + Datum(signed short number_) : Datum(static_cast(number_)) { } + Datum(unsigned int number_) : Datum(static_cast(number_)) { } + Datum(signed int number_) : Datum(static_cast(number_)) { } + Datum(unsigned long number_) : Datum(static_cast(number_)) { } + Datum(signed long number_) : Datum(static_cast(number_)) { } + Datum(unsigned long long number_) : Datum(static_cast(number_)) { } + Datum(signed long long number_) : Datum(static_cast(number_)) { } + + Datum(Protocol::Term::TermType type) : Datum(static_cast(type)) { } + Datum(const char* string) : Datum(static_cast(string)) { } + + // Cursors are implicitly converted into datums + Datum(Cursor&&); + Datum(const Cursor&); + + template + Datum(const std::map& map) : type(Type::OBJECT), value(Object()) { + for (const auto& it : map) { + value.object.emplace(it.left, Datum(it.right)); + } + } + + template + Datum(std::map&& map) : type(Type::OBJECT), value(Object()) { + for (auto& it : map) { + value.object.emplace(it.first, Datum(std::move(it.second))); + } + } + + template + Datum(const std::vector& vec) : type(Type::ARRAY), value(Array()) { + for (const auto& it : vec) { + value.array.emplace_back(it); + } + } + + template + Datum(std::vector&& vec) : type(Type::ARRAY), value(Array()) { + for (auto& it : vec) { + value.array.emplace_back(std::move(it)); + } + } + + ~Datum() { + value.destroy(type); + } + + // Apply a visitor + template + R apply(F f, A&& ...args) const & { + switch (type) { + case Type::NIL: return f(Nil(), std::forward(args)...); break; + case Type::BOOLEAN: return f(value.boolean, std::forward(args)...); break; + case Type::NUMBER: return f(value.number, std::forward(args)...); break; + case Type::STRING: return f(value.string, std::forward(args)...); break; + case Type::OBJECT: return f(value.object, std::forward(args)...); break; + case Type::ARRAY: return f(value.array, std::forward(args)...); break; + case Type::BINARY: return f(value.binary, std::forward(args)...); break; + case Type::TIME: return f(value.time, std::forward(args)...); break; + default: + throw Error("internal error: no such datum type %d", static_cast(type)); + } + } + + template + R apply(F f, A&& ...args) && { + switch (type) { + case Type::NIL: return f(Nil(), std::forward(args)...); break; + case Type::BOOLEAN: return f(std::move(value.boolean), std::forward(args)...); break; + case Type::NUMBER: return f(std::move(value.number), std::forward(args)...); break; + case Type::STRING: return f(std::move(value.string), std::forward(args)...); break; + case Type::OBJECT: return f(std::move(value.object), std::forward(args)...); break; + case Type::ARRAY: return f(std::move(value.array), std::forward(args)...); break; + case Type::BINARY: return f(std::move(value.binary), std::forward(args)...); break; + case Type::TIME: return f(std::move(value.time), std::forward(args)...); break; + default: + throw Error("internal error: no such datum type %d", static_cast(type)); + } + } + + bool is_nil() const; + bool is_boolean() const; + bool is_number() const; + bool is_string() const; + bool is_object() const; + bool is_array() const; + bool is_binary() const; + bool is_time() const; + + // get_* returns nullptr if the datum has a different type + + bool* get_boolean(); + const bool* get_boolean() const; + double* get_number(); + const double* get_number() const; + std::string* get_string(); + const std::string* get_string() const; + Object* get_object(); + const Object* get_object() const; + Datum* get_field(std::string); + const Datum* get_field(std::string) const; + Array* get_array(); + const Array* get_array() const; + Datum* get_nth(size_t); + const Datum* get_nth(size_t) const; + Binary* get_binary(); + const Binary* get_binary() const; + Time* get_time(); + const Time* get_time() const; + + // extract_* throws an exception if the types don't match + + bool& extract_boolean(); + double& extract_number(); + std::string& extract_string(); + Object& extract_object(); + Datum& extract_field(std::string); + Array& extract_array(); + Datum& extract_nth(size_t); + Binary& extract_binary(); + Time& extract_time(); + + // negative, zero or positive if this datum is smaller, identical or larger than the other one, respectively + // This is meant to match the results of RethinkDB's comparison operators + int compare(const Datum&) const; + + // Deep equality + bool operator== (const Datum&) const; + + // Recusively replace non-JSON types into objects that represent them + Datum to_raw() const; + + // Recursively replace objects with a $reql_type$ field into the datum they represent + Datum from_raw() const; + + template void write_json(json_writer_t *writer) const; + + std::string as_json() const; + static Datum from_json(const std::string&); + + bool is_valid() const { return type != Type::INVALID; } + +private: + enum class Type { + INVALID, // default constructed + ARRAY, BOOLEAN, NIL, NUMBER, OBJECT, BINARY, STRING, TIME + // POINT, LINE, POLYGON + }; + Type type; + + union datum_value { + bool boolean; + double number; + std::string string; + Object object; + Array array; + Binary binary; + Time time; + + datum_value() { } + datum_value(bool boolean_) : boolean(boolean_) { } + datum_value(double number_) : number(number_) { } + datum_value(const std::string& string_) : string(string_) { } + datum_value(std::string&& string_) : string(std::move(string_)) { } + datum_value(const Object& object_) : object(object_) { } + datum_value(Object&& object_) : object(std::move(object_)) { } + datum_value(const Array& array_) : array(array_) { } + datum_value(Array&& array_) : array(std::move(array_)) { } + datum_value(const Binary& binary_) : binary(binary_) { } + datum_value(Binary&& binary_) : binary(std::move(binary_)) { } + datum_value(Time time) : time(std::move(time)) { } + + datum_value(Type type, const datum_value& other){ + set(type, other); + } + + datum_value(Type type, datum_value&& other){ + set(type, std::move(other)); + } + + void set(Type type, datum_value&& other) { + switch(type){ + case Type::NIL: case Type::INVALID: break; + case Type::BOOLEAN: new (this) bool(other.boolean); break; + case Type::NUMBER: new (this) double(other.number); break; + case Type::STRING: new (this) std::string(std::move(other.string)); break; + case Type::OBJECT: new (this) Object(std::move(other.object)); break; + case Type::ARRAY: new (this) Array(std::move(other.array)); break; + case Type::BINARY: new (this) Binary(std::move(other.binary)); break; + case Type::TIME: new (this) Time(std::move(other.time)); break; + } + } + + void set(Type type, const datum_value& other) { + switch(type){ + case Type::NIL: case Type::INVALID: break; + case Type::BOOLEAN: new (this) bool(other.boolean); break; + case Type::NUMBER: new (this) double(other.number); break; + case Type::STRING: new (this) std::string(other.string); break; + case Type::OBJECT: new (this) Object(other.object); break; + case Type::ARRAY: new (this) Array(other.array); break; + case Type::BINARY: new (this) Binary(other.binary); break; + case Type::TIME: new (this) Time(other.time); break; + } + } + + void destroy(Type type) { + switch(type){ + case Type::INVALID: break; + case Type::NIL: break; + case Type::BOOLEAN: break; + case Type::NUMBER: break; + case Type::STRING: { typedef std::string str; string.~str(); } break; + case Type::OBJECT: object.~Object(); break; + case Type::ARRAY: array.~Array(); break; + case Type::BINARY: binary.~Binary(); break; + case Type::TIME: time.~Time(); break; + } + } + + ~datum_value() { } + }; + + datum_value value; +}; + +} diff --git a/ext/librethinkdbxx/src/error.h b/ext/librethinkdbxx/src/error.h new file mode 100644 index 00000000..ab75e248 --- /dev/null +++ b/ext/librethinkdbxx/src/error.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + +namespace RethinkDB { + +// All errors thrown by the server have this type +struct Error { + template + explicit Error(const char* format_, T... A) { + format(format_, A...); + } + + Error() = default; + Error(Error&&) = default; + Error(const Error&) = default; + + Error& operator= (Error&& other) { + message = std::move(other.message); + return *this; + } + + static Error from_errno(const char* str){ + return Error("%s: %s", str, strerror(errno)); + } + + // The error message + std::string message; + +private: + const size_t max_message_size = 2048; + + void format(const char* format_, ...) { + va_list args; + va_start(args, format_); + char message_[max_message_size]; + vsnprintf(message_, max_message_size, format_, args); + va_end(args); + message = message_; + } +}; + +} diff --git a/ext/librethinkdbxx/src/exceptions.h b/ext/librethinkdbxx/src/exceptions.h new file mode 100644 index 00000000..08c0b0a0 --- /dev/null +++ b/ext/librethinkdbxx/src/exceptions.h @@ -0,0 +1,13 @@ +#ifndef EXCEPTIONS_H +#define EXCEPTIONS_H + +namespace RethinkDB { + +class TimeoutException : public std::exception { +public: + const char *what() const throw () { return "operation timed out"; } +}; + +} + +#endif // EXCEPTIONS_H diff --git a/ext/librethinkdbxx/src/json.cc b/ext/librethinkdbxx/src/json.cc new file mode 100644 index 00000000..c908eefb --- /dev/null +++ b/ext/librethinkdbxx/src/json.cc @@ -0,0 +1,62 @@ +#include "json_p.h" +#include "error.h" +#include "utils.h" + +#include "rapidjson-config.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" +#include "rapidjson/prettywriter.h" + +namespace RethinkDB { + +Datum read_datum(const std::string& json) { + rapidjson::Document document; + document.Parse(json); + return read_datum(document); +} + +Datum read_datum(const rapidjson::Value &json) { + switch(json.GetType()) { + case rapidjson::kNullType: return Nil(); + case rapidjson::kFalseType: return false; + case rapidjson::kTrueType: return true; + case rapidjson::kNumberType: return json.GetDouble(); + case rapidjson::kStringType: + return std::string(json.GetString(), json.GetStringLength()); + + case rapidjson::kObjectType: { + Object result; + for (rapidjson::Value::ConstMemberIterator it = json.MemberBegin(); + it != json.MemberEnd(); ++it) { + result.insert(std::make_pair(std::string(it->name.GetString(), + it->name.GetStringLength()), + read_datum(it->value))); + } + + if (result.count("$reql_type$")) + return Datum(std::move(result)).from_raw(); + return std::move(result); + } break; + case rapidjson::kArrayType: { + Array result; + result.reserve(json.Size()); + for (rapidjson::Value::ConstValueIterator it = json.Begin(); + it != json.End(); ++it) { + result.push_back(read_datum(*it)); + } + return std::move(result); + } break; + default: + throw Error("invalid rapidjson value"); + } +} + +std::string write_datum(const Datum& datum) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + datum.write_json(&writer); + return std::string(buffer.GetString(), buffer.GetSize()); +} + +} diff --git a/ext/librethinkdbxx/src/json_p.h b/ext/librethinkdbxx/src/json_p.h new file mode 100644 index 00000000..ebf537a9 --- /dev/null +++ b/ext/librethinkdbxx/src/json_p.h @@ -0,0 +1,19 @@ +#pragma once + +#include "datum.h" + +namespace rapidjson { + class CrtAllocator; + template struct UTF8; + template class GenericValue; + template class MemoryPoolAllocator; + typedef GenericValue, MemoryPoolAllocator > Value; +} + +namespace RethinkDB { + +Datum read_datum(const std::string&); +Datum read_datum(const rapidjson::Value &json); +std::string write_datum(const Datum&); + +} diff --git a/ext/librethinkdbxx/src/rapidjson-config.h b/ext/librethinkdbxx/src/rapidjson-config.h new file mode 100644 index 00000000..320c4048 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson-config.h @@ -0,0 +1,8 @@ +#pragma once + +#define RAPIDJSON_HAS_STDSTRING 1 +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#define RAPIDJSON_HAS_CXX11_TYPETRAITS 1 +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 +#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseFullPrecisionFlag diff --git a/ext/librethinkdbxx/src/rapidjson/allocators.h b/ext/librethinkdbxx/src/rapidjson/allocators.h new file mode 100644 index 00000000..c7059697 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/allocators.h @@ -0,0 +1,263 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ALLOCATORS_H_ +#define RAPIDJSON_ALLOCATORS_H_ + +#include "rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Allocator + +/*! \class rapidjson::Allocator + \brief Concept for allocating, resizing and freeing memory block. + + Note that Malloc() and Realloc() are non-static but Free() is static. + + So if an allocator need to support Free(), it needs to put its pointer in + the header of memory block. + +\code +concept Allocator { + static const bool kNeedFree; //!< Whether this allocator needs to call Free(). + + // Allocate a memory block. + // \param size of the memory block in bytes. + // \returns pointer to the memory block. + void* Malloc(size_t size); + + // Resize a memory block. + // \param originalPtr The pointer to current memory block. Null pointer is permitted. + // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) + // \param newSize the new size in bytes. + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); + + // Free a memory block. + // \param pointer to the memory block. Null pointer is permitted. + static void Free(void *ptr); +}; +\endcode +*/ + +/////////////////////////////////////////////////////////////////////////////// +// CrtAllocator + +//! C-runtime library allocator. +/*! This class is just wrapper for standard C library memory routines. + \note implements Allocator concept +*/ +class CrtAllocator { +public: + static const bool kNeedFree = true; + void* Malloc(size_t size) { + if (size) // behavior of malloc(0) is implementation defined. + return std::malloc(size); + else + return NULL; // standardize to returning NULL. + } + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { + (void)originalSize; + if (newSize == 0) { + std::free(originalPtr); + return NULL; + } + return std::realloc(originalPtr, newSize); + } + static void Free(void *ptr) { std::free(ptr); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// MemoryPoolAllocator + +//! Default memory allocator used by the parser and DOM. +/*! This allocator allocate memory blocks from pre-allocated memory chunks. + + It does not free memory blocks. And Realloc() only allocate new memory. + + The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. + + User may also supply a buffer as the first chunk. + + If the user-buffer is full then additional chunks are allocated by BaseAllocator. + + The user-buffer is not deallocated by this allocator. + + \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. + \note implements Allocator concept +*/ +template +class MemoryPoolAllocator { +public: + static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) + + //! Constructor with chunkSize. + /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + } + + //! Constructor with user-supplied buffer. + /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. + + The user buffer will not be deallocated when this allocator is destructed. + + \param buffer User supplied buffer. + \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). + \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + RAPIDJSON_ASSERT(buffer != 0); + RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); + chunkHead_ = reinterpret_cast(buffer); + chunkHead_->capacity = size - sizeof(ChunkHeader); + chunkHead_->size = 0; + chunkHead_->next = 0; + } + + //! Destructor. + /*! This deallocates all memory chunks, excluding the user-supplied buffer. + */ + ~MemoryPoolAllocator() { + Clear(); + RAPIDJSON_DELETE(ownBaseAllocator_); + } + + //! Deallocates all memory chunks, excluding the user-supplied buffer. + void Clear() { + while (chunkHead_ && chunkHead_ != userBuffer_) { + ChunkHeader* next = chunkHead_->next; + baseAllocator_->Free(chunkHead_); + chunkHead_ = next; + } + if (chunkHead_ && chunkHead_ == userBuffer_) + chunkHead_->size = 0; // Clear user buffer + } + + //! Computes the total capacity of allocated memory chunks. + /*! \return total capacity in bytes. + */ + size_t Capacity() const { + size_t capacity = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + capacity += c->capacity; + return capacity; + } + + //! Computes the memory blocks allocated. + /*! \return total used bytes. + */ + size_t Size() const { + size_t size = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + size += c->size; + return size; + } + + //! Allocates a memory block. (concept Allocator) + void* Malloc(size_t size) { + if (!size) + return NULL; + + size = RAPIDJSON_ALIGN(size); + if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) + AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); + + void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; + chunkHead_->size += size; + return buffer; + } + + //! Resizes a memory block (concept Allocator) + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { + if (originalPtr == 0) + return Malloc(newSize); + + if (newSize == 0) + return NULL; + + originalSize = RAPIDJSON_ALIGN(originalSize); + newSize = RAPIDJSON_ALIGN(newSize); + + // Do not shrink if new size is smaller than original + if (originalSize >= newSize) + return originalPtr; + + // Simply expand it if it is the last allocation and there is sufficient space + if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { + size_t increment = static_cast(newSize - originalSize); + if (chunkHead_->size + increment <= chunkHead_->capacity) { + chunkHead_->size += increment; + return originalPtr; + } + } + + // Realloc process: allocate and copy memory, do not free original buffer. + void* newBuffer = Malloc(newSize); + RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. + if (originalSize) + std::memcpy(newBuffer, originalPtr, originalSize); + return newBuffer; + } + + //! Frees a memory block (concept Allocator) + static void Free(void *ptr) { (void)ptr; } // Do nothing + +private: + //! Copy constructor is not permitted. + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; + //! Copy assignment operator is not permitted. + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; + + //! Creates a new chunk. + /*! \param capacity Capacity of the chunk in bytes. + */ + void AddChunk(size_t capacity) { + if (!baseAllocator_) + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); + ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity)); + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; + } + + static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. + + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader { + size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. + }; + + ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. + void *userBuffer_; //!< User supplied buffer. + BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. + BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. +}; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/document.h b/ext/librethinkdbxx/src/rapidjson/document.h new file mode 100644 index 00000000..f7f846f2 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/document.h @@ -0,0 +1,2575 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_DOCUMENT_H_ +#define RAPIDJSON_DOCUMENT_H_ + +/*! \file document.h */ + +#include "reader.h" +#include "internal/meta.h" +#include "internal/strfunc.h" +#include "memorystream.h" +#include "encodedstream.h" +#include // placement new +#include + +RAPIDJSON_DIAG_PUSH +#ifdef _MSC_VER +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 6 +RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions +#endif +#endif // __GNUC__ + +#ifndef RAPIDJSON_NOMEMBERITERATORCLASS +#include // std::iterator, std::random_access_iterator_tag +#endif + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +// Forward declaration. +template +class GenericValue; + +template +class GenericDocument; + +//! Name-value pair in a JSON object value. +/*! + This class was internal to GenericValue. It used to be a inner struct. + But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. + https://code.google.com/p/rapidjson/issues/detail?id=64 +*/ +template +struct GenericMember { + GenericValue name; //!< name of member (must be a string) + GenericValue value; //!< value of member. +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericMemberIterator + +#ifndef RAPIDJSON_NOMEMBERITERATORCLASS + +//! (Constant) member iterator for a JSON object value +/*! + \tparam Const Is this a constant iterator? + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. + + This class implements a Random Access Iterator for GenericMember elements + of a GenericValue, see ISO/IEC 14882:2003(E) C++ standard, 24.1 [lib.iterator.requirements]. + + \note This iterator implementation is mainly intended to avoid implicit + conversions from iterator values to \c NULL, + e.g. from GenericValue::FindMember. + + \note Define \c RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a + pointer-based implementation, if your platform doesn't provide + the C++ header. + + \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator + */ +template +class GenericMemberIterator + : public std::iterator >::Type> { + + friend class GenericValue; + template friend class GenericMemberIterator; + + typedef GenericMember PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef std::iterator BaseType; + +public: + //! Iterator type itself + typedef GenericMemberIterator Iterator; + //! Constant iterator type + typedef GenericMemberIterator ConstIterator; + //! Non-constant iterator type + typedef GenericMemberIterator NonConstIterator; + + //! Pointer to (const) GenericMember + typedef typename BaseType::pointer Pointer; + //! Reference to (const) GenericMember + typedef typename BaseType::reference Reference; + //! Signed integer type (e.g. \c ptrdiff_t) + typedef typename BaseType::difference_type DifferenceType; + + //! Default constructor (singular value) + /*! Creates an iterator pointing to no element. + \note All operations, except for comparisons, are undefined on such values. + */ + GenericMemberIterator() : ptr_() {} + + //! Iterator conversions to more const + /*! + \param it (Non-const) iterator to copy from + + Allows the creation of an iterator from another GenericMemberIterator + that is "less const". Especially, creating a non-constant iterator + from a constant iterator are disabled: + \li const -> non-const (not ok) + \li const -> const (ok) + \li non-const -> const (ok) + \li non-const -> non-const (ok) + + \note If the \c Const template parameter is already \c false, this + constructor effectively defines a regular copy-constructor. + Otherwise, the copy constructor is implicitly defined. + */ + GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} + Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } + + //! @name stepping + //@{ + Iterator& operator++(){ ++ptr_; return *this; } + Iterator& operator--(){ --ptr_; return *this; } + Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } + Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } + //@} + + //! @name increment/decrement + //@{ + Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } + Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } + + Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } + Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } + //@} + + //! @name relations + //@{ + bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } + bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } + bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } + bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } + bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } + bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } + //@} + + //! @name dereference + //@{ + Reference operator*() const { return *ptr_; } + Pointer operator->() const { return ptr_; } + Reference operator[](DifferenceType n) const { return ptr_[n]; } + //@} + + //! Distance + DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } + +private: + //! Internal constructor from plain pointer + explicit GenericMemberIterator(Pointer p) : ptr_(p) {} + + Pointer ptr_; //!< raw pointer +}; + +#else // RAPIDJSON_NOMEMBERITERATORCLASS + +// class-based member iterator implementation disabled, use plain pointers + +template +struct GenericMemberIterator; + +//! non-const GenericMemberIterator +template +struct GenericMemberIterator { + //! use plain pointer as iterator type + typedef GenericMember* Iterator; +}; +//! const GenericMemberIterator +template +struct GenericMemberIterator { + //! use plain const pointer as iterator type + typedef const GenericMember* Iterator; +}; + +#endif // RAPIDJSON_NOMEMBERITERATORCLASS + +/////////////////////////////////////////////////////////////////////////////// +// GenericStringRef + +//! Reference to a constant string (not taking a copy) +/*! + \tparam CharType character type of the string + + This helper class is used to automatically infer constant string + references for string literals, especially from \c const \b (!) + character arrays. + + The main use is for creating JSON string values without copying the + source string via an \ref Allocator. This requires that the referenced + string pointers have a sufficient lifetime, which exceeds the lifetime + of the associated GenericValue. + + \b Example + \code + Value v("foo"); // ok, no need to copy & calculate length + const char foo[] = "foo"; + v.SetString(foo); // ok + + const char* bar = foo; + // Value x(bar); // not ok, can't rely on bar's lifetime + Value x(StringRef(bar)); // lifetime explicitly guaranteed by user + Value y(StringRef(bar, 3)); // ok, explicitly pass length + \endcode + + \see StringRef, GenericValue::SetString +*/ +template +struct GenericStringRef { + typedef CharType Ch; //!< character type of the string + + //! Create string reference from \c const character array +#ifndef __clang__ // -Wdocumentation + /*! + This constructor implicitly creates a constant string reference from + a \c const character array. It has better performance than + \ref StringRef(const CharType*) by inferring the string \ref length + from the array length, and also supports strings containing null + characters. + + \tparam N length of the string, automatically inferred + + \param str Constant character array, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note Constant complexity. + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ +#endif + template + GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT + : s(str), length(N-1) {} + + //! Explicitly create string reference from \c const character pointer +#ifndef __clang__ // -Wdocumentation + /*! + This constructor can be used to \b explicitly create a reference to + a constant string pointer. + + \see StringRef(const CharType*) + + \param str Constant character pointer, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ +#endif + explicit GenericStringRef(const CharType* str) + : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != 0); } + + //! Create constant string reference from pointer and length +#ifndef __clang__ // -Wdocumentation + /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param len length of the string, excluding the trailing NULL terminator + + \post \ref s == str && \ref length == len + \note Constant complexity. + */ +#endif + GenericStringRef(const CharType* str, SizeType len) + : s(str), length(len) { RAPIDJSON_ASSERT(s != 0); } + + GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} + + GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } + + //! implicit conversion to plain CharType pointer + operator const Ch *() const { return s; } + + const Ch* const s; //!< plain CharType pointer + const SizeType length; //!< length of the string (excluding the trailing NULL terminator) + +private: + //! Disallow construction from non-const array + template + GenericStringRef(CharType (&str)[N]) /* = delete */; +}; + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + \tparam CharType Character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + + \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember +*/ +template +inline GenericStringRef StringRef(const CharType* str) { + return GenericStringRef(str, internal::StrLen(str)); +} + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + This version has better performance with supplied length, and also + supports string containing null characters. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param length The length of source string. + \return GenericStringRef string reference object + \relatesalso GenericStringRef +*/ +template +inline GenericStringRef StringRef(const CharType* str, size_t length) { + return GenericStringRef(str, SizeType(length)); +} + +#if RAPIDJSON_HAS_STDSTRING +//! Mark a string object as constant string +/*! Mark a string object (e.g. \c std::string) as a "string literal". + This function can be used to avoid copying a string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. +*/ +template +inline GenericStringRef StringRef(const std::basic_string& str) { + return GenericStringRef(str.data(), SizeType(str.size())); +} +#endif + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue type traits +namespace internal { + +template +struct IsGenericValueImpl : FalseType {}; + +// select candidates according to nested encoding and allocator types +template struct IsGenericValueImpl::Type, typename Void::Type> + : IsBaseOf, T>::Type {}; + +// helper to match arbitrary GenericValue instantiations, including derived classes +template struct IsGenericValue : IsGenericValueImpl::Type {}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// TypeHelper + +namespace internal { + +template +struct TypeHelper {}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsBool(); } + static bool Get(const ValueType& v) { return v.GetBool(); } + static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } + static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt(); } + static int Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt64(); } + static int64_t Get(const ValueType& v) { return v.GetInt64(); } + static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } + static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint64(); } + static uint64_t Get(const ValueType& v) { return v.GetUint64(); } + static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } + static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsDouble(); } + static double Get(const ValueType& v) { return v.GetDouble(); } + static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } + static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsFloat(); } + static float Get(const ValueType& v) { return v.GetFloat(); } + static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } + static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } +}; + +template +struct TypeHelper { + typedef const typename ValueType::Ch* StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } + static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; + +#if RAPIDJSON_HAS_STDSTRING +template +struct TypeHelper > { + typedef std::basic_string StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; +#endif + +template +struct TypeHelper { + typedef typename ValueType::Array ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(ValueType& v) { return v.GetArray(); } + static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } + static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstArray ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(const ValueType& v) { return v.GetArray(); } +}; + +template +struct TypeHelper { + typedef typename ValueType::Object ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(ValueType& v) { return v.GetObject(); } + static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } + static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstObject ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(const ValueType& v) { return v.GetObject(); } +}; + +} // namespace internal + +// Forward declarations +template class GenericArray; +template class GenericObject; + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue + +//! Represents a JSON value. Use Value for UTF8 encoding and default allocator. +/*! + A JSON value can be one of 7 types. This class is a variant type supporting + these types. + + Use the Value if UTF8 and default allocator + + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. +*/ +template > +class GenericValue { +public: + //! Name-value pair in an object. + typedef GenericMember Member; + typedef Encoding EncodingType; //!< Encoding type from template parameter. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericStringRef StringRefType; //!< Reference to a constant string + typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. + typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. + typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. + typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. + typedef GenericValue ValueType; //!< Value type of itself. + typedef GenericArray Array; + typedef GenericArray ConstArray; + typedef GenericObject Object; + typedef GenericObject ConstObject; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor creates a null value. + GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { + rhs.data_.f.flags = kNullFlag; // give up contents + } +#endif + +private: + //! Copy constructor is not permitted. + GenericValue(const GenericValue& rhs); + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Moving from a GenericDocument is not permitted. + template + GenericValue(GenericDocument&& rhs); + + //! Move assignment from a GenericDocument is not permitted. + template + GenericValue& operator=(GenericDocument&& rhs); +#endif + +public: + + //! Constructor with JSON value type. + /*! This creates a Value of specified type with default content. + \param type Type of the value. + \note Default content for number is zero. + */ + explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { + static const uint16_t defaultFlags[7] = { + kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, + kNumberAnyFlag + }; + RAPIDJSON_ASSERT(type <= kNumberType); + data_.f.flags = defaultFlags[type]; + + // Use ShortString to store empty string. + if (type == kStringType) + data_.ss.SetLength(0); + } + + //! Explicit copy constructor (with allocator) + /*! Creates a copy of a Value by using the given Allocator + \tparam SourceAllocator allocator of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \see CopyFrom() + */ + template< typename SourceAllocator > + GenericValue(const GenericValue& rhs, Allocator & allocator); + + //! Constructor for boolean value. + /*! \param b Boolean value + \note This constructor is limited to \em real boolean values and rejects + implicitly converted types like arbitrary pointers. Use an explicit cast + to \c bool, if you want to construct a boolean JSON value in such cases. + */ +#ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen + template + explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 +#else + explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT +#endif + : data_() { + // safe-guard against failing SFINAE + RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); + data_.f.flags = b ? kTrueFlag : kFalseFlag; + } + + //! Constructor for int value. + explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { + data_.n.i64 = i; + data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; + } + + //! Constructor for unsigned value. + explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { + data_.n.u64 = u; + data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); + } + + //! Constructor for int64_t value. + explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { + data_.n.i64 = i64; + data_.f.flags = kNumberInt64Flag; + if (i64 >= 0) { + data_.f.flags |= kNumberUint64Flag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for uint64_t value. + explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { + data_.n.u64 = u64; + data_.f.flags = kNumberUint64Flag; + if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) + data_.f.flags |= kInt64Flag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for double value. + explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } + + //! Constructor for constant string (i.e. do not make a copy of string) + GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } + + //! Constructor for constant string (i.e. do not make a copy of string) + explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } + +#if RAPIDJSON_HAS_STDSTRING + //! Constructor for copy-string from a string object (i.e. do make a copy of string) + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } +#endif + + //! Constructor for Array. + /*! + \param a An array obtained by \c GetArray(). + \note \c Array is always pass-by-value. + \note the source array is moved into this value and the sourec array becomes empty. + */ + GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { + a.value_.data_ = Data(); + a.value_.data_.f.flags = kArrayFlag; + } + + //! Constructor for Object. + /*! + \param o An object obtained by \c GetObject(). + \note \c Object is always pass-by-value. + \note the source object is moved into this value and the sourec object becomes empty. + */ + GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { + o.value_.data_ = Data(); + o.value_.data_.f.flags = kObjectFlag; + } + + //! Destructor. + /*! Need to destruct elements of array, members of object, or copy-string. + */ + ~GenericValue() { + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + switch(data_.f.flags) { + case kArrayFlag: + { + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(e); + } + break; + + case kObjectFlag: + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(GetMembersPointer()); + break; + + case kCopyStringFlag: + Allocator::Free(const_cast(GetStringPointer())); + break; + + default: + break; // Do nothing for other types. + } + } + } + + //@} + + //!@name Assignment operators + //@{ + + //! Assignment with move semantics. + /*! \param rhs Source of the assignment. It will become a null value after assignment. + */ + GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { + RAPIDJSON_ASSERT(this != &rhs); + this->~GenericValue(); + RawAssign(rhs); + return *this; + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT { + return *this = rhs.Move(); + } +#endif + + //! Assignment of constant string reference (no copy) + /*! \param str Constant string reference to be assigned + \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. + \see GenericStringRef, operator=(T) + */ + GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT { + GenericValue s(str); + return *this = s; + } + + //! Assignment with primitive types. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value The value to be assigned. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref SetString(const Ch*, Allocator&) (for copying) or + \ref StringRef() (to explicitly mark the pointer as constant) instead. + All other pointer types would implicitly convert to \c bool, + use \ref SetBool() instead. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) + operator=(T value) { + GenericValue v(value); + return *this = v; + } + + //! Deep-copy assignment from Value + /*! Assigns a \b copy of the Value to the current Value object + \tparam SourceAllocator Allocator type of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator to use for copying + */ + template + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { + RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); + this->~GenericValue(); + new (this) GenericValue(rhs, allocator); + return *this; + } + + //! Exchange the contents of this value with those of other. + /*! + \param other Another value. + \note Constant complexity. + */ + GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT { + GenericValue temp; + temp.RawAssign(*this); + RawAssign(other); + other.RawAssign(temp); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.value, b.value); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Prepare Value for move semantics + /*! \return *this */ + GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } + //@} + + //!@name Equal-to and not-equal-to operators + //@{ + //! Equal-to operator + /*! + \note If an object contains duplicated named member, comparing equality with any object is always \c false. + \note Linear time complexity (number of all values in the subtree and total lengths of all strings). + */ + template + bool operator==(const GenericValue& rhs) const { + typedef GenericValue RhsType; + if (GetType() != rhs.GetType()) + return false; + + switch (GetType()) { + case kObjectType: // Warning: O(n^2) inner-loop + if (data_.o.size != rhs.data_.o.size) + return false; + for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { + typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); + if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) + return false; + } + return true; + + case kArrayType: + if (data_.a.size != rhs.data_.a.size) + return false; + for (SizeType i = 0; i < data_.a.size; i++) + if ((*this)[i] != rhs[i]) + return false; + return true; + + case kStringType: + return StringEqual(rhs); + + case kNumberType: + if (IsDouble() || rhs.IsDouble()) { + double a = GetDouble(); // May convert from integer to double. + double b = rhs.GetDouble(); // Ditto + return a >= b && a <= b; // Prevent -Wfloat-equal + } + else + return data_.n.u64 == rhs.data_.n.u64; + + default: + return true; + } + } + + //! Equal-to operator with const C-string pointer + bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } + +#if RAPIDJSON_HAS_STDSTRING + //! Equal-to operator with string object + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } +#endif + + //! Equal-to operator with primitive types + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } + + //! Not-equal-to operator + /*! \return !(*this == rhs) + */ + template + bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with const C-string pointer + bool operator!=(const Ch* rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with arbitrary types + /*! \return !(*this == rhs) + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } + + //! Equal-to operator with arbitrary types (symmetric version) + /*! \return (rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } + + //! Not-Equal-to operator with arbitrary types (symmetric version) + /*! \return !(rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } + //@} + + //!@name Type + //@{ + + Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } + bool IsNull() const { return data_.f.flags == kNullFlag; } + bool IsFalse() const { return data_.f.flags == kFalseFlag; } + bool IsTrue() const { return data_.f.flags == kTrueFlag; } + bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } + bool IsObject() const { return data_.f.flags == kObjectFlag; } + bool IsArray() const { return data_.f.flags == kArrayFlag; } + bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } + bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } + bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } + bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } + bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } + bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } + bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } + + // Checks whether a number can be losslessly converted to a double. + bool IsLosslessDouble() const { + if (!IsNumber()) return false; + if (IsUint64()) { + uint64_t u = GetUint64(); + volatile double d = static_cast(u); + return (d >= 0.0) + && (d < static_cast(std::numeric_limits::max())) + && (u == static_cast(d)); + } + if (IsInt64()) { + int64_t i = GetInt64(); + volatile double d = static_cast(i); + return (d >= static_cast(std::numeric_limits::min())) + && (d < static_cast(std::numeric_limits::max())) + && (i == static_cast(d)); + } + return true; // double, int, uint are always lossless + } + + // Checks whether a number is a float (possible lossy). + bool IsFloat() const { + if ((data_.f.flags & kDoubleFlag) == 0) + return false; + double d = GetDouble(); + return d >= -3.4028234e38 && d <= 3.4028234e38; + } + // Checks whether a number can be losslessly converted to a float. + bool IsLosslessFloat() const { + if (!IsNumber()) return false; + double a = GetDouble(); + if (a < static_cast(-std::numeric_limits::max()) + || a > static_cast(std::numeric_limits::max())) + return false; + double b = static_cast(static_cast(a)); + return a >= b && a <= b; // Prevent -Wfloat-equal + } + + //@} + + //!@name Null + //@{ + + GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } + + //@} + + //!@name Bool + //@{ + + bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } + //!< Set boolean value + /*! \post IsBool() == true */ + GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } + + //@} + + //!@name Object + //@{ + + //! Set this value as an empty object. + /*! \post IsObject() == true */ + GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } + + //! Get the number of members in the object. + SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } + + //! Check whether the object is empty. + bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) + \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. + Since 0.2, if the name is not correct, it will assert. + If user is unsure whether a member exists, user should use HasMember() first. + A better approach is to use FindMember(). + \note Linear time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) { + GenericValue n(StringRef(name)); + return (*this)[n]; + } + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam SourceAllocator Allocator of the \c name value + + \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). + And it can also handle strings with embedded null characters. + + \note Linear time complexity. + */ + template + GenericValue& operator[](const GenericValue& name) { + MemberIterator member = FindMember(name); + if (member != MemberEnd()) + return member->value; + else { + RAPIDJSON_ASSERT(false); // see above note + + // This will generate -Wexit-time-destructors in clang + // static GenericValue NullValue; + // return NullValue; + + // Use static buffer and placement-new to prevent destruction + static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); + } + } + template + const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } + +#if RAPIDJSON_HAS_STDSTRING + //! Get a value from an object associated with name (string object). + GenericValue& operator[](const std::basic_string& name) { return (*this)[GenericValue(StringRef(name))]; } + const GenericValue& operator[](const std::basic_string& name) const { return (*this)[GenericValue(StringRef(name))]; } +#endif + + //! Const member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } + //! Const \em past-the-end member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } + //! Member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } + //! \em Past-the-end member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } + + //! Check whether a member exists in the object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } + +#if RAPIDJSON_HAS_STDSTRING + //! Check whether a member exists in the object with string object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const std::basic_string& name) const { return FindMember(name) != MemberEnd(); } +#endif + + //! Check whether a member exists in the object with GenericValue name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + template + bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } + + //! Find member by name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + MemberIterator FindMember(const Ch* name) { + GenericValue n(StringRef(name)); + return FindMember(n); + } + + ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } + + //! Find member by name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + template + MemberIterator FindMember(const GenericValue& name) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; + } + template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } + +#if RAPIDJSON_HAS_STDSTRING + //! Find member by string object name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + */ + MemberIterator FindMember(const std::basic_string& name) { return FindMember(StringRef(name)); } + ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(StringRef(name)); } +#endif + + //! Add a member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c name and \c value will be transferred to this object on success. + \pre IsObject() && name.IsString() + \post name.IsNull() && value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + + ObjectData& o = data_.o; + if (o.size >= o.capacity) { + if (o.capacity == 0) { + o.capacity = kDefaultObjectCapacity; + SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); + } + else { + SizeType oldCapacity = o.capacity; + o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); + } + } + Member* members = GetMembersPointer(); + members[o.size].name.RawAssign(name); + members[o.size].value.RawAssign(value); + o.size++; + return *this; + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, StringRefType value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Add a string object as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, std::basic_string& value, Allocator& allocator) { + GenericValue v(value, allocator); + return AddMember(name, v, allocator); + } +#endif + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A string value as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(GenericValue& name, T value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + + + //! Add a member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this object on success. + \pre IsObject() + \post value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A constant string reference as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(StringRefType name, T value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Remove all members in the object. + /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void RemoveAllMembers() { + RAPIDJSON_ASSERT(IsObject()); + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; + } + + //! Remove a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Linear time complexity. + */ + bool RemoveMember(const Ch* name) { + GenericValue n(StringRef(name)); + return RemoveMember(n); + } + +#if RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) { return RemoveMember(GenericValue(StringRef(name))); } +#endif + + template + bool RemoveMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + RemoveMember(m); + return true; + } + else + return false; + } + + //! Remove a member in object by iterator. + /*! \param m member iterator (obtained by FindMember() or MemberBegin()). + \return the new iterator after removal. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Constant time complexity. + */ + MemberIterator RemoveMember(MemberIterator m) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); + + MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) + *m = *last; // Move the last one to this place + else + m->~Member(); // Only one left, just destroy + --data_.o.size; + return m; + } + + //! Remove a member from an object by iterator. + /*! \param pos iterator to the member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() + \return Iterator following the removed element. + If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. + \note This function preserves the relative order of the remaining object + members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator pos) { + return EraseMember(pos, pos +1); + } + + //! Remove members in the range [first, last) from an object. + /*! \param first iterator to the first member to remove + \param last iterator following the last member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() + \return Iterator following the last removed element. + \note This function preserves the relative order of the remaining object + members. + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(first >= MemberBegin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= MemberEnd()); + + MemberIterator pos = MemberBegin() + (first - MemberBegin()); + for (MemberIterator itr = pos; itr != last; ++itr) + itr->~Member(); + std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); + return pos; + } + + //! Erase a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note Linear time complexity. + */ + bool EraseMember(const Ch* name) { + GenericValue n(StringRef(name)); + return EraseMember(n); + } + +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } +#endif + + template + bool EraseMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + EraseMember(m); + return true; + } + else + return false; + } + + Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + + //@} + + //!@name Array + //@{ + + //! Set this value as an empty array. + /*! \post IsArray == true */ + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + + //! Get the number of elements in array. + SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } + + //! Get the capacity of array. + SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } + + //! Check whether the array is empty. + bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } + + //! Remove all elements in the array. + /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void Clear() { + RAPIDJSON_ASSERT(IsArray()); + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + data_.a.size = 0; + } + + //! Get an element from array by index. + /*! \pre IsArray() == true + \param index Zero-based index of element. + \see operator[](T*) + */ + GenericValue& operator[](SizeType index) { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(index < data_.a.size); + return GetElementsPointer()[index]; + } + const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } + + //! Element iterator + /*! \pre IsArray() == true */ + ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } + //! \em Past-the-end element iterator + /*! \pre IsArray() == true */ + ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } + //! Constant element iterator + /*! \pre IsArray() == true */ + ConstValueIterator Begin() const { return const_cast(*this).Begin(); } + //! Constant \em past-the-end element iterator + /*! \pre IsArray() == true */ + ConstValueIterator End() const { return const_cast(*this).End(); } + + //! Request the array to have enough capacity to store elements. + /*! \param newCapacity The capacity that the array at least need to have. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note Linear time complexity. + */ + GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { + RAPIDJSON_ASSERT(IsArray()); + if (newCapacity > data_.a.capacity) { + SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); + data_.a.capacity = newCapacity; + } + return *this; + } + + //! Append a GenericValue at the end of the array. + /*! \param value Value to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \post value.IsNull() == true + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this array on success. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + */ + GenericValue& PushBack(GenericValue& value, Allocator& allocator) { + RAPIDJSON_ASSERT(IsArray()); + if (data_.a.size >= data_.a.capacity) + Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); + GetElementsPointer()[data_.a.size++].RawAssign(value); + return *this; + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { + return PushBack(value, allocator); + } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + + //! Append a constant string reference at the end of the array. + /*! \param value Constant string reference to be appended. + \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + \see GenericStringRef + */ + GenericValue& PushBack(StringRefType value, Allocator& allocator) { + return (*this).template PushBack(value, allocator); + } + + //! Append a primitive value at the end of the array. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value Value of primitive type T to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref PushBack(GenericValue&, Allocator&) or \ref + PushBack(StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + PushBack(T value, Allocator& allocator) { + GenericValue v(value); + return PushBack(v, allocator); + } + + //! Remove the last element in the array. + /*! + \note Constant time complexity. + */ + GenericValue& PopBack() { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(!Empty()); + GetElementsPointer()[--data_.a.size].~GenericValue(); + return *this; + } + + //! Remove an element of array by iterator. + /*! + \param pos iterator to the element to remove + \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() + \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator pos) { + return Erase(pos, pos + 1); + } + + //! Remove elements in the range [first, last) of the array. + /*! + \param first iterator to the first element to remove + \param last iterator following the last element to remove + \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() + \return Iterator following the last removed element. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(data_.a.size > 0); + RAPIDJSON_ASSERT(GetElementsPointer() != 0); + RAPIDJSON_ASSERT(first >= Begin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= End()); + ValueIterator pos = Begin() + (first - Begin()); + for (ValueIterator itr = pos; itr != last; ++itr) + itr->~GenericValue(); + std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); + data_.a.size -= static_cast(last - first); + return pos; + } + + Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } + ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } + + //@} + + //!@name Number + //@{ + + int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } + + //! Get the value as double type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. + */ + double GetDouble() const { + RAPIDJSON_ASSERT(IsNumber()); + if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) + RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) + } + + //! Get the value as float type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. + */ + float GetFloat() const { + return static_cast(GetDouble()); + } + + GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } + GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } + GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } + GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } + GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } + GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(f); return *this; } + + //@} + + //!@name String + //@{ + + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } + + //! Get the length of string. + /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). + */ + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + + //! Set this value as a string without copying source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string pointer. + \param length The length of source string, excluding the trailing null terminator. + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == length + \see SetString(StringRefType) + */ + GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } + + //! Set this value as a string without copying source string. + /*! \param s source string reference + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == s.length + */ + GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } + + //! Set this value as a string by copying from source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string. + \param length The length of source string, excluding the trailing null terminator. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } + + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } + +#if RAPIDJSON_HAS_STDSTRING + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } +#endif + + //@} + + //!@name Array + //@{ + + //! Templated version for checking whether this value is type T. + /*! + \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string + */ + template + bool Is() const { return internal::TypeHelper::Is(*this); } + + template + T Get() const { return internal::TypeHelper::Get(*this); } + + template + T Get() { return internal::TypeHelper::Get(*this); } + + template + ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } + + template + ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } + + //@} + + //! Generate events of this value to a Handler. + /*! This function adopts the GoF visitor pattern. + Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. + It can also be used to deep clone this value via GenericDocument, which is also a Handler. + \tparam Handler type of handler. + \param handler An object implementing concept Handler. + */ + template + bool Accept(Handler& handler) const { + switch(GetType()) { + case kNullType: return handler.Null(); + case kFalseType: return handler.Bool(false); + case kTrueType: return handler.Bool(true); + + case kObjectType: + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + return false; + for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { + RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. + if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) + return false; + if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) + return false; + } + return handler.EndObject(data_.o.size); + + case kArrayType: + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + return false; + for (const GenericValue* v = Begin(); v != End(); ++v) + if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) + return false; + return handler.EndArray(data_.a.size); + + case kStringType: + return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); + + default: + RAPIDJSON_ASSERT(GetType() == kNumberType); + if (IsDouble()) return handler.Double(data_.n.d); + else if (IsInt()) return handler.Int(data_.n.i.i); + else if (IsUint()) return handler.Uint(data_.n.u.u); + else if (IsInt64()) return handler.Int64(data_.n.i64); + else return handler.Uint64(data_.n.u64); + } + } + +private: + template friend class GenericValue; + template friend class GenericDocument; + + enum { + kBoolFlag = 0x0008, + kNumberFlag = 0x0010, + kIntFlag = 0x0020, + kUintFlag = 0x0040, + kInt64Flag = 0x0080, + kUint64Flag = 0x0100, + kDoubleFlag = 0x0200, + kStringFlag = 0x0400, + kCopyFlag = 0x0800, + kInlineStrFlag = 0x1000, + + // Initial flags of different types. + kNullFlag = kNullType, + kTrueFlag = kTrueType | kBoolFlag, + kFalseFlag = kFalseType | kBoolFlag, + kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, + kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, + kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, + kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, + kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, + kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, + kConstStringFlag = kStringType | kStringFlag, + kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, + kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, + kObjectFlag = kObjectType, + kArrayFlag = kArrayType, + + kTypeMask = 0x07 + }; + + static const SizeType kDefaultArrayCapacity = 16; + static const SizeType kDefaultObjectCapacity = 16; + + struct Flag { +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION + char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer +#elif RAPIDJSON_64BIT + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes +#else + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes +#endif + uint16_t flags; + }; + + struct String { + SizeType length; + SizeType hashcode; //!< reserved + const Ch* str; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars + // (excluding the terminating zero) and store a value to determine the length of the contained + // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string + // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as + // the string terminator as well. For getting the string length back from that value just use + // "MaxSize - str[LenPos]". + // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, + // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). + struct ShortString { + enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + Ch str[MaxChars]; + + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } + inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } + }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // By using proper binary layout, retrieval of different integer types do not need conversions. + union Number { +#if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN + struct I { + int i; + char padding[4]; + }i; + struct U { + unsigned u; + char padding2[4]; + }u; +#else + struct I { + char padding[4]; + int i; + }i; + struct U { + char padding2[4]; + unsigned u; + }u; +#endif + int64_t i64; + uint64_t u64; + double d; + }; // 8 bytes + + struct ObjectData { + SizeType size; + SizeType capacity; + Member* members; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + struct ArrayData { + SizeType size; + SizeType capacity; + GenericValue* elements; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + union Data { + String s; + ShortString ss; + Number n; + ObjectData o; + ArrayData a; + Flag f; + }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION + + RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } + RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } + RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } + RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } + RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } + RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } + + // Initialize this value as array with initial data, without calling destructor. + void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { + data_.f.flags = kArrayFlag; + if (count) { + GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); + SetElementsPointer(e); + std::memcpy(e, values, count * sizeof(GenericValue)); + } + else + SetElementsPointer(0); + data_.a.size = data_.a.capacity = count; + } + + //! Initialize this value as object with initial data, without calling destructor. + void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { + data_.f.flags = kObjectFlag; + if (count) { + Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + SetMembersPointer(m); + std::memcpy(m, members, count * sizeof(Member)); + } + else + SetMembersPointer(0); + data_.o.size = data_.o.capacity = count; + } + + //! Initialize this value as constant string, without calling destructor. + void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { + data_.f.flags = kConstStringFlag; + SetStringPointer(s); + data_.s.length = s.length; + } + + //! Initialize this value as copy string with initial data, without calling destructor. + void SetStringRaw(StringRefType s, Allocator& allocator) { + Ch* str = 0; + if (ShortString::Usable(s.length)) { + data_.f.flags = kShortStringFlag; + data_.ss.SetLength(s.length); + str = data_.ss.str; + } else { + data_.f.flags = kCopyStringFlag; + data_.s.length = s.length; + str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); + SetStringPointer(str); + } + std::memcpy(str, s, s.length * sizeof(Ch)); + str[s.length] = '\0'; + } + + //! Assignment without calling destructor + void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { + data_ = rhs.data_; + // data_.f.flags = rhs.data_.f.flags; + rhs.data_.f.flags = kNullFlag; + } + + template + bool StringEqual(const GenericValue& rhs) const { + RAPIDJSON_ASSERT(IsString()); + RAPIDJSON_ASSERT(rhs.IsString()); + + const SizeType len1 = GetStringLength(); + const SizeType len2 = rhs.GetStringLength(); + if(len1 != len2) { return false; } + + const Ch* const str1 = GetString(); + const Ch* const str2 = rhs.GetString(); + if(str1 == str2) { return true; } // fast path for constant string + + return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); + } + + Data data_; +}; + +//! GenericValue with UTF8 encoding +typedef GenericValue > Value; + +/////////////////////////////////////////////////////////////////////////////// +// GenericDocument + +//! A document for parsing JSON text as DOM. +/*! + \note implements Handler concept + \tparam Encoding Encoding for both parsing and string storage. + \tparam Allocator Allocator for allocating memory for the DOM + \tparam StackAllocator Allocator for allocating memory for stack during parsing. + \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. +*/ +template , typename StackAllocator = CrtAllocator> +class GenericDocument : public GenericValue { +public: + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericValue ValueType; //!< Value type of the document. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + + //! Constructor + /*! Creates an empty document of specified type. + \param type Mandatory type of object to create. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + } + + //! Constructor + /*! Creates an empty document which type is Null. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT + : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(std::move(rhs.stack_)), + parseResult_(rhs.parseResult_) + { + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + } +#endif + + ~GenericDocument() { + Destroy(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT + { + // The cast to ValueType is necessary here, because otherwise it would + // attempt to call GenericValue's templated assignment operator. + ValueType::operator=(std::forward(rhs)); + + // Calling the destructor here would prematurely call stack_'s destructor + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = std::move(rhs.stack_); + parseResult_ = rhs.parseResult_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + + return *this; + } +#endif + + //! Exchange the contents of this document with those of another. + /*! + \param rhs Another document. + \note Constant complexity. + \see GenericValue::Swap + */ + GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { + ValueType::Swap(rhs); + stack_.Swap(rhs.stack_); + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(parseResult_, rhs.parseResult_); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.doc, b.doc); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Populate this document by a generator which produces SAX events. + /*! \tparam Generator A functor with bool f(Handler) prototype. + \param g Generator functor which sends SAX events to the parameter. + \return The document itself for fluent API. + */ + template + GenericDocument& Populate(Generator& g) { + ClearStackOnExit scope(*this); + if (g(*this)) { + RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //!@name Parse from stream + //!@{ + + //! Parse JSON text from an input stream (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam SourceEncoding Encoding of input stream + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + GenericReader reader( + stack_.HasAllocator() ? &stack_.GetAllocator() : 0); + ClearStackOnExit scope(*this); + parseResult_ = reader.template Parse(is, *this); + if (parseResult_) { + RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //! Parse JSON text from an input stream + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + + //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + //!@} + + //!@name Parse in-place from mutable string + //!@{ + + //! Parse JSON text from a mutable string + /*! \tparam parseFlags Combination of \ref ParseFlag. + \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseInsitu(Ch* str) { + GenericInsituStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) + /*! \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + GenericDocument& ParseInsitu(Ch* str) { + return ParseInsitu(str); + } + //!@} + + //!@name Parse from read-only string + //!@{ + + //! Parse JSON text from a read-only string (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \tparam SourceEncoding Transcoding from input Encoding + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str) { + RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + GenericStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a read-only string + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + + //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) + /*! \param str Read-only zero-terminated string to be parsed. + */ + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { + RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + MemoryStream ms(static_cast(str), length * sizeof(typename SourceEncoding::Ch)); + EncodedInputStream is(ms); + ParseStream(is); + return *this; + } + + template + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + template + GenericDocument& Parse(const std::basic_string& str) { + // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) + return Parse(str.c_str()); + } + + template + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str.c_str()); + } + + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str); + } +#endif // RAPIDJSON_HAS_STDSTRING + + //!@} + + //!@name Handling parse errors + //!@{ + + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseError() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + + //! Implicit conversion to get the last parse result +#ifndef __clang // -Wdocumentation + /*! \return \ref ParseResult of the last parse operation + + \code + Document doc; + ParseResult ok = doc.Parse(json); + if (!ok) + printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); + \endcode + */ +#endif + operator ParseResult() const { return parseResult_; } + //!@} + + //! Get the allocator of this document. + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } + + //! Get the capacity of stack in bytes. + size_t GetStackCapacity() const { return stack_.GetCapacity(); } + +private: + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} + ~ClearStackOnExit() { d_.ClearStack(); } + private: + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + GenericDocument& d_; + }; + + // callers of the following private Handler functions + // template friend class GenericReader; // for parsing + template friend class GenericValue; // for deep copying + +public: + // Implementation of Handler + bool Null() { new (stack_.template Push()) ValueType(); return true; } + bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } + bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } + bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + + bool RawNumber(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool String(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } + + bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount) { + typename ValueType::Member* members = stack_.template Pop(memberCount); + stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); + return true; + } + + bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } + + bool EndArray(SizeType elementCount) { + ValueType* elements = stack_.template Pop(elementCount); + stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); + return true; + } + +private: + //! Prohibit copying + GenericDocument(const GenericDocument&); + //! Prohibit assignment + GenericDocument& operator=(const GenericDocument&); + + void ClearStack() { + if (Allocator::kNeedFree) + while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) + (stack_.template Pop(1))->~ValueType(); + else + stack_.Clear(); + stack_.ShrinkToFit(); + } + + void Destroy() { + RAPIDJSON_DELETE(ownAllocator_); + } + + static const size_t kDefaultStackCapacity = 1024; + Allocator* allocator_; + Allocator* ownAllocator_; + internal::Stack stack_; + ParseResult parseResult_; +}; + +//! GenericDocument with UTF8 encoding +typedef GenericDocument > Document; + +// defined here due to the dependency on GenericDocument +template +template +inline +GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) +{ + switch (rhs.GetType()) { + case kObjectType: + case kArrayType: { // perform deep copy via SAX Handler + GenericDocument d(&allocator); + rhs.Accept(d); + RawAssign(*d.stack_.template Pop(1)); + } + break; + case kStringType: + if (rhs.data_.f.flags == kConstStringFlag) { + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + } else { + SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + } + break; + default: + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + break; + } +} + +//! Helper class for accessing Value of array type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetArray(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericArray { +public: + typedef GenericArray ConstArray; + typedef GenericArray Array; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef ValueType* ValueIterator; // This may be const or non-const iterator + typedef const ValueT* ConstValueIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + + template + friend class GenericValue; + + GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} + GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } + ~GenericArray() {} + + SizeType Size() const { return value_.Size(); } + SizeType Capacity() const { return value_.Capacity(); } + bool Empty() const { return value_.Empty(); } + void Clear() const { value_.Clear(); } + ValueType& operator[](SizeType index) const { return value_[index]; } + ValueIterator Begin() const { return value_.Begin(); } + ValueIterator End() const { return value_.End(); } + GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } + GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + GenericArray PopBack() const { value_.PopBack(); return *this; } + ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + ValueIterator begin() const { return value_.Begin(); } + ValueIterator end() const { return value_.End(); } +#endif + +private: + GenericArray(); + GenericArray(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +//! Helper class for accessing Value of object type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetObject(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericObject { +public: + typedef GenericObject ConstObject; + typedef GenericObject Object; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator + typedef GenericMemberIterator ConstMemberIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename ValueType::Ch Ch; + + template + friend class GenericValue; + + GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} + GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } + ~GenericObject() {} + + SizeType MemberCount() const { return value_.MemberCount(); } + bool ObjectEmpty() const { return value_.ObjectEmpty(); } + template ValueType& operator[](T* name) const { return value_[name]; } + template ValueType& operator[](const GenericValue& name) const { return value_[name]; } +#if RAPIDJSON_HAS_STDSTRING + ValueType& operator[](const std::basic_string& name) const { return value_[name]; } +#endif + MemberIterator MemberBegin() const { return value_.MemberBegin(); } + MemberIterator MemberEnd() const { return value_.MemberEnd(); } + bool HasMember(const Ch* name) const { return value_.HasMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } +#endif + template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } + MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } + template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } +#if RAPIDJSON_HAS_STDSTRING + MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } +#endif + GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_STDSTRING + GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + void RemoveAllMembers() { return value_.RemoveAllMembers(); } + bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } +#endif + template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } + MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } + MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } + bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } +#endif + template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + MemberIterator begin() const { return value_.MemberBegin(); } + MemberIterator end() const { return value_.MemberEnd(); } +#endif + +private: + GenericObject(); + GenericObject(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + +#endif // RAPIDJSON_DOCUMENT_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/encodedstream.h b/ext/librethinkdbxx/src/rapidjson/encodedstream.h new file mode 100644 index 00000000..14506838 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/encodedstream.h @@ -0,0 +1,299 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ENCODEDSTREAM_H_ +#define RAPIDJSON_ENCODEDSTREAM_H_ + +#include "stream.h" +#include "memorystream.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Input byte stream wrapper with a statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam InputByteStream Type of input byte stream. For example, FileReadStream. +*/ +template +class EncodedInputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedInputStream(InputByteStream& is) : is_(is) { + current_ = Encoding::TakeBOM(is_); + } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); + + InputByteStream& is_; + Ch current_; +}; + +//! Specialized for UTF8 MemoryStream. +template <> +class EncodedInputStream, MemoryStream> { +public: + typedef UTF8<>::Ch Ch; + + EncodedInputStream(MemoryStream& is) : is_(is) { + if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); + } + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) {} + void Flush() {} + Ch* PutBegin() { return 0; } + size_t PutEnd(Ch*) { return 0; } + + MemoryStream& is_; + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); +}; + +//! Output byte stream wrapper with statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. +*/ +template +class EncodedOutputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { + if (putBOM) + Encoding::PutBOM(os_); + } + + void Put(Ch c) { Encoding::Put(os_, c); } + void Flush() { os_.Flush(); } + + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedOutputStream(const EncodedOutputStream&); + EncodedOutputStream& operator=(const EncodedOutputStream&); + + OutputByteStream& os_; +}; + +#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + +//! Input stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for reading. + \tparam InputByteStream type of input byte stream to be wrapped. +*/ +template +class AutoUTFInputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param is input stream to be wrapped. + \param type UTF encoding type if it is not detected from the stream. + */ + AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { + RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + DetectType(); + static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; + takeFunc_ = f[type_]; + current_ = takeFunc_(*is_); + } + + UTFType GetType() const { return type_; } + bool HasBOM() const { return hasBOM_; } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } + size_t Tell() const { return is_->Tell(); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFInputStream(const AutoUTFInputStream&); + AutoUTFInputStream& operator=(const AutoUTFInputStream&); + + // Detect encoding type with BOM or RFC 4627 + void DetectType() { + // BOM (Byte Order Mark): + // 00 00 FE FF UTF-32BE + // FF FE 00 00 UTF-32LE + // FE FF UTF-16BE + // FF FE UTF-16LE + // EF BB BF UTF-8 + + const unsigned char* c = reinterpret_cast(is_->Peek4()); + if (!c) + return; + + unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); + hasBOM_ = false; + if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } + + // RFC 4627: Section 3 + // "Since the first two characters of a JSON text will always be ASCII + // characters [RFC0020], it is possible to determine whether an octet + // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking + // at the pattern of nulls in the first four octets." + // 00 00 00 xx UTF-32BE + // 00 xx 00 xx UTF-16BE + // xx 00 00 00 UTF-32LE + // xx 00 xx 00 UTF-16LE + // xx xx xx xx UTF-8 + + if (!hasBOM_) { + unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + switch (pattern) { + case 0x08: type_ = kUTF32BE; break; + case 0x0A: type_ = kUTF16BE; break; + case 0x01: type_ = kUTF32LE; break; + case 0x05: type_ = kUTF16LE; break; + case 0x0F: type_ = kUTF8; break; + default: break; // Use type defined by user. + } + } + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + } + + typedef Ch (*TakeFunc)(InputByteStream& is); + InputByteStream* is_; + UTFType type_; + Ch current_; + TakeFunc takeFunc_; + bool hasBOM_; +}; + +//! Output stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for writing. + \tparam OutputByteStream type of output byte stream to be wrapped. +*/ +template +class AutoUTFOutputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param os output stream to be wrapped. + \param type UTF encoding type. + \param putBOM Whether to write BOM at the beginning of the stream. + */ + AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { + RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + + static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; + putFunc_ = f[type_]; + + if (putBOM) + PutBOM(); + } + + UTFType GetType() const { return type_; } + + void Put(Ch c) { putFunc_(*os_, c); } + void Flush() { os_->Flush(); } + + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFOutputStream(const AutoUTFOutputStream&); + AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); + + void PutBOM() { + typedef void (*PutBOMFunc)(OutputByteStream&); + static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; + f[type_](*os_); + } + + typedef void (*PutFunc)(OutputByteStream&, Ch); + + OutputByteStream* os_; + UTFType type_; + PutFunc putFunc_; +}; + +#undef RAPIDJSON_ENCODINGS_FUNC + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/encodings.h b/ext/librethinkdbxx/src/rapidjson/encodings.h new file mode 100644 index 00000000..baa7c2b1 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/encodings.h @@ -0,0 +1,716 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ENCODINGS_H_ +#define RAPIDJSON_ENCODINGS_H_ + +#include "rapidjson.h" + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data +RAPIDJSON_DIAG_OFF(4702) // unreachable code +#elif defined(__GNUC__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +RAPIDJSON_DIAG_OFF(overflow) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Encoding + +/*! \class rapidjson::Encoding + \brief Concept for encoding of Unicode characters. + +\code +concept Encoding { + typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. + + enum { supportUnicode = 1 }; // or 0 if not supporting unicode + + //! \brief Encode a Unicode codepoint to an output stream. + //! \param os Output stream. + //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. + template + static void Encode(OutputStream& os, unsigned codepoint); + + //! \brief Decode a Unicode codepoint from an input stream. + //! \param is Input stream. + //! \param codepoint Output of the unicode codepoint. + //! \return true if a valid codepoint can be decoded from the stream. + template + static bool Decode(InputStream& is, unsigned* codepoint); + + //! \brief Validate one Unicode codepoint from an encoded stream. + //! \param is Input stream to obtain codepoint. + //! \param os Output for copying one codepoint. + //! \return true if it is valid. + //! \note This function just validating and copying the codepoint without actually decode it. + template + static bool Validate(InputStream& is, OutputStream& os); + + // The following functions are deal with byte streams. + + //! Take a character from input byte stream, skip BOM if exist. + template + static CharType TakeBOM(InputByteStream& is); + + //! Take a character from input byte stream. + template + static Ch Take(InputByteStream& is); + + //! Put BOM to output byte stream. + template + static void PutBOM(OutputByteStream& os); + + //! Put a character to output byte stream. + template + static void Put(OutputByteStream& os, Ch c); +}; +\endcode +*/ + +/////////////////////////////////////////////////////////////////////////////// +// UTF8 + +//! UTF-8 encoding. +/*! http://en.wikipedia.org/wiki/UTF-8 + http://tools.ietf.org/html/rfc3629 + \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. + \note implements Encoding concept +*/ +template +struct UTF8 { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + os.Put(static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + PutUnsafe(os, static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { +#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + typename InputStream::Ch c = is.Take(); + if (!(c & 0x80)) { + *codepoint = static_cast(c); + return true; + } + + unsigned char type = GetRange(static_cast(c)); + if (type >= 32) { + *codepoint = 0; + } else { + *codepoint = (0xFF >> type) & static_cast(c); + } + bool result = true; + switch (type) { + case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } +#undef COPY +#undef TRANS +#undef TAIL + } + + template + static bool Validate(InputStream& is, OutputStream& os) { +#define COPY() os.Put(c = is.Take()) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + Ch c; + COPY(); + if (!(c & 0x80)) + return true; + + bool result = true; + switch (GetRange(static_cast(c))) { + case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } +#undef COPY +#undef TRANS +#undef TAIL + } + + static unsigned char GetRange(unsigned char c) { + // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. + static const unsigned char type[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, + 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + }; + return type[c]; + } + + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + typename InputByteStream::Ch c = Take(is); + if (static_cast(c) != 0xEFu) return c; + c = is.Take(); + if (static_cast(c) != 0xBBu) return c; + c = is.Take(); + if (static_cast(c) != 0xBFu) return c; + c = is.Take(); + return c; + } + + template + static Ch Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xEFu)); + os.Put(static_cast(0xBBu)); + os.Put(static_cast(0xBFu)); + } + + template + static void Put(OutputByteStream& os, Ch c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF16 + +//! UTF-16 encoding. +/*! http://en.wikipedia.org/wiki/UTF-16 + http://tools.ietf.org/html/rfc2781 + \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF16LE and UTF16BE, which handle endianness. +*/ +template +struct UTF16 { + typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + os.Put(static_cast(codepoint)); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + os.Put(static_cast((v >> 10) | 0xD800)); + os.Put((v & 0x3FF) | 0xDC00); + } + } + + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + PutUnsafe(os, static_cast(codepoint)); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + PutUnsafe(os, static_cast((v >> 10) | 0xD800)); + PutUnsafe(os, (v & 0x3FF) | 0xDC00); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + typename InputStream::Ch c = is.Take(); + if (c < 0xD800 || c > 0xDFFF) { + *codepoint = static_cast(c); + return true; + } + else if (c <= 0xDBFF) { + *codepoint = (static_cast(c) & 0x3FF) << 10; + c = is.Take(); + *codepoint |= (static_cast(c) & 0x3FF); + *codepoint += 0x10000; + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + typename InputStream::Ch c; + os.Put(static_cast(c = is.Take())); + if (c < 0xD800 || c > 0xDFFF) + return true; + else if (c <= 0xDBFF) { + os.Put(c = is.Take()); + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } +}; + +//! UTF-16 little endian encoding. +template +struct UTF16LE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(static_cast(c) & 0xFFu)); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + } +}; + +//! UTF-16 big endian encoding. +template +struct UTF16BE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 8; + c |= static_cast(is.Take()); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + os.Put(static_cast(static_cast(c) & 0xFFu)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF32 + +//! UTF-32 encoding. +/*! http://en.wikipedia.org/wiki/UTF-32 + \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF32LE and UTF32BE, which handle endianness. +*/ +template +struct UTF32 { + typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(codepoint); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, codepoint); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c = is.Take(); + *codepoint = c; + return c <= 0x10FFFF; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c; + os.Put(c = is.Take()); + return c <= 0x10FFFF; + } +}; + +//! UTF-32 little endian enocoding. +template +struct UTF32LE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 24; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 24) & 0xFFu)); + } +}; + +//! UTF-32 big endian encoding. +template +struct UTF32BE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 24; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((c >> 24) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast(c & 0xFFu)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// ASCII + +//! ASCII encoding. +/*! http://en.wikipedia.org/wiki/ASCII + \tparam CharType Code unit for storing 7-bit ASCII data. Default is char. + \note implements Encoding concept +*/ +template +struct ASCII { + typedef CharType Ch; + + enum { supportUnicode = 0 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_ASSERT(codepoint <= 0x7F); + os.Put(static_cast(codepoint & 0xFF)); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_ASSERT(codepoint <= 0x7F); + PutUnsafe(os, static_cast(codepoint & 0xFF)); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + uint8_t c = static_cast(is.Take()); + *codepoint = c; + return c <= 0X7F; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + uint8_t c = static_cast(is.Take()); + os.Put(static_cast(c)); + return c <= 0x7F; + } + + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + uint8_t c = static_cast(Take(is)); + return static_cast(c); + } + + template + static Ch Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + (void)os; + } + + template + static void Put(OutputByteStream& os, Ch c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// AutoUTF + +//! Runtime-specified UTF encoding type of a stream. +enum UTFType { + kUTF8 = 0, //!< UTF-8. + kUTF16LE = 1, //!< UTF-16 little endian. + kUTF16BE = 2, //!< UTF-16 big endian. + kUTF32LE = 3, //!< UTF-32 little endian. + kUTF32BE = 4 //!< UTF-32 big endian. +}; + +//! Dynamically select encoding according to stream's runtime-specified UTF encoding type. +/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). +*/ +template +struct AutoUTF { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + +#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + + template + RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; + (*f[os.GetType()])(os, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; + (*f[os.GetType()])(os, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { + typedef bool (*DecodeFunc)(InputStream&, unsigned*); + static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; + return (*f[is.GetType()])(is, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + typedef bool (*ValidateFunc)(InputStream&, OutputStream&); + static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; + return (*f[is.GetType()])(is, os); + } + +#undef RAPIDJSON_ENCODINGS_FUNC +}; + +/////////////////////////////////////////////////////////////////////////////// +// Transcoder + +//! Encoding conversion. +template +struct Transcoder { + //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. + template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::Encode(os, codepoint); + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::EncodeUnsafe(os, codepoint); + return true; + } + + //! Validate one Unicode codepoint from an encoded stream. + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + return Transcode(is, os); // Since source/target encoding is different, must transcode. + } +}; + +// Forward declaration. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c); + +//! Specialization of Transcoder with same source and target encoding. +template +struct Transcoder { + template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + return Encoding::Validate(is, os); // source/target encoding are the same + } +}; + +RAPIDJSON_NAMESPACE_END + +#if defined(__GNUC__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/error/en.h b/ext/librethinkdbxx/src/rapidjson/error/en.h new file mode 100644 index 00000000..2db838bf --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/error/en.h @@ -0,0 +1,74 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ERROR_EN_H_ +#define RAPIDJSON_ERROR_EN_H_ + +#include "error.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(covered-switch-default) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Maps error code of parsing into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param parseErrorCode Error code obtained in parsing. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ +inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { + switch (parseErrorCode) { + case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); + case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); + + case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); + + case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); + case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); + case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); + + case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); + + case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); + case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); + case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); + case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); + case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); + + case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); + case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); + case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); + + case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); + case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); + + default: return RAPIDJSON_ERROR_STRING("Unknown error."); + } +} + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_EN_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/error/error.h b/ext/librethinkdbxx/src/rapidjson/error/error.h new file mode 100644 index 00000000..95cb31a7 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/error/error.h @@ -0,0 +1,155 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ERROR_ERROR_H_ +#define RAPIDJSON_ERROR_ERROR_H_ + +#include "../rapidjson.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +/*! \file error.h */ + +/*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ERROR_CHARTYPE + +//! Character type of error messages. +/*! \ingroup RAPIDJSON_ERRORS + The default character type is \c char. + On Windows, user can define this macro as \c TCHAR for supporting both + unicode/non-unicode settings. +*/ +#ifndef RAPIDJSON_ERROR_CHARTYPE +#define RAPIDJSON_ERROR_CHARTYPE char +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ERROR_STRING + +//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. +/*! \ingroup RAPIDJSON_ERRORS + By default this conversion macro does nothing. + On Windows, user can define this macro as \c _T(x) for supporting both + unicode/non-unicode settings. +*/ +#ifndef RAPIDJSON_ERROR_STRING +#define RAPIDJSON_ERROR_STRING(x) x +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// ParseErrorCode + +//! Error code of parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericReader::Parse, GenericReader::GetParseErrorCode +*/ +enum ParseErrorCode { + kParseErrorNone = 0, //!< No error. + + kParseErrorDocumentEmpty, //!< The document is empty. + kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. + + kParseErrorValueInvalid, //!< Invalid value. + + kParseErrorObjectMissName, //!< Missing a name for object member. + kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. + kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. + + kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. + + kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. + kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. + kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. + kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. + kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. + + kParseErrorNumberTooBig, //!< Number too big to be stored in double. + kParseErrorNumberMissFraction, //!< Miss fraction part in number. + kParseErrorNumberMissExponent, //!< Miss exponent in number. + + kParseErrorTermination, //!< Parsing was terminated. + kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. +}; + +//! Result of parsing (wraps ParseErrorCode) +/*! + \ingroup RAPIDJSON_ERRORS + \code + Document doc; + ParseResult ok = doc.Parse("[42]"); + if (!ok) { + fprintf(stderr, "JSON parse error: %s (%u)", + GetParseError_En(ok.Code()), ok.Offset()); + exit(EXIT_FAILURE); + } + \endcode + \see GenericReader::Parse, GenericDocument::Parse +*/ +struct ParseResult { +public: + //! Default constructor, no error. + ParseResult() : code_(kParseErrorNone), offset_(0) {} + //! Constructor to set an error. + ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} + + //! Get the error code. + ParseErrorCode Code() const { return code_; } + //! Get the error offset, if \ref IsError(), 0 otherwise. + size_t Offset() const { return offset_; } + + //! Conversion to \c bool, returns \c true, iff !\ref IsError(). + operator bool() const { return !IsError(); } + //! Whether the result is an error. + bool IsError() const { return code_ != kParseErrorNone; } + + bool operator==(const ParseResult& that) const { return code_ == that.code_; } + bool operator==(ParseErrorCode code) const { return code_ == code; } + friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + + //! Reset error code. + void Clear() { Set(kParseErrorNone); } + //! Update error code and offset. + void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } + +private: + ParseErrorCode code_; + size_t offset_; +}; + +//! Function pointer type of GetParseError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetParseError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetParseErrorFunc GetParseError = GetParseError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_ERROR_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/filereadstream.h b/ext/librethinkdbxx/src/rapidjson/filereadstream.h new file mode 100644 index 00000000..b56ea13b --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/filereadstream.h @@ -0,0 +1,99 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FILEREADSTREAM_H_ +#define RAPIDJSON_FILEREADSTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! File byte stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileReadStream { +public: + typedef char Ch; //!< Character type (byte). + + //! Constructor. + /*! + \param fp File pointer opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + RAPIDJSON_ASSERT(fp_ != 0); + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); + } + + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return (current_ + 4 <= bufferLast_) ? current_ : 0; + } + +private: + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = fread(buffer_, 1, bufferSize_, fp_); + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (readCount_ < bufferSize_) { + buffer_[readCount_] = '\0'; + ++bufferLast_; + eof_ = true; + } + } + } + + std::FILE* fp_; + Ch *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; +}; + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/filewritestream.h b/ext/librethinkdbxx/src/rapidjson/filewritestream.h new file mode 100644 index 00000000..6378dd60 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/filewritestream.h @@ -0,0 +1,104 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FILEWRITESTREAM_H_ +#define RAPIDJSON_FILEWRITESTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of C file stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileWriteStream { +public: + typedef char Ch; //!< Character type. Only support char. + + FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { + RAPIDJSON_ASSERT(fp_ != 0); + } + + void Put(char c) { + if (current_ >= bufferEnd_) + Flush(); + + *current_++ = c; + } + + void PutN(char c, size_t n) { + size_t avail = static_cast(bufferEnd_ - current_); + while (n > avail) { + std::memset(current_, c, avail); + current_ += avail; + Flush(); + n -= avail; + avail = static_cast(bufferEnd_ - current_); + } + + if (n > 0) { + std::memset(current_, c, n); + current_ += n; + } + } + + void Flush() { + if (current_ != buffer_) { + size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + if (result < static_cast(current_ - buffer_)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } + current_ = buffer_; + } + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + // Prohibit copy constructor & assignment operator. + FileWriteStream(const FileWriteStream&); + FileWriteStream& operator=(const FileWriteStream&); + + std::FILE* fp_; + char *buffer_; + char *bufferEnd_; + char *current_; +}; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(FileWriteStream& stream, char c, size_t n) { + stream.PutN(c, n); +} + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/fwd.h b/ext/librethinkdbxx/src/rapidjson/fwd.h new file mode 100644 index 00000000..e8104e84 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/fwd.h @@ -0,0 +1,151 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FWD_H_ +#define RAPIDJSON_FWD_H_ + +#include "rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN + +// encodings.h + +template struct UTF8; +template struct UTF16; +template struct UTF16BE; +template struct UTF16LE; +template struct UTF32; +template struct UTF32BE; +template struct UTF32LE; +template struct ASCII; +template struct AutoUTF; + +template +struct Transcoder; + +// allocators.h + +class CrtAllocator; + +template +class MemoryPoolAllocator; + +// stream.h + +template +struct GenericStringStream; + +typedef GenericStringStream > StringStream; + +template +struct GenericInsituStringStream; + +typedef GenericInsituStringStream > InsituStringStream; + +// stringbuffer.h + +template +class GenericStringBuffer; + +typedef GenericStringBuffer, CrtAllocator> StringBuffer; + +// filereadstream.h + +class FileReadStream; + +// filewritestream.h + +class FileWriteStream; + +// memorybuffer.h + +template +struct GenericMemoryBuffer; + +typedef GenericMemoryBuffer MemoryBuffer; + +// memorystream.h + +struct MemoryStream; + +// reader.h + +template +struct BaseReaderHandler; + +template +class GenericReader; + +typedef GenericReader, UTF8, CrtAllocator> Reader; + +// writer.h + +template +class Writer; + +// prettywriter.h + +template +class PrettyWriter; + +// document.h + +template +struct GenericMember; + +template +class GenericMemberIterator; + +template +struct GenericStringRef; + +template +class GenericValue; + +typedef GenericValue, MemoryPoolAllocator > Value; + +template +class GenericDocument; + +typedef GenericDocument, MemoryPoolAllocator, CrtAllocator> Document; + +// pointer.h + +template +class GenericPointer; + +typedef GenericPointer Pointer; + +// schema.h + +template +class IGenericRemoteSchemaDocumentProvider; + +template +class GenericSchemaDocument; + +typedef GenericSchemaDocument SchemaDocument; +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +template < + typename SchemaDocumentType, + typename OutputHandler, + typename StateAllocator> +class GenericSchemaValidator; + +typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_RAPIDJSONFWD_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h b/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h new file mode 100644 index 00000000..9d3e88c9 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h @@ -0,0 +1,290 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_BIGINTEGER_H_ +#define RAPIDJSON_BIGINTEGER_H_ + +#include "../rapidjson.h" + +#if defined(_MSC_VER) && defined(_M_AMD64) +#include // for _umul128 +#pragma intrinsic(_umul128) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +class BigInteger { +public: + typedef uint64_t Type; + + BigInteger(const BigInteger& rhs) : count_(rhs.count_) { + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + + explicit BigInteger(uint64_t u) : count_(1) { + digits_[0] = u; + } + + BigInteger(const char* decimals, size_t length) : count_(1) { + RAPIDJSON_ASSERT(length > 0); + digits_[0] = 0; + size_t i = 0; + const size_t kMaxDigitPerIteration = 19; // 2^64 = 18446744073709551616 > 10^19 + while (length >= kMaxDigitPerIteration) { + AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration); + length -= kMaxDigitPerIteration; + i += kMaxDigitPerIteration; + } + + if (length > 0) + AppendDecimal64(decimals + i, decimals + i + length); + } + + BigInteger& operator=(const BigInteger &rhs) + { + if (this != &rhs) { + count_ = rhs.count_; + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + return *this; + } + + BigInteger& operator=(uint64_t u) { + digits_[0] = u; + count_ = 1; + return *this; + } + + BigInteger& operator+=(uint64_t u) { + Type backup = digits_[0]; + digits_[0] += u; + for (size_t i = 0; i < count_ - 1; i++) { + if (digits_[i] >= backup) + return *this; // no carry + backup = digits_[i + 1]; + digits_[i + 1] += 1; + } + + // Last carry + if (digits_[count_ - 1] < backup) + PushBack(1); + + return *this; + } + + BigInteger& operator*=(uint64_t u) { + if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; + + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) { + uint64_t hi; + digits_[i] = MulAdd64(digits_[i], u, k, &hi); + k = hi; + } + + if (k > 0) + PushBack(k); + + return *this; + } + + BigInteger& operator*=(uint32_t u) { + if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; + + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) { + const uint64_t c = digits_[i] >> 32; + const uint64_t d = digits_[i] & 0xFFFFFFFF; + const uint64_t uc = u * c; + const uint64_t ud = u * d; + const uint64_t p0 = ud + k; + const uint64_t p1 = uc + (p0 >> 32); + digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32); + k = p1 >> 32; + } + + if (k > 0) + PushBack(k); + + return *this; + } + + BigInteger& operator<<=(size_t shift) { + if (IsZero() || shift == 0) return *this; + + size_t offset = shift / kTypeBit; + size_t interShift = shift % kTypeBit; + RAPIDJSON_ASSERT(count_ + offset <= kCapacity); + + if (interShift == 0) { + std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type)); + count_ += offset; + } + else { + digits_[count_] = 0; + for (size_t i = count_; i > 0; i--) + digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift)); + digits_[offset] = digits_[0] << interShift; + count_ += offset; + if (digits_[count_]) + count_++; + } + + std::memset(digits_, 0, offset * sizeof(Type)); + + return *this; + } + + bool operator==(const BigInteger& rhs) const { + return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0; + } + + bool operator==(const Type rhs) const { + return count_ == 1 && digits_[0] == rhs; + } + + BigInteger& MultiplyPow5(unsigned exp) { + static const uint32_t kPow5[12] = { + 5, + 5 * 5, + 5 * 5 * 5, + 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 + }; + if (exp == 0) return *this; + for (; exp >= 27; exp -= 27) *this *= RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27 + for (; exp >= 13; exp -= 13) *this *= static_cast(1220703125u); // 5^13 + if (exp > 0) *this *= kPow5[exp - 1]; + return *this; + } + + // Compute absolute difference of this and rhs. + // Assume this != rhs + bool Difference(const BigInteger& rhs, BigInteger* out) const { + int cmp = Compare(rhs); + RAPIDJSON_ASSERT(cmp != 0); + const BigInteger *a, *b; // Makes a > b + bool ret; + if (cmp < 0) { a = &rhs; b = this; ret = true; } + else { a = this; b = &rhs; ret = false; } + + Type borrow = 0; + for (size_t i = 0; i < a->count_; i++) { + Type d = a->digits_[i] - borrow; + if (i < b->count_) + d -= b->digits_[i]; + borrow = (d > a->digits_[i]) ? 1 : 0; + out->digits_[i] = d; + if (d != 0) + out->count_ = i + 1; + } + + return ret; + } + + int Compare(const BigInteger& rhs) const { + if (count_ != rhs.count_) + return count_ < rhs.count_ ? -1 : 1; + + for (size_t i = count_; i-- > 0;) + if (digits_[i] != rhs.digits_[i]) + return digits_[i] < rhs.digits_[i] ? -1 : 1; + + return 0; + } + + size_t GetCount() const { return count_; } + Type GetDigit(size_t index) const { RAPIDJSON_ASSERT(index < count_); return digits_[index]; } + bool IsZero() const { return count_ == 1 && digits_[0] == 0; } + +private: + void AppendDecimal64(const char* begin, const char* end) { + uint64_t u = ParseUint64(begin, end); + if (IsZero()) + *this = u; + else { + unsigned exp = static_cast(end - begin); + (MultiplyPow5(exp) <<= exp) += u; // *this = *this * 10^exp + u + } + } + + void PushBack(Type digit) { + RAPIDJSON_ASSERT(count_ < kCapacity); + digits_[count_++] = digit; + } + + static uint64_t ParseUint64(const char* begin, const char* end) { + uint64_t r = 0; + for (const char* p = begin; p != end; ++p) { + RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); + r = r * 10u + static_cast(*p - '0'); + } + return r; + } + + // Assume a * b + k < 2^128 + static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) { +#if defined(_MSC_VER) && defined(_M_AMD64) + uint64_t low = _umul128(a, b, outHigh) + k; + if (low < k) + (*outHigh)++; + return low; +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(a) * static_cast(b); + p += k; + *outHigh = static_cast(p >> 64); + return static_cast(p); +#else + const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32; + uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1; + x1 += (x0 >> 32); // can't give carry + x1 += x2; + if (x1 < x2) + x3 += (static_cast(1) << 32); + uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF); + uint64_t hi = x3 + (x1 >> 32); + + lo += k; + if (lo < k) + hi++; + *outHigh = hi; + return lo; +#endif + } + + static const size_t kBitCount = 3328; // 64bit * 54 > 10^1000 + static const size_t kCapacity = kBitCount / sizeof(Type); + static const size_t kTypeBit = sizeof(Type) * 8; + + Type digits_[kCapacity]; + size_t count_; +}; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_BIGINTEGER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h b/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h new file mode 100644 index 00000000..c9fefdc6 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h @@ -0,0 +1,258 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// This is a C++ header-only implementation of Grisu2 algorithm from the publication: +// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with +// integers." ACM Sigplan Notices 45.6 (2010): 233-243. + +#ifndef RAPIDJSON_DIYFP_H_ +#define RAPIDJSON_DIYFP_H_ + +#include "../rapidjson.h" + +#if defined(_MSC_VER) && defined(_M_AMD64) +#include +#pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_umul128) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +struct DiyFp { + DiyFp() : f(), e() {} + + DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} + + explicit DiyFp(double d) { + union { + double d; + uint64_t u64; + } u = { d }; + + int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); + uint64_t significand = (u.u64 & kDpSignificandMask); + if (biased_e != 0) { + f = significand + kDpHiddenBit; + e = biased_e - kDpExponentBias; + } + else { + f = significand; + e = kDpMinExponent + 1; + } + } + + DiyFp operator-(const DiyFp& rhs) const { + return DiyFp(f - rhs.f, e); + } + + DiyFp operator*(const DiyFp& rhs) const { +#if defined(_MSC_VER) && defined(_M_AMD64) + uint64_t h; + uint64_t l = _umul128(f, rhs.f, &h); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(f) * static_cast(rhs.f); + uint64_t h = static_cast(p >> 64); + uint64_t l = static_cast(p); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#else + const uint64_t M32 = 0xFFFFFFFF; + const uint64_t a = f >> 32; + const uint64_t b = f & M32; + const uint64_t c = rhs.f >> 32; + const uint64_t d = rhs.f & M32; + const uint64_t ac = a * c; + const uint64_t bc = b * c; + const uint64_t ad = a * d; + const uint64_t bd = b * d; + uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); + tmp += 1U << 31; /// mult_round + return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); +#endif + } + + DiyFp Normalize() const { +#if defined(_MSC_VER) && defined(_M_AMD64) + unsigned long index; + _BitScanReverse64(&index, f); + return DiyFp(f << (63 - index), e - (63 - index)); +#elif defined(__GNUC__) && __GNUC__ >= 4 + int s = __builtin_clzll(f); + return DiyFp(f << s, e - s); +#else + DiyFp res = *this; + while (!(res.f & (static_cast(1) << 63))) { + res.f <<= 1; + res.e--; + } + return res; +#endif + } + + DiyFp NormalizeBoundary() const { + DiyFp res = *this; + while (!(res.f & (kDpHiddenBit << 1))) { + res.f <<= 1; + res.e--; + } + res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); + res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); + return res; + } + + void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { + DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); + DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); + mi.f <<= mi.e - pl.e; + mi.e = pl.e; + *plus = pl; + *minus = mi; + } + + double ToDouble() const { + union { + double d; + uint64_t u64; + }u; + const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : + static_cast(e + kDpExponentBias); + u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); + return u.d; + } + + static const int kDiySignificandSize = 64; + static const int kDpSignificandSize = 52; + static const int kDpExponentBias = 0x3FF + kDpSignificandSize; + static const int kDpMaxExponent = 0x7FF - kDpExponentBias; + static const int kDpMinExponent = -kDpExponentBias; + static const int kDpDenormalExponent = -kDpExponentBias + 1; + static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + uint64_t f; + int e; +}; + +inline DiyFp GetCachedPowerByIndex(size_t index) { + // 10^-348, 10^-340, ..., 10^340 + static const uint64_t kCachedPowers_F[] = { + RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), + RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), + RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), + RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), + RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), + RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), + RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), + RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), + RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), + RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), + RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), + RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), + RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), + RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), + RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), + RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), + RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), + RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), + RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), + RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), + RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), + RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), + RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), + RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), + RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), + RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), + RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), + RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), + RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), + RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), + RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), + RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), + RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), + RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), + RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), + RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), + RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), + RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), + RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), + RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), + RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), + RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), + RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), + RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) + }; + static const int16_t kCachedPowers_E[] = { + -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, + -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, + -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, + -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, + -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, + 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, + 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, + 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, + 907, 933, 960, 986, 1013, 1039, 1066 + }; + return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); +} + +inline DiyFp GetCachedPower(int e, int* K) { + + //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; + double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive + int k = static_cast(dk); + if (dk - k > 0.0) + k++; + + unsigned index = static_cast((k >> 3) + 1); + *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table + + return GetCachedPowerByIndex(index); +} + +inline DiyFp GetCachedPower10(int exp, int *outExp) { + unsigned index = (static_cast(exp) + 348u) / 8u; + *outExp = -348 + static_cast(index) * 8; + return GetCachedPowerByIndex(index); + } + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +RAPIDJSON_DIAG_OFF(padded) +#endif + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_DIYFP_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h b/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h new file mode 100644 index 00000000..8d6350e6 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h @@ -0,0 +1,245 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// This is a C++ header-only implementation of Grisu2 algorithm from the publication: +// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with +// integers." ACM Sigplan Notices 45.6 (2010): 233-243. + +#ifndef RAPIDJSON_DTOA_ +#define RAPIDJSON_DTOA_ + +#include "itoa.h" // GetDigitsLut() +#include "diyfp.h" +#include "ieee754.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 +#endif + +inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { + while (rest < wp_w && delta - rest >= ten_kappa && + (rest + ten_kappa < wp_w || /// closer + wp_w - rest > rest + ten_kappa - wp_w)) { + buffer[len - 1]--; + rest += ten_kappa; + } +} + +inline unsigned CountDecimalDigit32(uint32_t n) { + // Simple pure C++ implementation was faster than __builtin_clz version in this situation. + if (n < 10) return 1; + if (n < 100) return 2; + if (n < 1000) return 3; + if (n < 10000) return 4; + if (n < 100000) return 5; + if (n < 1000000) return 6; + if (n < 10000000) return 7; + if (n < 100000000) return 8; + // Will not reach 10 digits in DigitGen() + //if (n < 1000000000) return 9; + //return 10; + return 9; +} + +inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { + static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); + const DiyFp wp_w = Mp - W; + uint32_t p1 = static_cast(Mp.f >> -one.e); + uint64_t p2 = Mp.f & (one.f - 1); + unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] + *len = 0; + + while (kappa > 0) { + uint32_t d = 0; + switch (kappa) { + case 9: d = p1 / 100000000; p1 %= 100000000; break; + case 8: d = p1 / 10000000; p1 %= 10000000; break; + case 7: d = p1 / 1000000; p1 %= 1000000; break; + case 6: d = p1 / 100000; p1 %= 100000; break; + case 5: d = p1 / 10000; p1 %= 10000; break; + case 4: d = p1 / 1000; p1 %= 1000; break; + case 3: d = p1 / 100; p1 %= 100; break; + case 2: d = p1 / 10; p1 %= 10; break; + case 1: d = p1; p1 = 0; break; + default:; + } + if (d || *len) + buffer[(*len)++] = static_cast('0' + static_cast(d)); + kappa--; + uint64_t tmp = (static_cast(p1) << -one.e) + p2; + if (tmp <= delta) { + *K += kappa; + GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); + return; + } + } + + // kappa = 0 + for (;;) { + p2 *= 10; + delta *= 10; + char d = static_cast(p2 >> -one.e); + if (d || *len) + buffer[(*len)++] = static_cast('0' + d); + p2 &= one.f - 1; + kappa--; + if (p2 < delta) { + *K += kappa; + int index = -static_cast(kappa); + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast(kappa)] : 0)); + return; + } + } +} + +inline void Grisu2(double value, char* buffer, int* length, int* K) { + const DiyFp v(value); + DiyFp w_m, w_p; + v.NormalizedBoundaries(&w_m, &w_p); + + const DiyFp c_mk = GetCachedPower(w_p.e, K); + const DiyFp W = v.Normalize() * c_mk; + DiyFp Wp = w_p * c_mk; + DiyFp Wm = w_m * c_mk; + Wm.f++; + Wp.f--; + DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); +} + +inline char* WriteExponent(int K, char* buffer) { + if (K < 0) { + *buffer++ = '-'; + K = -K; + } + + if (K >= 100) { + *buffer++ = static_cast('0' + static_cast(K / 100)); + K %= 100; + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else if (K >= 10) { + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else + *buffer++ = static_cast('0' + static_cast(K)); + + return buffer; +} + +inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { + const int kk = length + k; // 10^(kk-1) <= v < 10^kk + + if (0 <= k && kk <= 21) { + // 1234e7 -> 12340000000 + for (int i = length; i < kk; i++) + buffer[i] = '0'; + buffer[kk] = '.'; + buffer[kk + 1] = '0'; + return &buffer[kk + 2]; + } + else if (0 < kk && kk <= 21) { + // 1234e-2 -> 12.34 + std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); + buffer[kk] = '.'; + if (0 > k + maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[kk + 2]; // Reserve one zero + } + else + return &buffer[length + 1]; + } + else if (-6 < kk && kk <= 0) { + // 1234e-6 -> 0.001234 + const int offset = 2 - kk; + std::memmove(&buffer[offset], &buffer[0], static_cast(length)); + buffer[0] = '0'; + buffer[1] = '.'; + for (int i = 2; i < offset; i++) + buffer[i] = '0'; + if (length - kk > maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = maxDecimalPlaces + 1; i > 2; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[3]; // Reserve one zero + } + else + return &buffer[length + offset]; + } + else if (kk < -maxDecimalPlaces) { + // Truncate to zero + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else if (length == 1) { + // 1e30 + buffer[1] = 'e'; + return WriteExponent(kk - 1, &buffer[2]); + } + else { + // 1234e30 -> 1.234e33 + std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); + buffer[1] = '.'; + buffer[length + 1] = 'e'; + return WriteExponent(kk - 1, &buffer[0 + length + 2]); + } +} + +inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { + RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); + Double d(value); + if (d.IsZero()) { + if (d.Sign()) + *buffer++ = '-'; // -0.0, Issue #289 + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else { + if (value < 0) { + *buffer++ = '-'; + value = -value; + } + int length, K; + Grisu2(value, buffer, &length, &K); + return Prettify(buffer, length, K, maxDecimalPlaces); + } +} + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_DTOA_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h b/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h new file mode 100644 index 00000000..82bb0b99 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h @@ -0,0 +1,78 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_IEEE754_ +#define RAPIDJSON_IEEE754_ + +#include "../rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +class Double { +public: + Double() {} + Double(double d) : d_(d) {} + Double(uint64_t u) : u_(u) {} + + double Value() const { return d_; } + uint64_t Uint64Value() const { return u_; } + + double NextPositiveDouble() const { + RAPIDJSON_ASSERT(!Sign()); + return Double(u_ + 1).Value(); + } + + bool Sign() const { return (u_ & kSignMask) != 0; } + uint64_t Significand() const { return u_ & kSignificandMask; } + int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } + + bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } + bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } + bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } + bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } + bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } + + uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } + int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } + uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } + + static unsigned EffectiveSignificandSize(int order) { + if (order >= -1021) + return 53; + else if (order <= -1074) + return 0; + else + return static_cast(order) + 1074; + } + +private: + static const int kSignificandSize = 52; + static const int kExponentBias = 0x3FF; + static const int kDenormalExponent = 1 - kExponentBias; + static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); + static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + union { + double d_; + uint64_t u_; + }; +}; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_IEEE754_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/itoa.h b/ext/librethinkdbxx/src/rapidjson/internal/itoa.h new file mode 100644 index 00000000..01a4e7e7 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/itoa.h @@ -0,0 +1,304 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ITOA_ +#define RAPIDJSON_ITOA_ + +#include "../rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +inline const char* GetDigitsLut() { + static const char cDigitsLut[200] = { + '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', + '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', + '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', + '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', + '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', + '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', + '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', + '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', + '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', + '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' + }; + return cDigitsLut; +} + +inline char* u32toa(uint32_t value, char* buffer) { + const char* cDigitsLut = GetDigitsLut(); + + if (value < 10000) { + const uint32_t d1 = (value / 100) << 1; + const uint32_t d2 = (value % 100) << 1; + + if (value >= 1000) + *buffer++ = cDigitsLut[d1]; + if (value >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else if (value < 100000000) { + // value = bbbbcccc + const uint32_t b = value / 10000; + const uint32_t c = value % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + else { + // value = aabbbbcccc in decimal + + const uint32_t a = value / 100000000; // 1 to 42 + value %= 100000000; + + if (a >= 10) { + const unsigned i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else + *buffer++ = static_cast('0' + static_cast(a)); + + const uint32_t b = value / 10000; // 0 to 9999 + const uint32_t c = value % 10000; // 0 to 9999 + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + return buffer; +} + +inline char* i32toa(int32_t value, char* buffer) { + uint32_t u = static_cast(value); + if (value < 0) { + *buffer++ = '-'; + u = ~u + 1; + } + + return u32toa(u, buffer); +} + +inline char* u64toa(uint64_t value, char* buffer) { + const char* cDigitsLut = GetDigitsLut(); + const uint64_t kTen8 = 100000000; + const uint64_t kTen9 = kTen8 * 10; + const uint64_t kTen10 = kTen8 * 100; + const uint64_t kTen11 = kTen8 * 1000; + const uint64_t kTen12 = kTen8 * 10000; + const uint64_t kTen13 = kTen8 * 100000; + const uint64_t kTen14 = kTen8 * 1000000; + const uint64_t kTen15 = kTen8 * 10000000; + const uint64_t kTen16 = kTen8 * kTen8; + + if (value < kTen8) { + uint32_t v = static_cast(value); + if (v < 10000) { + const uint32_t d1 = (v / 100) << 1; + const uint32_t d2 = (v % 100) << 1; + + if (v >= 1000) + *buffer++ = cDigitsLut[d1]; + if (v >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (v >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else { + // value = bbbbcccc + const uint32_t b = v / 10000; + const uint32_t c = v % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + } + else if (value < kTen16) { + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + if (value >= kTen15) + *buffer++ = cDigitsLut[d1]; + if (value >= kTen14) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= kTen13) + *buffer++ = cDigitsLut[d2]; + if (value >= kTen12) + *buffer++ = cDigitsLut[d2 + 1]; + if (value >= kTen11) + *buffer++ = cDigitsLut[d3]; + if (value >= kTen10) + *buffer++ = cDigitsLut[d3 + 1]; + if (value >= kTen9) + *buffer++ = cDigitsLut[d4]; + if (value >= kTen8) + *buffer++ = cDigitsLut[d4 + 1]; + + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + else { + const uint32_t a = static_cast(value / kTen16); // 1 to 1844 + value %= kTen16; + + if (a < 10) + *buffer++ = static_cast('0' + static_cast(a)); + else if (a < 100) { + const uint32_t i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else if (a < 1000) { + *buffer++ = static_cast('0' + static_cast(a / 100)); + + const uint32_t i = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else { + const uint32_t i = (a / 100) << 1; + const uint32_t j = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + *buffer++ = cDigitsLut[j]; + *buffer++ = cDigitsLut[j + 1]; + } + + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + + return buffer; +} + +inline char* i64toa(int64_t value, char* buffer) { + uint64_t u = static_cast(value); + if (value < 0) { + *buffer++ = '-'; + u = ~u + 1; + } + + return u64toa(u, buffer); +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ITOA_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/meta.h b/ext/librethinkdbxx/src/rapidjson/internal/meta.h new file mode 100644 index 00000000..5a9aaa42 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/meta.h @@ -0,0 +1,181 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_META_H_ +#define RAPIDJSON_INTERNAL_META_H_ + +#include "../rapidjson.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif +#if defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(6334) +#endif + +#if RAPIDJSON_HAS_CXX11_TYPETRAITS +#include +#endif + +//@cond RAPIDJSON_INTERNAL +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +// Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching +template struct Void { typedef void Type; }; + +/////////////////////////////////////////////////////////////////////////////// +// BoolType, TrueType, FalseType +// +template struct BoolType { + static const bool Value = Cond; + typedef BoolType Type; +}; +typedef BoolType TrueType; +typedef BoolType FalseType; + + +/////////////////////////////////////////////////////////////////////////////// +// SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr +// + +template struct SelectIfImpl { template struct Apply { typedef T1 Type; }; }; +template <> struct SelectIfImpl { template struct Apply { typedef T2 Type; }; }; +template struct SelectIfCond : SelectIfImpl::template Apply {}; +template struct SelectIf : SelectIfCond {}; + +template struct AndExprCond : FalseType {}; +template <> struct AndExprCond : TrueType {}; +template struct OrExprCond : TrueType {}; +template <> struct OrExprCond : FalseType {}; + +template struct BoolExpr : SelectIf::Type {}; +template struct NotExpr : SelectIf::Type {}; +template struct AndExpr : AndExprCond::Type {}; +template struct OrExpr : OrExprCond::Type {}; + + +/////////////////////////////////////////////////////////////////////////////// +// AddConst, MaybeAddConst, RemoveConst +template struct AddConst { typedef const T Type; }; +template struct MaybeAddConst : SelectIfCond {}; +template struct RemoveConst { typedef T Type; }; +template struct RemoveConst { typedef T Type; }; + + +/////////////////////////////////////////////////////////////////////////////// +// IsSame, IsConst, IsMoreConst, IsPointer +// +template struct IsSame : FalseType {}; +template struct IsSame : TrueType {}; + +template struct IsConst : FalseType {}; +template struct IsConst : TrueType {}; + +template +struct IsMoreConst + : AndExpr::Type, typename RemoveConst::Type>, + BoolType::Value >= IsConst::Value> >::Type {}; + +template struct IsPointer : FalseType {}; +template struct IsPointer : TrueType {}; + +/////////////////////////////////////////////////////////////////////////////// +// IsBaseOf +// +#if RAPIDJSON_HAS_CXX11_TYPETRAITS + +template struct IsBaseOf + : BoolType< ::std::is_base_of::value> {}; + +#else // simplified version adopted from Boost + +template struct IsBaseOfImpl { + RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); + RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); + + typedef char (&Yes)[1]; + typedef char (&No) [2]; + + template + static Yes Check(const D*, T); + static No Check(const B*, int); + + struct Host { + operator const B*() const; + operator const D*(); + }; + + enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; +}; + +template struct IsBaseOf + : OrExpr, BoolExpr > >::Type {}; + +#endif // RAPIDJSON_HAS_CXX11_TYPETRAITS + + +////////////////////////////////////////////////////////////////////////// +// EnableIf / DisableIf +// +template struct EnableIfCond { typedef T Type; }; +template struct EnableIfCond { /* empty */ }; + +template struct DisableIfCond { typedef T Type; }; +template struct DisableIfCond { /* empty */ }; + +template +struct EnableIf : EnableIfCond {}; + +template +struct DisableIf : DisableIfCond {}; + +// SFINAE helpers +struct SfinaeTag {}; +template struct RemoveSfinaeTag; +template struct RemoveSfinaeTag { typedef T Type; }; + +#define RAPIDJSON_REMOVEFPTR_(type) \ + typename ::RAPIDJSON_NAMESPACE::internal::RemoveSfinaeTag \ + < ::RAPIDJSON_NAMESPACE::internal::SfinaeTag&(*) type>::Type + +#define RAPIDJSON_ENABLEIF(cond) \ + typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ + ::Type * = NULL + +#define RAPIDJSON_DISABLEIF(cond) \ + typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ + ::Type * = NULL + +#define RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ + typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ + ::Type + +#define RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ + typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ + ::Type + +} // namespace internal +RAPIDJSON_NAMESPACE_END +//@endcond + +#if defined(__GNUC__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_META_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/pow10.h b/ext/librethinkdbxx/src/rapidjson/internal/pow10.h new file mode 100644 index 00000000..02f475d7 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/pow10.h @@ -0,0 +1,55 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_POW10_ +#define RAPIDJSON_POW10_ + +#include "../rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Computes integer powers of 10 in double (10.0^n). +/*! This function uses lookup table for fast and accurate results. + \param n non-negative exponent. Must <= 308. + \return 10.0^n +*/ +inline double Pow10(int n) { + static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes + 1e+0, + 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, + 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, + 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, + 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, + 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, + 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, + 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, + 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, + 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, + 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, + 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, + 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, + 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, + 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, + 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, + 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 + }; + RAPIDJSON_ASSERT(n >= 0 && n <= 308); + return e[n]; +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_POW10_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/regex.h b/ext/librethinkdbxx/src/rapidjson/internal/regex.h new file mode 100644 index 00000000..422a5240 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/regex.h @@ -0,0 +1,701 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_REGEX_H_ +#define RAPIDJSON_INTERNAL_REGEX_H_ + +#include "../allocators.h" +#include "../stream.h" +#include "stack.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +#ifndef RAPIDJSON_REGEX_VERBOSE +#define RAPIDJSON_REGEX_VERBOSE 0 +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// GenericRegex + +static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 +static const SizeType kRegexInvalidRange = ~SizeType(0); + +//! Regular expression engine with subset of ECMAscript grammar. +/*! + Supported regular expression syntax: + - \c ab Concatenation + - \c a|b Alternation + - \c a? Zero or one + - \c a* Zero or more + - \c a+ One or more + - \c a{3} Exactly 3 times + - \c a{3,} At least 3 times + - \c a{3,5} 3 to 5 times + - \c (ab) Grouping + - \c ^a At the beginning + - \c a$ At the end + - \c . Any character + - \c [abc] Character classes + - \c [a-c] Character class range + - \c [a-z0-9_] Character class combination + - \c [^abc] Negated character classes + - \c [^a-c] Negated character class range + - \c [\b] Backspace (U+0008) + - \c \\| \\\\ ... Escape characters + - \c \\f Form feed (U+000C) + - \c \\n Line feed (U+000A) + - \c \\r Carriage return (U+000D) + - \c \\t Tab (U+0009) + - \c \\v Vertical tab (U+000B) + + \note This is a Thompson NFA engine, implemented with reference to + Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", + https://swtch.com/~rsc/regexp/regexp1.html +*/ +template +class GenericRegex { +public: + typedef typename Encoding::Ch Ch; + + GenericRegex(const Ch* source, Allocator* allocator = 0) : + states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + stateSet_(), state0_(allocator, 0), state1_(allocator, 0), anchorBegin_(), anchorEnd_() + { + GenericStringStream ss(source); + DecodedStream > ds(ss); + Parse(ds); + } + + ~GenericRegex() { + Allocator::Free(stateSet_); + } + + bool IsValid() const { + return root_ != kRegexInvalidState; + } + + template + bool Match(InputStream& is) const { + return SearchWithAnchoring(is, true, true); + } + + bool Match(const Ch* s) const { + GenericStringStream is(s); + return Match(is); + } + + template + bool Search(InputStream& is) const { + return SearchWithAnchoring(is, anchorBegin_, anchorEnd_); + } + + bool Search(const Ch* s) const { + GenericStringStream is(s); + return Search(is); + } + +private: + enum Operator { + kZeroOrOne, + kZeroOrMore, + kOneOrMore, + kConcatenation, + kAlternation, + kLeftParenthesis + }; + + static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' + static const unsigned kRangeCharacterClass = 0xFFFFFFFE; + static const unsigned kRangeNegationFlag = 0x80000000; + + struct Range { + unsigned start; // + unsigned end; + SizeType next; + }; + + struct State { + SizeType out; //!< Equals to kInvalid for matching state + SizeType out1; //!< Equals to non-kInvalid for split + SizeType rangeStart; + unsigned codepoint; + }; + + struct Frag { + Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} + SizeType start; + SizeType out; //!< link-list of all output states + SizeType minIndex; + }; + + template + class DecodedStream { + public: + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() { + unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } + + private: + void Decode() { + if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } + + SourceStream& ss_; + unsigned codepoint_; + }; + + State& GetState(SizeType index) { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + const State& GetState(SizeType index) const { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + Range& GetRange(SizeType index) { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + const Range& GetRange(SizeType index) const { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + template + void Parse(DecodedStream& ds) { + Allocator allocator; + Stack operandStack(&allocator, 256); // Frag + Stack operatorStack(&allocator, 256); // Operator + Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) + + *atomCountStack.template Push() = 0; + + unsigned codepoint; + while (ds.Peek() != 0) { + switch (codepoint = ds.Take()) { + case '^': + anchorBegin_ = true; + break; + + case '$': + anchorEnd_ = true; + break; + + case '|': + while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + *operatorStack.template Push() = kAlternation; + *atomCountStack.template Top() = 0; + break; + + case '(': + *operatorStack.template Push() = kLeftParenthesis; + *atomCountStack.template Push() = 0; + break; + + case ')': + while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + if (operatorStack.Empty()) + return; + operatorStack.template Pop(1); + atomCountStack.template Pop(1); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '?': + if (!Eval(operandStack, kZeroOrOne)) + return; + break; + + case '*': + if (!Eval(operandStack, kZeroOrMore)) + return; + break; + + case '+': + if (!Eval(operandStack, kOneOrMore)) + return; + break; + + case '{': + { + unsigned n, m; + if (!ParseUnsigned(ds, &n)) + return; + + if (ds.Peek() == ',') { + ds.Take(); + if (ds.Peek() == '}') + m = kInfinityQuantifier; + else if (!ParseUnsigned(ds, &m) || m < n) + return; + } + else + m = n; + + if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') + return; + ds.Take(); + } + break; + + case '.': + PushOperand(operandStack, kAnyCharacterClass); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '[': + { + SizeType range; + if (!ParseRange(ds, &range)) + return; + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); + GetState(s).rangeStart = range; + *operandStack.template Push() = Frag(s, s, s); + } + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '\\': // Escape character + if (!CharacterEscape(ds, &codepoint)) + return; // Unsupported escape character + // fall through to default + + default: // Pattern character + PushOperand(operandStack, codepoint); + ImplicitConcatenation(atomCountStack, operatorStack); + } + } + + while (!operatorStack.Empty()) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + + // Link the operand to matching state. + if (operandStack.GetSize() == sizeof(Frag)) { + Frag* e = operandStack.template Pop(1); + Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); + root_ = e->start; + +#if RAPIDJSON_REGEX_VERBOSE + printf("root: %d\n", root_); + for (SizeType i = 0; i < stateCount_ ; i++) { + State& s = GetState(i); + printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); + } + printf("\n"); +#endif + } + + // Preallocate buffer for SearchWithAnchoring() + RAPIDJSON_ASSERT(stateSet_ == 0); + if (stateCount_ > 0) { + stateSet_ = static_cast(states_.GetAllocator().Malloc(GetStateSetSize())); + state0_.template Reserve(stateCount_); + state1_.template Reserve(stateCount_); + } + } + + SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { + State* s = states_.template Push(); + s->out = out; + s->out1 = out1; + s->codepoint = codepoint; + s->rangeStart = kRegexInvalidRange; + return stateCount_++; + } + + void PushOperand(Stack& operandStack, unsigned codepoint) { + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); + *operandStack.template Push() = Frag(s, s, s); + } + + void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { + if (*atomCountStack.template Top()) + *operatorStack.template Push() = kConcatenation; + (*atomCountStack.template Top())++; + } + + SizeType Append(SizeType l1, SizeType l2) { + SizeType old = l1; + while (GetState(l1).out != kRegexInvalidState) + l1 = GetState(l1).out; + GetState(l1).out = l2; + return old; + } + + void Patch(SizeType l, SizeType s) { + for (SizeType next; l != kRegexInvalidState; l = next) { + next = GetState(l).out; + GetState(l).out = s; + } + } + + bool Eval(Stack& operandStack, Operator op) { + switch (op) { + case kConcatenation: + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + Patch(e1.out, e2.start); + *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); + } + return true; + + case kAlternation: + if (operandStack.GetSize() >= sizeof(Frag) * 2) { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + SizeType s = NewState(e1.start, e2.start, 0); + *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); + return true; + } + return false; + + case kZeroOrOne: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); + return true; + } + return false; + + case kZeroOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(s, s, e.minIndex); + return true; + } + return false; + + default: + RAPIDJSON_ASSERT(op == kOneOrMore); + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(e.start, s, e.minIndex); + return true; + } + return false; + } + } + + bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { + RAPIDJSON_ASSERT(n <= m); + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); + + if (n == 0) { + if (m == 0) // a{0} not support + return false; + else if (m == kInfinityQuantifier) + Eval(operandStack, kZeroOrMore); // a{0,} -> a* + else { + Eval(operandStack, kZeroOrOne); // a{0,5} -> a? + for (unsigned i = 0; i < m - 1; i++) + CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? + for (unsigned i = 0; i < m - 1; i++) + Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? + } + return true; + } + + for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a + CloneTopOperand(operandStack); + + if (m == kInfinityQuantifier) + Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ + else if (m > n) { + CloneTopOperand(operandStack); // a{3,5} -> a a a a + Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? + for (unsigned i = n; i < m - 1; i++) + CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? + for (unsigned i = n; i < m; i++) + Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + } + + for (unsigned i = 0; i < n - 1; i++) + Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? + + return true; + } + + static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } + + void CloneTopOperand(Stack& operandStack) { + const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation + SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) + State* s = states_.template Push(count); + memcpy(s, &GetState(src.minIndex), count * sizeof(State)); + for (SizeType j = 0; j < count; j++) { + if (s[j].out != kRegexInvalidState) + s[j].out += count; + if (s[j].out1 != kRegexInvalidState) + s[j].out1 += count; + } + *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); + stateCount_ += count; + } + + template + bool ParseUnsigned(DecodedStream& ds, unsigned* u) { + unsigned r = 0; + if (ds.Peek() < '0' || ds.Peek() > '9') + return false; + while (ds.Peek() >= '0' && ds.Peek() <= '9') { + if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 + return false; // overflow + r = r * 10 + (ds.Take() - '0'); + } + *u = r; + return true; + } + + template + bool ParseRange(DecodedStream& ds, SizeType* range) { + bool isBegin = true; + bool negate = false; + int step = 0; + SizeType start = kRegexInvalidRange; + SizeType current = kRegexInvalidRange; + unsigned codepoint; + while ((codepoint = ds.Take()) != 0) { + if (isBegin) { + isBegin = false; + if (codepoint == '^') { + negate = true; + continue; + } + } + + switch (codepoint) { + case ']': + if (start == kRegexInvalidRange) + return false; // Error: nothing inside [] + if (step == 2) { // Add trailing '-' + SizeType r = NewRange('-'); + RAPIDJSON_ASSERT(current != kRegexInvalidRange); + GetRange(current).next = r; + } + if (negate) + GetRange(start).start |= kRangeNegationFlag; + *range = start; + return true; + + case '\\': + if (ds.Peek() == 'b') { + ds.Take(); + codepoint = 0x0008; // Escape backspace character + } + else if (!CharacterEscape(ds, &codepoint)) + return false; + // fall through to default + + default: + switch (step) { + case 1: + if (codepoint == '-') { + step++; + break; + } + // fall through to step 0 for other characters + + case 0: + { + SizeType r = NewRange(codepoint); + if (current != kRegexInvalidRange) + GetRange(current).next = r; + if (start == kRegexInvalidRange) + start = r; + current = r; + } + step = 1; + break; + + default: + RAPIDJSON_ASSERT(step == 2); + GetRange(current).end = codepoint; + step = 0; + } + } + } + return false; + } + + SizeType NewRange(unsigned codepoint) { + Range* r = ranges_.template Push(); + r->start = r->end = codepoint; + r->next = kRegexInvalidRange; + return rangeCount_++; + } + + template + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { + unsigned codepoint; + switch (codepoint = ds.Take()) { + case '^': + case '$': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '.': + case '[': + case ']': + case '{': + case '}': + case '\\': + *escapedCodepoint = codepoint; return true; + case 'f': *escapedCodepoint = 0x000C; return true; + case 'n': *escapedCodepoint = 0x000A; return true; + case 'r': *escapedCodepoint = 0x000D; return true; + case 't': *escapedCodepoint = 0x0009; return true; + case 'v': *escapedCodepoint = 0x000B; return true; + default: + return false; // Unsupported escape character + } + } + + template + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) const { + RAPIDJSON_ASSERT(IsValid()); + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) { + std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { + const State& sr = GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == kAnyCharacterClass || + (sr.codepoint == kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { + matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, root_); + } + internal::Swap(current, next); + } + + return matched; + } + + size_t GetStateSetSize() const { + return (stateCount_ + 31) / 32 * 4; + } + + // Return whether the added states is a match state + bool AddState(Stack& l, SizeType index) const { + RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = GetState(index); + if (s.out1 != kRegexInvalidState) { // Split + bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) { + stateSet_[index >> 5] |= (1 << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + } + + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { + bool yes = (GetRange(rangeIndex).start & kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) { + const Range& r = GetRange(rangeIndex); + if (codepoint >= (r.start & ~kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; + } + + Stack states_; + Stack ranges_; + SizeType root_; + SizeType stateCount_; + SizeType rangeCount_; + + static const unsigned kInfinityQuantifier = ~0u; + + // For SearchWithAnchoring() + uint32_t* stateSet_; // allocated by states_.GetAllocator() + mutable Stack state0_; + mutable Stack state1_; + bool anchorBegin_; + bool anchorEnd_; +}; + +typedef GenericRegex > Regex; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/stack.h b/ext/librethinkdbxx/src/rapidjson/internal/stack.h new file mode 100644 index 00000000..022c9aab --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/stack.h @@ -0,0 +1,230 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_STACK_H_ +#define RAPIDJSON_INTERNAL_STACK_H_ + +#include "../allocators.h" +#include "swap.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// Stack + +//! A type-unsafe stack for storing different types of data. +/*! \tparam Allocator Allocator for allocating stack memory. +*/ +template +class Stack { +public: + // Optimization note: Do not allocate memory for stack_ in constructor. + // Do it lazily when first Push() -> Expand() -> Resize(). + Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack(Stack&& rhs) + : allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(rhs.stack_), + stackTop_(rhs.stackTop_), + stackEnd_(rhs.stackEnd_), + initialCapacity_(rhs.initialCapacity_) + { + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } +#endif + + ~Stack() { + Destroy(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack& operator=(Stack&& rhs) { + if (&rhs != this) + { + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = rhs.stack_; + stackTop_ = rhs.stackTop_; + stackEnd_ = rhs.stackEnd_; + initialCapacity_ = rhs.initialCapacity_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } + return *this; + } +#endif + + void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(stack_, rhs.stack_); + internal::Swap(stackTop_, rhs.stackTop_); + internal::Swap(stackEnd_, rhs.stackEnd_); + internal::Swap(initialCapacity_, rhs.initialCapacity_); + } + + void Clear() { stackTop_ = stack_; } + + void ShrinkToFit() { + if (Empty()) { + // If the stack is empty, completely deallocate the memory. + Allocator::Free(stack_); + stack_ = 0; + stackTop_ = 0; + stackEnd_ = 0; + } + else + Resize(GetSize()); + } + + // Optimization note: try to minimize the size of this function for force inline. + // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. + template + RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { + // Expand the stack if needed + if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) + Expand(count); + } + + template + RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + Reserve(count); + return PushUnsafe(count); + } + + template + RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { + RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); + T* ret = reinterpret_cast(stackTop_); + stackTop_ += sizeof(T) * count; + return ret; + } + + template + T* Pop(size_t count) { + RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); + stackTop_ -= count * sizeof(T); + return reinterpret_cast(stackTop_); + } + + template + T* Top() { + RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + const T* Top() const { + RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + T* End() { return reinterpret_cast(stackTop_); } + + template + const T* End() const { return reinterpret_cast(stackTop_); } + + template + T* Bottom() { return reinterpret_cast(stack_); } + + template + const T* Bottom() const { return reinterpret_cast(stack_); } + + bool HasAllocator() const { + return allocator_ != 0; + } + + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } + + bool Empty() const { return stackTop_ == stack_; } + size_t GetSize() const { return static_cast(stackTop_ - stack_); } + size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } + +private: + template + void Expand(size_t count) { + // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. + size_t newCapacity; + if (stack_ == 0) { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + newCapacity = initialCapacity_; + } else { + newCapacity = GetCapacity(); + newCapacity += (newCapacity + 1) / 2; + } + size_t newSize = GetSize() + sizeof(T) * count; + if (newCapacity < newSize) + newCapacity = newSize; + + Resize(newCapacity); + } + + void Resize(size_t newCapacity) { + const size_t size = GetSize(); // Backup the current size + stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); + stackTop_ = stack_ + size; + stackEnd_ = stack_ + newCapacity; + } + + void Destroy() { + Allocator::Free(stack_); + RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack + } + + // Prohibit copy constructor & assignment operator. + Stack(const Stack&); + Stack& operator=(const Stack&); + + Allocator* allocator_; + Allocator* ownAllocator_; + char *stack_; + char *stackTop_; + char *stackEnd_; + size_t initialCapacity_; +}; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_STACK_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h b/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h new file mode 100644 index 00000000..2edfae52 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h @@ -0,0 +1,55 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ +#define RAPIDJSON_INTERNAL_STRFUNC_H_ + +#include "../stream.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Custom strlen() which works on different character types. +/*! \tparam Ch Character type (e.g. char, wchar_t, short) + \param s Null-terminated input string. + \return Number of characters in the string. + \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. +*/ +template +inline SizeType StrLen(const Ch* s) { + const Ch* p = s; + while (*p) ++p; + return SizeType(p - s); +} + +//! Returns number of code points in a encoded string. +template +bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { + GenericStringStream is(s); + const typename Encoding::Ch* end = s + length; + SizeType count = 0; + while (is.src_ < end) { + unsigned codepoint; + if (!Encoding::Decode(is, &codepoint)) + return false; + count++; + } + *outCount = count; + return true; +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_INTERNAL_STRFUNC_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/strtod.h b/ext/librethinkdbxx/src/rapidjson/internal/strtod.h new file mode 100644 index 00000000..289c413b --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/strtod.h @@ -0,0 +1,269 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_STRTOD_ +#define RAPIDJSON_STRTOD_ + +#include "ieee754.h" +#include "biginteger.h" +#include "diyfp.h" +#include "pow10.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +inline double FastPath(double significand, int exp) { + if (exp < -308) + return 0.0; + else if (exp >= 0) + return significand * internal::Pow10(exp); + else + return significand / internal::Pow10(-exp); +} + +inline double StrtodNormalPrecision(double d, int p) { + if (p < -308) { + // Prevent expSum < -308, making Pow10(p) = 0 + d = FastPath(d, -308); + d = FastPath(d, p + 308); + } + else + d = FastPath(d, p); + return d; +} + +template +inline T Min3(T a, T b, T c) { + T m = a; + if (m > b) m = b; + if (m > c) m = c; + return m; +} + +inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { + const Double db(b); + const uint64_t bInt = db.IntegerSignificand(); + const int bExp = db.IntegerExponent(); + const int hExp = bExp - 1; + + int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; + + // Adjust for decimal exponent + if (dExp >= 0) { + dS_Exp2 += dExp; + dS_Exp5 += dExp; + } + else { + bS_Exp2 -= dExp; + bS_Exp5 -= dExp; + hS_Exp2 -= dExp; + hS_Exp5 -= dExp; + } + + // Adjust for binary exponent + if (bExp >= 0) + bS_Exp2 += bExp; + else { + dS_Exp2 -= bExp; + hS_Exp2 -= bExp; + } + + // Adjust for half ulp exponent + if (hExp >= 0) + hS_Exp2 += hExp; + else { + dS_Exp2 -= hExp; + bS_Exp2 -= hExp; + } + + // Remove common power of two factor from all three scaled values + int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); + dS_Exp2 -= common_Exp2; + bS_Exp2 -= common_Exp2; + hS_Exp2 -= common_Exp2; + + BigInteger dS = d; + dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); + + BigInteger bS(bInt); + bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); + + BigInteger hS(1); + hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); + + BigInteger delta(0); + dS.Difference(bS, &delta); + + return delta.Compare(hS); +} + +inline bool StrtodFast(double d, int p, double* result) { + // Use fast path for string-to-double conversion if possible + // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + if (p > 22 && p < 22 + 16) { + // Fast Path Cases In Disguise + d *= internal::Pow10(p - 22); + p = 22; + } + + if (p >= -22 && p <= 22 && d <= 9007199254740991.0) { // 2^53 - 1 + *result = FastPath(d, p); + return true; + } + else + return false; +} + +// Compute an approximation and see if it is within 1/2 ULP +inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) { + uint64_t significand = 0; + size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 + for (; i < length; i++) { + if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || + (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) + break; + significand = significand * 10u + static_cast(decimals[i] - '0'); + } + + if (i < length && decimals[i] >= '5') // Rounding + significand++; + + size_t remaining = length - i; + const unsigned kUlpShift = 3; + const unsigned kUlp = 1 << kUlpShift; + int64_t error = (remaining == 0) ? 0 : kUlp / 2; + + DiyFp v(significand, 0); + v = v.Normalize(); + error <<= -v.e; + + const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; + + int actualExp; + DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); + if (actualExp != dExp) { + static const DiyFp kPow10[] = { + DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 + DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 + DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 + DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 + DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 + DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 + DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 + }; + int adjustment = dExp - actualExp - 1; + RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); + v = v * kPow10[adjustment]; + if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit + error += kUlp / 2; + } + + v = v * cachedPower; + + error += kUlp + (error == 0 ? 0 : 1); + + const int oldExp = v.e; + v = v.Normalize(); + error <<= oldExp - v.e; + + const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + unsigned precisionSize = 64 - effectiveSignificandSize; + if (precisionSize + kUlpShift >= 64) { + unsigned scaleExp = (precisionSize + kUlpShift) - 63; + v.f >>= scaleExp; + v.e += scaleExp; + error = (error >> scaleExp) + 1 + static_cast(kUlp); + precisionSize -= scaleExp; + } + + DiyFp rounded(v.f >> precisionSize, v.e + static_cast(precisionSize)); + const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; + const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; + if (precisionBits >= halfWay + static_cast(error)) { + rounded.f++; + if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) + rounded.f >>= 1; + rounded.e++; + } + } + + *result = rounded.ToDouble(); + + return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); +} + +inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { + const BigInteger dInt(decimals, length); + const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; + Double a(approx); + int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); + if (cmp < 0) + return a.Value(); // within half ULP + else if (cmp == 0) { + // Round towards even + if (a.Significand() & 1) + return a.NextPositiveDouble(); + else + return a.Value(); + } + else // adjustment + return a.NextPositiveDouble(); +} + +inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { + RAPIDJSON_ASSERT(d >= 0.0); + RAPIDJSON_ASSERT(length >= 1); + + double result; + if (StrtodFast(d, p, &result)) + return result; + + // Trim leading zeros + while (*decimals == '0' && length > 1) { + length--; + decimals++; + decimalPosition--; + } + + // Trim trailing zeros + while (decimals[length - 1] == '0' && length > 1) { + length--; + decimalPosition--; + exp++; + } + + // Trim right-most digits + const int kMaxDecimalDigit = 780; + if (static_cast(length) > kMaxDecimalDigit) { + int delta = (static_cast(length) - kMaxDecimalDigit); + exp += delta; + decimalPosition -= static_cast(delta); + length = kMaxDecimalDigit; + } + + // If too small, underflow to zero + if (int(length) + exp < -324) + return 0.0; + + if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) + return result; + + // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison + return StrtodBigInteger(result, decimals, length, decimalPosition, exp); +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_STRTOD_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/swap.h b/ext/librethinkdbxx/src/rapidjson/internal/swap.h new file mode 100644 index 00000000..666e49f9 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/swap.h @@ -0,0 +1,46 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_SWAP_H_ +#define RAPIDJSON_INTERNAL_SWAP_H_ + +#include "../rapidjson.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Custom swap() to avoid dependency on C++ header +/*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. + \note This has the same semantics as std::swap(). +*/ +template +inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { + T tmp = a; + a = b; + b = tmp; +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_SWAP_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h b/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h new file mode 100644 index 00000000..f5fe2897 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h @@ -0,0 +1,115 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ISTREAMWRAPPER_H_ +#define RAPIDJSON_ISTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::istringstream + - \c std::stringstream + - \c std::wistringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wifstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_istream. +*/ + +template +class BasicIStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} + + Ch Peek() const { + typename StreamType::int_type c = stream_.peek(); + return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : '\0'; + } + + Ch Take() { + typename StreamType::int_type c = stream_.get(); + if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { + count_++; + return static_cast(c); + } + else + return '\0'; + } + + // tellg() may return -1 when failed. So we count by ourself. + size_t Tell() const { return count_; } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. + int i; + bool hasError = false; + for (i = 0; i < 4; ++i) { + typename StreamType::int_type c = stream_.get(); + if (c == StreamType::traits_type::eof()) { + hasError = true; + stream_.clear(); + break; + } + peekBuffer_[i] = static_cast(c); + } + for (--i; i >= 0; --i) + stream_.putback(peekBuffer_[i]); + return !hasError ? peekBuffer_ : 0; + } + +private: + BasicIStreamWrapper(const BasicIStreamWrapper&); + BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); + + StreamType& stream_; + size_t count_; //!< Number of characters read. Note: + mutable Ch peekBuffer_[4]; +}; + +typedef BasicIStreamWrapper IStreamWrapper; +typedef BasicIStreamWrapper WIStreamWrapper; + +#if defined(__clang__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ISTREAMWRAPPER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/memorybuffer.h b/ext/librethinkdbxx/src/rapidjson/memorybuffer.h new file mode 100644 index 00000000..39bee1de --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/memorybuffer.h @@ -0,0 +1,70 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_MEMORYBUFFER_H_ +#define RAPIDJSON_MEMORYBUFFER_H_ + +#include "stream.h" +#include "internal/stack.h" + +RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory output byte stream. +/*! + This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. + + It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. + + Differences between MemoryBuffer and StringBuffer: + 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. + 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. + + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +struct GenericMemoryBuffer { + typedef char Ch; // byte + + GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + + void Put(Ch c) { *stack_.template Push() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { stack_.ShrinkToFit(); } + Ch* Push(size_t count) { return stack_.template Push(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetBuffer() const { + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; +}; + +typedef GenericMemoryBuffer<> MemoryBuffer; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { + std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); +} + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/memorystream.h b/ext/librethinkdbxx/src/rapidjson/memorystream.h new file mode 100644 index 00000000..1d71d8a4 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/memorystream.h @@ -0,0 +1,71 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_MEMORYSTREAM_H_ +#define RAPIDJSON_MEMORYSTREAM_H_ + +#include "stream.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory input byte stream. +/*! + This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. + + It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. + + Differences between MemoryStream and StringStream: + 1. StringStream has encoding but MemoryStream is a byte stream. + 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. + 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). + \note implements Stream concept +*/ +struct MemoryStream { + typedef char Ch; // byte + + MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} + + Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } + Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } + size_t Tell() const { return static_cast(src_ - begin_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return Tell() + 4 <= size_ ? src_ : 0; + } + + const Ch* src_; //!< Current read position. + const Ch* begin_; //!< Original head of the string. + const Ch* end_; //!< End of stream. + size_t size_; //!< Size of the stream. +}; + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h b/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h new file mode 100644 index 00000000..18111286 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h @@ -0,0 +1,316 @@ +// ISO C9x compliant inttypes.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_INTTYPES_H_ // [ +#define _MSC_INTTYPES_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include "stdint.h" + +// miloyip: VC supports inttypes.h since VC2013 +#if _MSC_VER >= 1800 +#include +#else + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "I32d" +#define PRIi32 "I32i" +#define PRIdLEAST32 "I32d" +#define PRIiLEAST32 "I32i" +#define PRIdFAST32 "I32d" +#define PRIiFAST32 "I32i" + +#define PRId64 "I64d" +#define PRIi64 "I64i" +#define PRIdLEAST64 "I64d" +#define PRIiLEAST64 "I64i" +#define PRIdFAST64 "I64d" +#define PRIiFAST64 "I64i" + +#define PRIdMAX "I64d" +#define PRIiMAX "I64i" + +#define PRIdPTR "Id" +#define PRIiPTR "Ii" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "I32o" +#define PRIu32 "I32u" +#define PRIx32 "I32x" +#define PRIX32 "I32X" +#define PRIoLEAST32 "I32o" +#define PRIuLEAST32 "I32u" +#define PRIxLEAST32 "I32x" +#define PRIXLEAST32 "I32X" +#define PRIoFAST32 "I32o" +#define PRIuFAST32 "I32u" +#define PRIxFAST32 "I32x" +#define PRIXFAST32 "I32X" + +#define PRIo64 "I64o" +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#define PRIX64 "I64X" +#define PRIoLEAST64 "I64o" +#define PRIuLEAST64 "I64u" +#define PRIxLEAST64 "I64x" +#define PRIXLEAST64 "I64X" +#define PRIoFAST64 "I64o" +#define PRIuFAST64 "I64u" +#define PRIxFAST64 "I64x" +#define PRIXFAST64 "I64X" + +#define PRIoMAX "I64o" +#define PRIuMAX "I64u" +#define PRIxMAX "I64x" +#define PRIXMAX "I64X" + +#define PRIoPTR "Io" +#define PRIuPTR "Iu" +#define PRIxPTR "Ix" +#define PRIXPTR "IX" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + +#endif // _MSC_VER >= 1800 + +#endif // _MSC_INTTYPES_H_ ] diff --git a/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h b/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h new file mode 100644 index 00000000..3d4477b9 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h @@ -0,0 +1,300 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. +#if _MSC_VER >= 1600 // [ +#include + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +#undef INT8_C +#undef INT16_C +#undef INT32_C +#undef INT64_C +#undef UINT8_C +#undef UINT16_C +#undef UINT32_C +#undef UINT64_C + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#else // ] _MSC_VER >= 1700 [ + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we have to wrap include with 'extern "C++" {}' +// or compiler would give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#if defined(__cplusplus) && !defined(_M_ARM) +extern "C" { +#endif +# include +#if defined(__cplusplus) && !defined(_M_ARM) +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#endif // _MSC_VER >= 1600 ] + +#endif // _MSC_STDINT_H_ ] diff --git a/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h b/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h new file mode 100644 index 00000000..6f4667c0 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ +#define RAPIDJSON_OSTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::ostringstream + - \c std::stringstream + - \c std::wpstringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wofstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_ostream. +*/ + +template +class BasicOStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} + + void Put(Ch c) { + stream_.put(c); + } + + void Flush() { + stream_.flush(); + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + BasicOStreamWrapper(const BasicOStreamWrapper&); + BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); + + StreamType& stream_; +}; + +typedef BasicOStreamWrapper OStreamWrapper; +typedef BasicOStreamWrapper WOStreamWrapper; + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/pointer.h b/ext/librethinkdbxx/src/rapidjson/pointer.h new file mode 100644 index 00000000..0206ac1c --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/pointer.h @@ -0,0 +1,1358 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_POINTER_H_ +#define RAPIDJSON_POINTER_H_ + +#include "document.h" +#include "internal/itoa.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token + +//! Error code of parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode +*/ +enum PointerParseErrorCode { + kPointerParseErrorNone = 0, //!< The parse is successful + + kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' + kPointerParseErrorInvalidEscape, //!< Invalid escape + kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment + kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericPointer + +//! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. +/*! + This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" + (https://tools.ietf.org/html/rfc6901). + + A JSON pointer is for identifying a specific value in a JSON document + (GenericDocument). It can simplify coding of DOM tree manipulation, because it + can access multiple-level depth of DOM tree with single API call. + + After it parses a string representation (e.g. "/foo/0" or URI fragment + representation (e.g. "#/foo/0") into its internal representation (tokens), + it can be used to resolve a specific value in multiple documents, or sub-tree + of documents. + + Contrary to GenericValue, Pointer can be copy constructed and copy assigned. + Apart from assignment, a Pointer cannot be modified after construction. + + Although Pointer is very convenient, please aware that constructing Pointer + involves parsing and dynamic memory allocation. A special constructor with user- + supplied tokens eliminates these. + + GenericPointer depends on GenericDocument and GenericValue. + + \tparam ValueType The value type of the DOM tree. E.g. GenericValue > + \tparam Allocator The allocator type for allocating memory for internal representation. + + \note GenericPointer uses same encoding of ValueType. + However, Allocator of GenericPointer is independent of Allocator of Value. +*/ +template +class GenericPointer { +public: + typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value + typedef typename ValueType::Ch Ch; //!< Character type from Value + + //! A token is the basic units of internal representation. + /*! + A JSON pointer string representation "/foo/123" is parsed to two tokens: + "foo" and 123. 123 will be represented in both numeric form and string form. + They are resolved according to the actual value type (object or array). + + For token that are not numbers, or the numeric value is out of bound + (greater than limits of SizeType), they are only treated as string form + (i.e. the token's index will be equal to kPointerInvalidIndex). + + This struct is public so that user can create a Pointer without parsing and + allocation, using a special constructor. + */ + struct Token { + const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. + SizeType length; //!< Length of the name. + SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. + }; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor. + GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A null-terminated, string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + */ + explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, internal::StrLen(source)); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source.c_str(), source.size()); + } +#endif + + //! Constructor that parses a string or URI fragment representation, with length of the source string. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param length Length of source. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Slightly faster than the overload without length. + */ + GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, length); + } + + //! Constructor with user-supplied tokens. + /*! + This constructor let user supplies const array of tokens. + This prevents the parsing process and eliminates allocation. + This is preferred for memory constrained environments. + + \param tokens An constant array of tokens representing the JSON pointer. + \param tokenCount Number of tokens. + + \b Example + \code + #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } + #define INDEX(i) { #i, sizeof(#i) - 1, i } + + static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; + static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + // Equivalent to static const Pointer p("/foo/123"); + + #undef NAME + #undef INDEX + \endcode + */ + GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + *this = rhs; + } + + //! Destructor. + ~GenericPointer() { + if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. + Allocator::Free(tokens_); + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Assignment operator. + GenericPointer& operator=(const GenericPointer& rhs) { + if (this != &rhs) { + // Do not delete ownAllcator + if (nameBuffer_) + Allocator::Free(tokens_); + + tokenCount_ = rhs.tokenCount_; + parseErrorOffset_ = rhs.parseErrorOffset_; + parseErrorCode_ = rhs.parseErrorCode_; + + if (rhs.nameBuffer_) + CopyFromRaw(rhs); // Normally parsed tokens. + else { + tokens_ = rhs.tokens_; // User supplied const tokens. + nameBuffer_ = 0; + } + } + return *this; + } + + //@} + + //!@name Append token + //@{ + + //! Append a token and return a new Pointer + /*! + \param token Token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Token& token, Allocator* allocator = 0) const { + GenericPointer r; + r.allocator_ = allocator; + Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); + std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); + r.tokens_[tokenCount_].name = p; + r.tokens_[tokenCount_].length = token.length; + r.tokens_[tokenCount_].index = token.index; + return r; + } + + //! Append a name token with length, and return a new Pointer + /*! + \param name Name to be appended. + \param length Length of name. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { + Token token = { name, length, kPointerInvalidIndex }; + return Append(token, allocator); + } + + //! Append a name token without length, and return a new Pointer + /*! + \param name Name (const Ch*) to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) + Append(T* name, Allocator* allocator = 0) const { + return Append(name, StrLen(name), allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Append a name token, and return a new Pointer + /*! + \param name Name to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const { + return Append(name.c_str(), static_cast(name.size()), allocator); + } +#endif + + //! Append a index token, and return a new Pointer + /*! + \param index Index to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(SizeType index, Allocator* allocator = 0) const { + char buffer[21]; + char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); + SizeType length = static_cast(end - buffer); + buffer[length] = '\0'; + + if (sizeof(Ch) == 1) { + Token token = { reinterpret_cast(buffer), length, index }; + return Append(token, allocator); + } + else { + Ch name[21]; + for (size_t i = 0; i <= length; i++) + name[i] = buffer[i]; + Token token = { name, length, index }; + return Append(token, allocator); + } + } + + //! Append a token by value, and return a new Pointer + /*! + \param token token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { + if (token.IsString()) + return Append(token.GetString(), token.GetStringLength(), allocator); + else { + RAPIDJSON_ASSERT(token.IsUint64()); + RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); + return Append(static_cast(token.GetUint64()), allocator); + } + } + + //!@name Handling Parse Error + //@{ + + //! Check whether this is a valid pointer. + bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } + + //! Get the parsing error offset in code unit. + size_t GetParseErrorOffset() const { return parseErrorOffset_; } + + //! Get the parsing error code. + PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } + + //@} + + //! Get the allocator of this pointer. + Allocator& GetAllocator() { return *allocator_; } + + //!@name Tokens + //@{ + + //! Get the token array (const version only). + const Token* GetTokens() const { return tokens_; } + + //! Get the number of tokens. + size_t GetTokenCount() const { return tokenCount_; } + + //@} + + //!@name Equality/inequality operators + //@{ + + //! Equality operator. + /*! + \note When any pointers are invalid, always returns false. + */ + bool operator==(const GenericPointer& rhs) const { + if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) + return false; + + for (size_t i = 0; i < tokenCount_; i++) { + if (tokens_[i].index != rhs.tokens_[i].index || + tokens_[i].length != rhs.tokens_[i].length || + (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) + { + return false; + } + } + + return true; + } + + //! Inequality operator. + /*! + \note When any pointers are invalid, always returns true. + */ + bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } + + //@} + + //!@name Stringify + //@{ + + //! Stringify the pointer into string representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + return Stringify(os); + } + + //! Stringify the pointer into URI fragment representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool StringifyUriFragment(OutputStream& os) const { + return Stringify(os); + } + + //@} + + //!@name Create value + //@{ + + //! Create a value in a subtree. + /*! + If the value is not exist, it creates all parent values and a JSON Null value. + So it always succeed and return the newly created or existing value. + + Remind that it may change types of parents according to tokens, so it + potentially removes previously stored values. For example, if a document + was an array, and "/foo" is used to create a value, then the document + will be changed to an object, and all existing array elements are lost. + + \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created (a JSON Null value), or already exists value. + */ + ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { + RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + bool exist = true; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + if (v->IsArray() && t->name[0] == '-' && t->length == 1) { + v->PushBack(ValueType().Move(), allocator); + v = &((*v)[v->Size() - 1]); + exist = false; + } + else { + if (t->index == kPointerInvalidIndex) { // must be object name + if (!v->IsObject()) + v->SetObject(); // Change to Object + } + else { // object name or array index + if (!v->IsArray() && !v->IsObject()) + v->SetArray(); // Change to Array + } + + if (v->IsArray()) { + if (t->index >= v->Size()) { + v->Reserve(t->index + 1, allocator); + while (t->index >= v->Size()) + v->PushBack(ValueType().Move(), allocator); + exist = false; + } + v = &((*v)[t->index]); + } + else { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) { + v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); + v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end + exist = false; + } + else + v = &m->value; + } + } + } + + if (alreadyExist) + *alreadyExist = exist; + + return *v; + } + + //! Creates a value in a document. + /*! + \param document A document to be resolved. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created, or already exists value. + */ + template + ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const { + return Create(document, document.GetAllocator(), alreadyExist); + } + + //@} + + //!@name Query value + //@{ + + //! Query a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. + \return Pointer to the value if it can be resolved. Otherwise null. + + \note + There are only 3 situations when a value cannot be resolved: + 1. A value in the path is not an array nor object. + 2. An object value does not contain the token. + 3. A token is out of range of an array value. + + Use unresolvedTokenIndex to retrieve the token index. + */ + ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { + RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + break; + v = &m->value; + } + continue; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + break; + v = &((*v)[t->index]); + continue; + default: + break; + } + + // Error: unresolved token + if (unresolvedTokenIndex) + *unresolvedTokenIndex = static_cast(t - tokens_); + return 0; + } + return v; + } + + //! Query a const value in a const subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Pointer to the value if it can be resolved. Otherwise null. + */ + const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { + return Get(const_cast(root), unresolvedTokenIndex); + } + + //@} + + //!@name Query a value with default + //@{ + + //! Query a value in a subtree with default value. + /*! + Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. + So that this function always succeed. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param defaultValue Default value to be cloned if the value was not exists. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); + } + + //! Query a value in a subtree with default null-terminated string. + ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Query a value in a subtree with default std::basic_string. + ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } +#endif + + //! Query a value in a subtree with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { + return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); + } + + //! Query a value in a document with default value. + template + ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //! Query a value in a document with default null-terminated string. + template + ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Query a value in a document with default std::basic_string. + template + ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } +#endif + + //! Query a value in a document with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(GenericDocument& document, T defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //@} + + //!@name Set a value + //@{ + + //! Set a value in a subtree, with move semantics. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be set. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = value; + } + + //! Set a value in a subtree, with copy semantics. + ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).CopyFrom(value, allocator); + } + + //! Set a null-terminated string in a subtree. + ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Set a std::basic_string in a subtree. + ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } +#endif + + //! Set a primitive value in a subtree. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value).Move(); + } + + //! Set a value in a document, with move semantics. + template + ValueType& Set(GenericDocument& document, ValueType& value) const { + return Create(document) = value; + } + + //! Set a value in a document, with copy semantics. + template + ValueType& Set(GenericDocument& document, const ValueType& value) const { + return Create(document).CopyFrom(value, document.GetAllocator()); + } + + //! Set a null-terminated string in a document. + template + ValueType& Set(GenericDocument& document, const Ch* value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Sets a std::basic_string in a document. + template + ValueType& Set(GenericDocument& document, const std::basic_string& value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } +#endif + + //! Set a primitive value in a document. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(GenericDocument& document, T value) const { + return Create(document) = value; + } + + //@} + + //!@name Swap a value + //@{ + + //! Swap a value with a value in a subtree. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be swapped. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).Swap(value); + } + + //! Swap a value with a value in a document. + template + ValueType& Swap(GenericDocument& document, ValueType& value) const { + return Create(document).Swap(value); + } + + //@} + + //! Erase a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Whether the resolved value is found and erased. + + \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. + */ + bool Erase(ValueType& root) const { + RAPIDJSON_ASSERT(IsValid()); + if (tokenCount_ == 0) // Cannot erase the root + return false; + + ValueType* v = &root; + const Token* last = tokens_ + (tokenCount_ - 1); + for (const Token *t = tokens_; t != last; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + return false; + v = &m->value; + } + break; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + return false; + v = &((*v)[t->index]); + break; + default: + return false; + } + } + + switch (v->GetType()) { + case kObjectType: + return v->EraseMember(GenericStringRef(last->name, last->length)); + case kArrayType: + if (last->index == kPointerInvalidIndex || last->index >= v->Size()) + return false; + v->Erase(v->Begin() + last->index); + return true; + default: + return false; + } + } + +private: + //! Clone the content from rhs to this. + /*! + \param rhs Source pointer. + \param extraToken Extra tokens to be allocated. + \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. + \return Start of non-occupied name buffer, for storing extra names. + */ + Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { + if (!allocator_) // allocator is independently owned. + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens + for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) + nameBufferSize += t->length; + + tokenCount_ = rhs.tokenCount_ + extraToken; + tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); + nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + if (rhs.tokenCount_ > 0) { + std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + } + if (nameBufferSize > 0) { + std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + } + + // Adjust pointers to name buffer + std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; + for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) + t->name += diff; + + return nameBuffer_ + nameBufferSize; + } + + //! Check whether a character should be percent-encoded. + /*! + According to RFC 3986 2.3 Unreserved Characters. + \param c The character (code unit) to be tested. + */ + bool NeedPercentEncode(Ch c) const { + return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); + } + + //! Parse a JSON String or its URI fragment representation into tokens. +#ifndef __clang__ // -Wdocumentation + /*! + \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. + \param length Length of the source string. + \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. + */ +#endif + void Parse(const Ch* source, size_t length) { + RAPIDJSON_ASSERT(source != NULL); + RAPIDJSON_ASSERT(nameBuffer_ == 0); + RAPIDJSON_ASSERT(tokens_ == 0); + + // Create own allocator if user did not supply. + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + // Count number of '/' as tokenCount + tokenCount_ = 0; + for (const Ch* s = source; s != source + length; s++) + if (*s == '/') + tokenCount_++; + + Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); + Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + size_t i = 0; + + // Detect if it is a URI fragment + bool uriFragment = false; + if (source[i] == '#') { + uriFragment = true; + i++; + } + + if (i != length && source[i] != '/') { + parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; + goto error; + } + + while (i < length) { + RAPIDJSON_ASSERT(source[i] == '/'); + i++; // consumes '/' + + token->name = name; + bool isNumber = true; + + while (i < length && source[i] != '/') { + Ch c = source[i]; + if (uriFragment) { + // Decoding percent-encoding for URI fragment + if (c == '%') { + PercentDecodeStream is(&source[i], source + length); + GenericInsituStringStream os(name); + Ch* begin = os.PutBegin(); + if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) { + parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; + goto error; + } + size_t len = os.PutEnd(begin); + i += is.Tell() - 1; + if (len == 1) + c = *name; + else { + name += len; + isNumber = false; + i++; + continue; + } + } + else if (NeedPercentEncode(c)) { + parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; + goto error; + } + } + + i++; + + // Escaping "~0" -> '~', "~1" -> '/' + if (c == '~') { + if (i < length) { + c = source[i]; + if (c == '0') c = '~'; + else if (c == '1') c = '/'; + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + i++; + } + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + } + + // First check for index: all of characters are digit + if (c < '0' || c > '9') + isNumber = false; + + *name++ = c; + } + token->length = static_cast(name - token->name); + if (token->length == 0) + isNumber = false; + *name++ = '\0'; // Null terminator + + // Second check for index: more than one digit cannot have leading zero + if (isNumber && token->length > 1 && token->name[0] == '0') + isNumber = false; + + // String to SizeType conversion + SizeType n = 0; + if (isNumber) { + for (size_t j = 0; j < token->length; j++) { + SizeType m = n * 10 + static_cast(token->name[j] - '0'); + if (m < n) { // overflow detection + isNumber = false; + break; + } + n = m; + } + } + + token->index = isNumber ? n : kPointerInvalidIndex; + token++; + } + + RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer + parseErrorCode_ = kPointerParseErrorNone; + return; + + error: + Allocator::Free(tokens_); + nameBuffer_ = 0; + tokens_ = 0; + tokenCount_ = 0; + parseErrorOffset_ = i; + return; + } + + //! Stringify to string or URI fragment representation. + /*! + \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. + \tparam OutputStream type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + RAPIDJSON_ASSERT(IsValid()); + + if (uriFragment) + os.Put('#'); + + for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + os.Put('/'); + for (size_t j = 0; j < t->length; j++) { + Ch c = t->name[j]; + if (c == '~') { + os.Put('~'); + os.Put('0'); + } + else if (c == '/') { + os.Put('~'); + os.Put('1'); + } + else if (uriFragment && NeedPercentEncode(c)) { + // Transcode to UTF8 sequence + GenericStringStream source(&t->name[j]); + PercentEncodeStream target(os); + if (!Transcoder >().Validate(source, target)) + return false; + j += source.Tell() - 1; + } + else + os.Put(c); + } + } + return true; + } + + //! A helper stream for decoding a percent-encoded sequence into code unit. + /*! + This stream decodes %XY triplet into code unit (0-255). + If it encounters invalid characters, it sets output code unit as 0 and + mark invalid, and to be checked by IsValid(). + */ + class PercentDecodeStream { + public: + typedef typename ValueType::Ch Ch; + + //! Constructor + /*! + \param source Start of the stream + \param end Past-the-end of the stream. + */ + PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} + + Ch Take() { + if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet + valid_ = false; + return 0; + } + src_++; + Ch c = 0; + for (int j = 0; j < 2; j++) { + c = static_cast(c << 4); + Ch h = *src_; + if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); + else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); + else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); + else { + valid_ = false; + return 0; + } + src_++; + } + return c; + } + + size_t Tell() const { return static_cast(src_ - head_); } + bool IsValid() const { return valid_; } + + private: + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. + const Ch* end_; //!< Past-the-end position. + bool valid_; //!< Whether the parsing is valid. + }; + + //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. + template + class PercentEncodeStream { + public: + PercentEncodeStream(OutputStream& os) : os_(os) {} + void Put(char c) { // UTF-8 must be byte + unsigned char u = static_cast(c); + static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + os_.Put('%'); + os_.Put(hexDigits[u >> 4]); + os_.Put(hexDigits[u & 15]); + } + private: + OutputStream& os_; + }; + + Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. + Allocator* ownAllocator_; //!< Allocator owned by this Pointer. + Ch* nameBuffer_; //!< A buffer containing all names in tokens. + Token* tokens_; //!< A list of tokens. + size_t tokenCount_; //!< Number of tokens in tokens_. + size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. + PointerParseErrorCode parseErrorCode_; //!< Parsing error code. +}; + +//! GenericPointer for Value (UTF-8, default allocator). +typedef GenericPointer Pointer; + +//!@name Helper functions for GenericPointer +//@{ + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) { + return pointer.Create(root, a); +} + +template +typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Create(root, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) { + return pointer.Create(document); +} + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Create(document); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) { + return pointer.Set(document, value); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) { + return pointer.Set(document, value); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) { + return GenericPointer(source, N - 1).Set(document, value); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Swap(root, value, a); +} + +template +typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Swap(root, value, a); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Swap(document, value); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Swap(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +bool EraseValueByPointer(T& root, const GenericPointer& pointer) { + return pointer.Erase(root); +} + +template +bool EraseValueByPointer(T& root, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Erase(root); +} + +//@} + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_POINTER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/prettywriter.h b/ext/librethinkdbxx/src/rapidjson/prettywriter.h new file mode 100644 index 00000000..75dc474f --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/prettywriter.h @@ -0,0 +1,249 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_PRETTYWRITER_H_ +#define RAPIDJSON_PRETTYWRITER_H_ + +#include "writer.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Combination of PrettyWriter format flags. +/*! \see PrettyWriter::SetFormatOptions + */ +enum PrettyFormatOptions { + kFormatDefault = 0, //!< Default pretty formatting. + kFormatSingleLineArray = 1 //!< Format arrays on a single line. +}; + +//! Writer with indentation and spacing. +/*! + \tparam OutputStream Type of ouptut os. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class PrettyWriter : public Writer { +public: + typedef Writer Base; + typedef typename Base::Ch Ch; + + //! Constructor + /*! \param os Output stream. + \param allocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} + + + explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + + //! Set custom indentation. + /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). + \param indentCharCount Number of indent characters for each indentation level. + \note The default indentation is 4 spaces. + */ + PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { + RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); + indentChar_ = indentChar; + indentCharCount_ = indentCharCount; + return *this; + } + + //! Set pretty writer formatting options. + /*! \param options Formatting options. + */ + PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { + formatOptions_ = options; + return *this; + } + + /*! @name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } + bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } + bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } + bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } + bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } + bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } + bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + PrettyPrefix(kNumberType); + return Base::WriteString(str, length); + } + + bool String(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + PrettyPrefix(kStringType); + return Base::WriteString(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + bool String(const std::basic_string& str) { + return String(str.data(), SizeType(str.size())); + } +#endif + + bool StartObject() { + PrettyPrefix(kObjectType); + new (Base::level_stack_.template Push()) typename Base::Level(false); + return Base::WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::WriteEndObject(); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } + + bool StartArray() { + PrettyPrefix(kArrayType); + new (Base::level_stack_.template Push()) typename Base::Level(true); + return Base::WriteStartArray(); + } + + bool EndArray(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::WriteEndArray(); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } + + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. + */ + bool RawValue(const Ch* json, size_t length, Type type) { PrettyPrefix(type); return Base::WriteRawValue(json, length); } + +protected: + void PrettyPrefix(Type type) { + (void)type; + if (Base::level_stack_.GetSize() != 0) { // this value is not at root + typename Base::Level* level = Base::level_stack_.template Top(); + + if (level->inArray) { + if (level->valueCount > 0) { + Base::os_->Put(','); // add comma if it is not the first element in array + if (formatOptions_ & kFormatSingleLineArray) + Base::os_->Put(' '); + } + + if (!(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + } + else { // in object + if (level->valueCount > 0) { + if (level->valueCount % 2 == 0) { + Base::os_->Put(','); + Base::os_->Put('\n'); + } + else { + Base::os_->Put(':'); + Base::os_->Put(' '); + } + } + else + Base::os_->Put('\n'); + + if (level->valueCount % 2 == 0) + WriteIndent(); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. + Base::hasRoot_ = true; + } + } + + void WriteIndent() { + size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; + PutN(*Base::os_, static_cast(indentChar_), count); + } + + Ch indentChar_; + unsigned indentCharCount_; + PrettyFormatOptions formatOptions_; + +private: + // Prohibit copy constructor & assignment operator. + PrettyWriter(const PrettyWriter&); + PrettyWriter& operator=(const PrettyWriter&); +}; + +RAPIDJSON_NAMESPACE_END + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/rapidjson.h b/ext/librethinkdbxx/src/rapidjson/rapidjson.h new file mode 100644 index 00000000..d666f202 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/rapidjson.h @@ -0,0 +1,615 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_RAPIDJSON_H_ +#define RAPIDJSON_RAPIDJSON_H_ + +/*!\file rapidjson.h + \brief common definitions and configuration + + \see RAPIDJSON_CONFIG + */ + +/*! \defgroup RAPIDJSON_CONFIG RapidJSON configuration + \brief Configuration macros for library features + + Some RapidJSON features are configurable to adapt the library to a wide + variety of platforms, environments and usage scenarios. Most of the + features can be configured in terms of overriden or predefined + preprocessor macros at compile-time. + + Some additional customization is available in the \ref RAPIDJSON_ERRORS APIs. + + \note These macros should be given on the compiler command-line + (where applicable) to avoid inconsistent values when compiling + different translation units of a single application. + */ + +#include // malloc(), realloc(), free(), size_t +#include // memset(), memcpy(), memmove(), memcmp() + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_VERSION_STRING +// +// ALWAYS synchronize the following 3 macros with corresponding variables in /CMakeLists.txt. +// + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +// token stringification +#define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) +#define RAPIDJSON_DO_STRINGIFY(x) #x +//!@endcond + +/*! \def RAPIDJSON_MAJOR_VERSION + \ingroup RAPIDJSON_CONFIG + \brief Major version of RapidJSON in integer. +*/ +/*! \def RAPIDJSON_MINOR_VERSION + \ingroup RAPIDJSON_CONFIG + \brief Minor version of RapidJSON in integer. +*/ +/*! \def RAPIDJSON_PATCH_VERSION + \ingroup RAPIDJSON_CONFIG + \brief Patch version of RapidJSON in integer. +*/ +/*! \def RAPIDJSON_VERSION_STRING + \ingroup RAPIDJSON_CONFIG + \brief Version of RapidJSON in ".." string format. +*/ +#define RAPIDJSON_MAJOR_VERSION 1 +#define RAPIDJSON_MINOR_VERSION 0 +#define RAPIDJSON_PATCH_VERSION 2 +#define RAPIDJSON_VERSION_STRING \ + RAPIDJSON_STRINGIFY(RAPIDJSON_MAJOR_VERSION.RAPIDJSON_MINOR_VERSION.RAPIDJSON_PATCH_VERSION) + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NAMESPACE_(BEGIN|END) +/*! \def RAPIDJSON_NAMESPACE + \ingroup RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace + + In order to avoid symbol clashes and/or "One Definition Rule" errors + between multiple inclusions of (different versions of) RapidJSON in + a single binary, users can customize the name of the main RapidJSON + namespace. + + In case of a single nesting level, defining \c RAPIDJSON_NAMESPACE + to a custom name (e.g. \c MyRapidJSON) is sufficient. If multiple + levels are needed, both \ref RAPIDJSON_NAMESPACE_BEGIN and \ref + RAPIDJSON_NAMESPACE_END need to be defined as well: + + \code + // in some .cpp file + #define RAPIDJSON_NAMESPACE my::rapidjson + #define RAPIDJSON_NAMESPACE_BEGIN namespace my { namespace rapidjson { + #define RAPIDJSON_NAMESPACE_END } } + #include "rapidjson/..." + \endcode + + \see rapidjson + */ +/*! \def RAPIDJSON_NAMESPACE_BEGIN + \ingroup RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace (opening expression) + \see RAPIDJSON_NAMESPACE +*/ +/*! \def RAPIDJSON_NAMESPACE_END + \ingroup RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace (closing expression) + \see RAPIDJSON_NAMESPACE +*/ +#ifndef RAPIDJSON_NAMESPACE +#define RAPIDJSON_NAMESPACE rapidjson +#endif +#ifndef RAPIDJSON_NAMESPACE_BEGIN +#define RAPIDJSON_NAMESPACE_BEGIN namespace RAPIDJSON_NAMESPACE { +#endif +#ifndef RAPIDJSON_NAMESPACE_END +#define RAPIDJSON_NAMESPACE_END } +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_HAS_STDSTRING + +#ifndef RAPIDJSON_HAS_STDSTRING +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation +#else +#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default +#endif +/*! \def RAPIDJSON_HAS_STDSTRING + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for \c std::string + + By defining this preprocessor symbol to \c 1, several convenience functions for using + \ref rapidjson::GenericValue with \c std::string are enabled, especially + for construction and comparison. + + \hideinitializer +*/ +#endif // !defined(RAPIDJSON_HAS_STDSTRING) + +#if RAPIDJSON_HAS_STDSTRING +#include +#endif // RAPIDJSON_HAS_STDSTRING + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NO_INT64DEFINE + +/*! \def RAPIDJSON_NO_INT64DEFINE + \ingroup RAPIDJSON_CONFIG + \brief Use external 64-bit integer types. + + RapidJSON requires the 64-bit integer types \c int64_t and \c uint64_t types + to be available at global scope. + + If users have their own definition, define RAPIDJSON_NO_INT64DEFINE to + prevent RapidJSON from defining its own types. +*/ +#ifndef RAPIDJSON_NO_INT64DEFINE +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 +#include "msinttypes/stdint.h" +#include "msinttypes/inttypes.h" +#else +// Other compilers should have this. +#include +#include +#endif +//!@endcond +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_NO_INT64DEFINE +#endif +#endif // RAPIDJSON_NO_INT64TYPEDEF + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_FORCEINLINE + +#ifndef RAPIDJSON_FORCEINLINE +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#if defined(_MSC_VER) && defined(NDEBUG) +#define RAPIDJSON_FORCEINLINE __forceinline +#elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) +#define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) +#else +#define RAPIDJSON_FORCEINLINE +#endif +//!@endcond +#endif // RAPIDJSON_FORCEINLINE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ENDIAN +#define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine +#define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine + +//! Endianness of the machine. +/*! + \def RAPIDJSON_ENDIAN + \ingroup RAPIDJSON_CONFIG + + GCC 4.6 provided macro for detecting endianness of the target machine. But other + compilers may not have this. User can define RAPIDJSON_ENDIAN to either + \ref RAPIDJSON_LITTLEENDIAN or \ref RAPIDJSON_BIGENDIAN. + + Default detection implemented with reference to + \li https://gcc.gnu.org/onlinedocs/gcc-4.6.0/cpp/Common-Predefined-Macros.html + \li http://www.boost.org/doc/libs/1_42_0/boost/detail/endian.hpp +*/ +#ifndef RAPIDJSON_ENDIAN +// Detect with GCC 4.6's macro +# ifdef __BYTE_ORDER__ +# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif // __BYTE_ORDER__ +// Detect with GLIBC's endian.h +# elif defined(__GLIBC__) +# include +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif (__BYTE_ORDER == __BIG_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif // __GLIBC__ +// Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro +# elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +// Detect with architecture macros +# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(_MSC_VER) && defined(_M_ARM) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(RAPIDJSON_DOXYGEN_RUNNING) +# define RAPIDJSON_ENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif +#endif // RAPIDJSON_ENDIAN + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_64BIT + +//! Whether using 64-bit architecture +#ifndef RAPIDJSON_64BIT +#if defined(__LP64__) || defined(_WIN64) || defined(__EMSCRIPTEN__) +#define RAPIDJSON_64BIT 1 +#else +#define RAPIDJSON_64BIT 0 +#endif +#endif // RAPIDJSON_64BIT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ALIGN + +//! Data alignment of the machine. +/*! \ingroup RAPIDJSON_CONFIG + \param x pointer to align + + Some machines require strict data alignment. Currently the default uses 4 bytes + alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. + User can customize by defining the RAPIDJSON_ALIGN function macro. +*/ +#ifndef RAPIDJSON_ALIGN +#if RAPIDJSON_64BIT == 1 +#define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) +#else +#define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) +#endif +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_UINT64_C2 + +//! Construct a 64-bit literal by a pair of 32-bit integer. +/*! + 64-bit literal with or without ULL suffix is prone to compiler warnings. + UINT64_C() is C macro which cause compilation problems. + Use this macro to define 64-bit constants by a pair of 32-bit integer. +*/ +#ifndef RAPIDJSON_UINT64_C2 +#define RAPIDJSON_UINT64_C2(high32, low32) ((static_cast(high32) << 32) | static_cast(low32)) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_48BITPOINTER_OPTIMIZATION + +//! Use only lower 48-bit address for some pointers. +/*! + \ingroup RAPIDJSON_CONFIG + + This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. + The higher 16-bit can be used for storing other data. + \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. +*/ +#ifndef RAPIDJSON_48BITPOINTER_OPTIMIZATION +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 +#else +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 +#endif +#endif // RAPIDJSON_48BITPOINTER_OPTIMIZATION + +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 +#if RAPIDJSON_64BIT != 1 +#error RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when RAPIDJSON_64BIT=1 +#endif +#define RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast((reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast(reinterpret_cast(x)))) +#define RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast(reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) +#else +#define RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) +#define RAPIDJSON_GETPOINTER(type, p) (p) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD + +/*! \def RAPIDJSON_SIMD + \ingroup RAPIDJSON_CONFIG + \brief Enable SSE2/SSE4.2 optimization. + + RapidJSON supports optimized implementations for some parsing operations + based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible + processors. + + To enable these optimizations, two different symbols can be defined; + \code + // Enable SSE2 optimization. + #define RAPIDJSON_SSE2 + + // Enable SSE4.2 optimization. + #define RAPIDJSON_SSE42 + \endcode + + \c RAPIDJSON_SSE42 takes precedence, if both are defined. + + If any of these symbols is defined, RapidJSON defines the macro + \c RAPIDJSON_SIMD to indicate the availability of the optimized code. +*/ +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ + || defined(RAPIDJSON_DOXYGEN_RUNNING) +#define RAPIDJSON_SIMD +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NO_SIZETYPEDEFINE + +#ifndef RAPIDJSON_NO_SIZETYPEDEFINE +/*! \def RAPIDJSON_NO_SIZETYPEDEFINE + \ingroup RAPIDJSON_CONFIG + \brief User-provided \c SizeType definition. + + In order to avoid using 32-bit size types for indexing strings and arrays, + define this preprocessor symbol and provide the type rapidjson::SizeType + before including RapidJSON: + \code + #define RAPIDJSON_NO_SIZETYPEDEFINE + namespace rapidjson { typedef ::std::size_t SizeType; } + #include "rapidjson/..." + \endcode + + \see rapidjson::SizeType +*/ +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_NO_SIZETYPEDEFINE +#endif +RAPIDJSON_NAMESPACE_BEGIN +//! Size type (for string lengths, array sizes, etc.) +/*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms, + instead of using \c size_t. Users may override the SizeType by defining + \ref RAPIDJSON_NO_SIZETYPEDEFINE. +*/ +typedef unsigned SizeType; +RAPIDJSON_NAMESPACE_END +#endif + +// always import std::size_t to rapidjson namespace +RAPIDJSON_NAMESPACE_BEGIN +using std::size_t; +RAPIDJSON_NAMESPACE_END + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ASSERT + +//! Assertion. +/*! \ingroup RAPIDJSON_CONFIG + By default, rapidjson uses C \c assert() for internal assertions. + User can override it by defining RAPIDJSON_ASSERT(x) macro. + + \note Parsing errors are handled and can be customized by the + \ref RAPIDJSON_ERRORS APIs. +*/ +#ifndef RAPIDJSON_ASSERT +#include +#define RAPIDJSON_ASSERT(x) assert(x) +#endif // RAPIDJSON_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_STATIC_ASSERT + +// Adopt from boost +#ifndef RAPIDJSON_STATIC_ASSERT +#ifndef __clang__ +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#endif +RAPIDJSON_NAMESPACE_BEGIN +template struct STATIC_ASSERTION_FAILURE; +template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; +template struct StaticAssertTest {}; +RAPIDJSON_NAMESPACE_END + +#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) +#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) +#define RAPIDJSON_DO_JOIN2(X, Y) X##Y + +#if defined(__GNUC__) +#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) +#else +#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif +#ifndef __clang__ +//!@endcond +#endif + +/*! \def RAPIDJSON_STATIC_ASSERT + \brief (Internal) macro to check for conditions at compile-time + \param x compile-time condition + \hideinitializer + */ +#define RAPIDJSON_STATIC_ASSERT(x) \ + typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ + sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ + RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY + +//! Compiler branching hint for expression with high probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression likely to be true. +*/ +#ifndef RAPIDJSON_LIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define RAPIDJSON_LIKELY(x) (x) +#endif +#endif + +//! Compiler branching hint for expression with low probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression unlikely to be true. +*/ +#ifndef RAPIDJSON_UNLIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define RAPIDJSON_UNLIKELY(x) (x) +#endif +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Helpers + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN + +#define RAPIDJSON_MULTILINEMACRO_BEGIN do { +#define RAPIDJSON_MULTILINEMACRO_END \ +} while((void)0, 0) + +// adopted from Boost +#define RAPIDJSON_VERSION_CODE(x,y,z) \ + (((x)*100000) + ((y)*100) + (z)) + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF + +#if defined(__GNUC__) +#define RAPIDJSON_GNUC \ + RAPIDJSON_VERSION_CODE(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) +#endif + +#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,2,0)) + +#define RAPIDJSON_PRAGMA(x) _Pragma(RAPIDJSON_STRINGIFY(x)) +#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(GCC diagnostic x) +#define RAPIDJSON_DIAG_OFF(x) \ + RAPIDJSON_DIAG_PRAGMA(ignored RAPIDJSON_STRINGIFY(RAPIDJSON_JOIN(-W,x))) + +// push/pop support in Clang and GCC>=4.6 +#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) +#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) +#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) +#else // GCC >= 4.2, < 4.6 +#define RAPIDJSON_DIAG_PUSH /* ignored */ +#define RAPIDJSON_DIAG_POP /* ignored */ +#endif + +#elif defined(_MSC_VER) + +// pragma (MSVC specific) +#define RAPIDJSON_PRAGMA(x) __pragma(x) +#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(warning(x)) + +#define RAPIDJSON_DIAG_OFF(x) RAPIDJSON_DIAG_PRAGMA(disable: x) +#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) +#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) + +#else + +#define RAPIDJSON_DIAG_OFF(x) /* ignored */ +#define RAPIDJSON_DIAG_PUSH /* ignored */ +#define RAPIDJSON_DIAG_POP /* ignored */ + +#endif // RAPIDJSON_DIAG_* + +/////////////////////////////////////////////////////////////////////////////// +// C++11 features + +#ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS +#if defined(__clang__) +#if __has_feature(cxx_rvalue_references) && \ + (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1600) + +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + +#ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) +// (defined(_MSC_VER) && _MSC_VER >= ????) // not yet supported +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#else +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 +#endif +#endif +#if RAPIDJSON_HAS_CXX11_NOEXCEPT +#define RAPIDJSON_NOEXCEPT noexcept +#else +#define RAPIDJSON_NOEXCEPT /* noexcept */ +#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT + +// no automatic detection, yet +#ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS +#define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 +#endif + +#ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1700) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 +#else +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 +#endif +#endif // RAPIDJSON_HAS_CXX11_RANGE_FOR + +//!@endcond + +/////////////////////////////////////////////////////////////////////////////// +// new/delete + +#ifndef RAPIDJSON_NEW +///! customization point for global \c new +#define RAPIDJSON_NEW(x) new x +#endif +#ifndef RAPIDJSON_DELETE +///! customization point for global \c delete +#define RAPIDJSON_DELETE(x) delete x +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Type + +/*! \namespace rapidjson + \brief main RapidJSON namespace + \see RAPIDJSON_NAMESPACE +*/ +RAPIDJSON_NAMESPACE_BEGIN + +//! Type of JSON value +enum Type { + kNullType = 0, //!< null + kFalseType = 1, //!< false + kTrueType = 2, //!< true + kObjectType = 3, //!< object + kArrayType = 4, //!< array + kStringType = 5, //!< string + kNumberType = 6 //!< number +}; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/reader.h b/ext/librethinkdbxx/src/rapidjson/reader.h new file mode 100644 index 00000000..19f8849b --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/reader.h @@ -0,0 +1,1879 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_READER_H_ +#define RAPIDJSON_READER_H_ + +/*! \file reader.h */ + +#include "allocators.h" +#include "stream.h" +#include "encodedstream.h" +#include "internal/meta.h" +#include "internal/stack.h" +#include "internal/strtod.h" +#include + +#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef RAPIDJSON_SSE42 +#include +#elif defined(RAPIDJSON_SSE2) +#include +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +RAPIDJSON_DIAG_OFF(4702) // unreachable code +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(old-style-cast) +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define RAPIDJSON_NOTHING /* deliberately empty */ +#ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN +#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + if (RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ + RAPIDJSON_MULTILINEMACRO_END +#endif +#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(RAPIDJSON_NOTHING) +//!@endcond + +/*! \def RAPIDJSON_PARSE_ERROR_NORETURN + \ingroup RAPIDJSON_ERRORS + \brief Macro to indicate a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + This macros can be used as a customization point for the internal + error handling mechanism of RapidJSON. + + A common usage model is to throw an exception instead of requiring the + caller to explicitly check the \ref rapidjson::GenericReader::Parse's + return value: + + \code + #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ + throw ParseException(parseErrorCode, #parseErrorCode, offset) + + #include // std::runtime_error + #include "rapidjson/error/error.h" // rapidjson::ParseResult + + struct ParseException : std::runtime_error, rapidjson::ParseResult { + ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) + : std::runtime_error(msg), ParseResult(code, offset) {} + }; + + #include "rapidjson/reader.h" + \endcode + + \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse + */ +#ifndef RAPIDJSON_PARSE_ERROR_NORETURN +#define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + RAPIDJSON_ASSERT(!HasParseError()); /* Error can only be assigned once */ \ + SetParseError(parseErrorCode, offset); \ + RAPIDJSON_MULTILINEMACRO_END +#endif + +/*! \def RAPIDJSON_PARSE_ERROR + \ingroup RAPIDJSON_ERRORS + \brief (Internal) macro to indicate and handle a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. + + \see RAPIDJSON_PARSE_ERROR_NORETURN + \hideinitializer + */ +#ifndef RAPIDJSON_PARSE_ERROR +#define RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset); \ + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; \ + RAPIDJSON_MULTILINEMACRO_END +#endif + +#include "error/error.h" // ParseErrorCode, ParseResult + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// ParseFlag + +/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS + \ingroup RAPIDJSON_CONFIG + \brief User-defined kParseDefaultFlags definition. + + User can define this as any \c ParseFlag combinations. +*/ +#ifndef RAPIDJSON_PARSE_DEFAULT_FLAGS +#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseNoFlags +#endif + +//! Combination of parseFlags +/*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream + */ +enum ParseFlag { + kParseNoFlags = 0, //!< No flags are set. + kParseInsituFlag = 1, //!< In-situ(destructive) parsing. + kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. + kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. + kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. + kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). + kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. + kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. + kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. + kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. + kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS +}; + +/////////////////////////////////////////////////////////////////////////////// +// Handler + +/*! \class rapidjson::Handler + \brief Concept for receiving events from GenericReader upon parsing. + The functions return true if no error occurs. If they return false, + the event publisher should terminate the process. +\code +concept Handler { + typename Ch; + + bool Null(); + bool Bool(bool b); + bool Int(int i); + bool Uint(unsigned i); + bool Int64(int64_t i); + bool Uint64(uint64_t i); + bool Double(double d); + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType length, bool copy); + bool String(const Ch* str, SizeType length, bool copy); + bool StartObject(); + bool Key(const Ch* str, SizeType length, bool copy); + bool EndObject(SizeType memberCount); + bool StartArray(); + bool EndArray(SizeType elementCount); +}; +\endcode +*/ +/////////////////////////////////////////////////////////////////////////////// +// BaseReaderHandler + +//! Default implementation of Handler. +/*! This can be used as base class of any reader handler. + \note implements Handler concept +*/ +template, typename Derived = void> +struct BaseReaderHandler { + typedef typename Encoding::Ch Ch; + + typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; + + bool Default() { return true; } + bool Null() { return static_cast(*this).Default(); } + bool Bool(bool) { return static_cast(*this).Default(); } + bool Int(int) { return static_cast(*this).Default(); } + bool Uint(unsigned) { return static_cast(*this).Default(); } + bool Int64(int64_t) { return static_cast(*this).Default(); } + bool Uint64(uint64_t) { return static_cast(*this).Default(); } + bool Double(double) { return static_cast(*this).Default(); } + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } + bool StartObject() { return static_cast(*this).Default(); } + bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool EndObject(SizeType) { return static_cast(*this).Default(); } + bool StartArray() { return static_cast(*this).Default(); } + bool EndArray(SizeType) { return static_cast(*this).Default(); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// StreamLocalCopy + +namespace internal { + +template::copyOptimization> +class StreamLocalCopy; + +//! Do copy optimization. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original), original_(original) {} + ~StreamLocalCopy() { original_ = s; } + + Stream s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; + + Stream& original_; +}; + +//! Keep reference. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original) {} + + Stream& s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// SkipWhitespace + +//! Skip the JSON white spaces in a stream. +/*! \param is A input stream for skipping white spaces. + \note This function has SSE2/SSE4.2 specialization. +*/ +template +void SkipWhitespace(InputStream& is) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + typename InputStream::Ch c; + while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') + s.Take(); +} + +inline const char* SkipWhitespace(const char* p, const char* end) { + while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + return p; +} + +#ifdef RAPIDJSON_SSE42 +//! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) { // some of characters is non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The middle of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) { // some of characters is non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } + + return SkipWhitespace(p, end); +} + +#elif defined(RAPIDJSON_SSE2) + +//! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } + + return SkipWhitespace(p, end); +} + +#endif // RAPIDJSON_SSE2 + +#ifdef RAPIDJSON_SIMD +//! Template function specialization for InsituStringStream +template<> inline void SkipWhitespace(InsituStringStream& is) { + is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); +} + +//! Template function specialization for StringStream +template<> inline void SkipWhitespace(StringStream& is) { + is.src_ = SkipWhitespace_SIMD(is.src_); +} + +template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { + is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); +} +#endif // RAPIDJSON_SIMD + +/////////////////////////////////////////////////////////////////////////////// +// GenericReader + +//! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. +/*! GenericReader parses JSON text from a stream, and send events synchronously to an + object implementing Handler concept. + + It needs to allocate a stack for storing a single decoded string during + non-destructive parsing. + + For in-situ parsing, the decoded string is directly written to the source + text string, no temporary buffer is required. + + A GenericReader object can be reused for parsing multiple JSON text. + + \tparam SourceEncoding Encoding of the input stream. + \tparam TargetEncoding Encoding of the parse output. + \tparam StackAllocator Allocator type for stack. +*/ +template +class GenericReader { +public: + typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type + + //! Constructor. + /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) + \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) + */ + GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} + + //! Parse JSON text. + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + if (parseFlags & kParseIterativeFlag) + return IterativeParse(is, handler); + + parseResult_.Clear(); + + ClearStackOnExit scope(*this); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + else { + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (!(parseFlags & kParseStopWhenDoneFlag)) { + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + } + } + + return parseResult_; + } + + //! Parse JSON text (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + return Parse(is, handler); + } + + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + +protected: + void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } + +private: + // Prohibit copy constructor & assignment operator. + GenericReader(const GenericReader&); + GenericReader& operator=(const GenericReader&); + + void ClearStack() { stack_.Clear(); } + + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericReader& r) : r_(r) {} + ~ClearStackOnExit() { r_.ClearStack(); } + private: + GenericReader& r_; + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + }; + + template + void SkipWhitespaceAndComments(InputStream& is) { + SkipWhitespace(is); + + if (parseFlags & kParseCommentsFlag) { + while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { + if (Consume(is, '*')) { + while (true) { + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + else if (Consume(is, '*')) { + if (Consume(is, '/')) + break; + } + else + is.Take(); + } + } + else if (RAPIDJSON_LIKELY(Consume(is, '/'))) + while (is.Peek() != '\0' && is.Take() != '\n'); + else + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + SkipWhitespace(is); + } + } + } + + // Parse object: { string : value, ... } + template + void ParseObject(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == '{'); + is.Take(); // Skip '{' + + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, '}')) { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType memberCount = 0;;) { + if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + + ParseString(is, handler, true); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ++memberCount; + + switch (is.Peek()) { + case ',': + is.Take(); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; + case '}': + is.Take(); + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + default: + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy + } + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == '}') { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } + } + } + } + + // Parse array: [ value, ... ] + template + void ParseArray(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == '['); + is.Take(); // Skip '[' + + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType elementCount = 0;;) { + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ++elementCount; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ',')) { + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + } + else if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == ']') { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } + } + } + } + + template + void ParseNull(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 'n'); + is.Take(); + + if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { + if (RAPIDJSON_UNLIKELY(!handler.Null())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + void ParseTrue(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 't'); + is.Take(); + + if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + void ParseFalse(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 'f'); + is.Take(); + + if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { + if (RAPIDJSON_LIKELY(is.Peek() == expect)) { + is.Take(); + return true; + } + else + return false; + } + + // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). + template + unsigned ParseHex4(InputStream& is, size_t escapeOffset) { + unsigned codepoint = 0; + for (int i = 0; i < 4; i++) { + Ch c = is.Peek(); + codepoint <<= 4; + codepoint += static_cast(c); + if (c >= '0' && c <= '9') + codepoint -= '0'; + else if (c >= 'A' && c <= 'F') + codepoint -= 'A' - 10; + else if (c >= 'a' && c <= 'f') + codepoint -= 'a' - 10; + else { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); + } + is.Take(); + } + return codepoint; + } + + template + class StackStream { + public: + typedef CharType Ch; + + StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} + RAPIDJSON_FORCEINLINE void Put(Ch c) { + *stack_.template Push() = c; + ++length_; + } + + RAPIDJSON_FORCEINLINE void* Push(SizeType count) { + length_ += count; + return stack_.template Push(count); + } + + size_t Length() const { return length_; } + + Ch* Pop() { + return stack_.template Pop(length_); + } + + private: + StackStream(const StackStream&); + StackStream& operator=(const StackStream&); + + internal::Stack& stack_; + SizeType length_; + }; + + // Parse string and generate String event. Different code paths for kParseInsituFlag. + template + void ParseString(InputStream& is, Handler& handler, bool isKey = false) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + RAPIDJSON_ASSERT(s.Peek() == '\"'); + s.Take(); // Skip '\"' + + bool success = false; + if (parseFlags & kParseInsituFlag) { + typename InputStream::Ch *head = s.PutBegin(); + ParseStringToStream(s, s); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + size_t length = s.PutEnd(head) - 1; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); + } + else { + StackStream stackStream(stack_); + ParseStringToStream(s, stackStream); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SizeType length = static_cast(stackStream.Length()) - 1; + const typename TargetEncoding::Ch* const str = stackStream.Pop(); + success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); + } + if (RAPIDJSON_UNLIKELY(!success)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + } + + // Parse string to an output is + // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. + template + RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + static const char escape[256] = { + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 + }; +#undef Z16 +//!@endcond + + for (;;) { + // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. + if (!(parseFlags & kParseValidateEncodingFlag)) + ScanCopyUnescapedString(is, os); + + Ch c = is.Peek(); + if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape + size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset + is.Take(); + Ch e = is.Peek(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { + is.Take(); + os.Put(static_cast(escape[static_cast(e)])); + } + else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode + is.Take(); + unsigned codepoint = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { + // Handle UTF-16 surrogate pair + if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + unsigned codepoint2 = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; + } + TEncoding::Encode(os, codepoint); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); + } + else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote + is.Take(); + os.Put('\0'); // null-terminate the string + return; + } + else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell()); + } + else { + size_t offset = is.Tell(); + if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); + } + } + } + + template + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { + // Do nothing for generic version + } + +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType length; + #ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; + #else + length = static_cast(__builtin_ffs(r) - 1); + #endif + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16, q += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + for (const char* pend = p + length; p != pend; ) + *q++ = *p++; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + p += length; + break; + } + } + + is.src_ = is.dst_ = p; + } +#endif + + template + class NumberStream; + + template + class NumberStream { + public: + typedef typename InputStream::Ch Ch; + + NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } + ~NumberStream() {} + + RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } + RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } + RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char) {} + + size_t Tell() { return is.Tell(); } + size_t Length() { return 0; } + const char* Pop() { return 0; } + + protected: + NumberStream& operator=(const NumberStream&); + + InputStream& is; + }; + + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} + ~NumberStream() {} + + RAPIDJSON_FORCEINLINE Ch TakePush() { + stackStream.Put(static_cast(Base::is.Peek())); + return Base::is.Take(); + } + + RAPIDJSON_FORCEINLINE void Push(char c) { + stackStream.Put(c); + } + + size_t Length() { return stackStream.Length(); } + + const char* Pop() { + stackStream.Put('\0'); + return stackStream.Pop(); + } + + private: + StackStream stackStream; + }; + + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + ~NumberStream() {} + + RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } + }; + + template + void ParseNumber(InputStream& is, Handler& handler) { + internal::StreamLocalCopy copy(is); + NumberStream s(*this, copy.s); + + size_t startOffset = s.Tell(); + double d = 0.0; + bool useNanOrInf = false; + + // Parse minus + bool minus = Consume(s, '-'); + + // Parse int: zero / ( digit1-9 *DIGIT ) + unsigned i = 0; + uint64_t i64 = 0; + bool use64bit = false; + int significandDigit = 0; + if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { + i = 0; + s.TakePush(); + } + else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { + i = static_cast(s.TakePush() - '0'); + + if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 + if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + else + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 + if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + } + // Parse NaN or Infinity here + else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { + useNanOrInf = true; + if (RAPIDJSON_LIKELY(Consume(s, 'N') && Consume(s, 'a') && Consume(s, 'N'))) { + d = std::numeric_limits::quiet_NaN(); + } + else if (RAPIDJSON_LIKELY(Consume(s, 'I') && Consume(s, 'n') && Consume(s, 'f'))) { + d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') + && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + + // Parse 64bit int + bool useDouble = false; + if (use64bit) { + if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { + d = static_cast(i64); + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + else + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { + d = static_cast(i64); + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + } + + // Force double for big integer + if (useDouble) { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + d = d * 10 + (s.TakePush() - '0'); + } + } + + // Parse frac = decimal-point 1*DIGIT + int expFrac = 0; + size_t decimalPosition; + if (Consume(s, '.')) { + decimalPosition = s.Length(); + + if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); + + if (!useDouble) { +#if RAPIDJSON_64BIT + // Use i64 to store significand in 64-bit architecture + if (!use64bit) + i64 = i; + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path + break; + else { + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + --expFrac; + if (i64 != 0) + significandDigit++; + } + } + + d = static_cast(i64); +#else + // Use double to store significand in 32-bit architecture + d = static_cast(use64bit ? i64 : i); +#endif + useDouble = true; + } + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (significandDigit < 17) { + d = d * 10.0 + (s.TakePush() - '0'); + --expFrac; + if (RAPIDJSON_LIKELY(d > 0.0)) + significandDigit++; + } + else + s.TakePush(); + } + } + else + decimalPosition = s.Length(); // decimal position at the end of integer. + + // Parse exp = e [ minus / plus ] 1*DIGIT + int exp = 0; + if (Consume(s, 'e') || Consume(s, 'E')) { + if (!useDouble) { + d = static_cast(use64bit ? i64 : i); + useDouble = true; + } + + bool expMinus = false; + if (Consume(s, '+')) + ; + else if (Consume(s, '-')) + expMinus = true; + + if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = static_cast(s.Take() - '0'); + if (expMinus) { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (exp >= 214748364) { // Issue #313: prevent overflow exponent + while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent + s.Take(); + } + } + } + else { // positive exp + int maxExp = 308 - expFrac; + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (RAPIDJSON_UNLIKELY(exp > maxExp)) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + } + } + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); + + if (expMinus) + exp = -exp; + } + + // Finish parsing, call event according to the type of number. + bool cont = true; + + if (parseFlags & kParseNumbersAsStringsFlag) { + if (parseFlags & kParseInsituFlag) { + s.Pop(); // Pop stack no matter if it will be used or not. + typename InputStream::Ch* head = is.PutBegin(); + const size_t length = s.Tell() - startOffset; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + // unable to insert the \0 character here, it will erase the comma after this number + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + cont = handler.RawNumber(str, SizeType(length), false); + } + else { + SizeType numCharsToCopy = static_cast(s.Length()); + StringStream srcStream(s.Pop()); + StackStream dstStream(stack_); + while (numCharsToCopy--) { + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + } + dstStream.Put('\0'); + const typename TargetEncoding::Ch* str = dstStream.Pop(); + const SizeType length = static_cast(dstStream.Length()) - 1; + cont = handler.RawNumber(str, SizeType(length), true); + } + } + else { + size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + + if (useDouble) { + int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + cont = handler.Double(minus ? -d : d); + } + else if (useNanOrInf) { + cont = handler.Double(d); + } + else { + if (use64bit) { + if (minus) + cont = handler.Int64(static_cast(~i64 + 1)); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(static_cast(~i + 1)); + else + cont = handler.Uint(i); + } + } + } + if (RAPIDJSON_UNLIKELY(!cont)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); + } + + // Parse any JSON value + template + void ParseValue(InputStream& is, Handler& handler) { + switch (is.Peek()) { + case 'n': ParseNull (is, handler); break; + case 't': ParseTrue (is, handler); break; + case 'f': ParseFalse (is, handler); break; + case '"': ParseString(is, handler); break; + case '{': ParseObject(is, handler); break; + case '[': ParseArray (is, handler); break; + default : + ParseNumber(is, handler); + break; + + } + } + + // Iterative Parsing + + // States + enum IterativeParsingState { + IterativeParsingStartState = 0, + IterativeParsingFinishState, + IterativeParsingErrorState, + + // Object states + IterativeParsingObjectInitialState, + IterativeParsingMemberKeyState, + IterativeParsingKeyValueDelimiterState, + IterativeParsingMemberValueState, + IterativeParsingMemberDelimiterState, + IterativeParsingObjectFinishState, + + // Array states + IterativeParsingArrayInitialState, + IterativeParsingElementState, + IterativeParsingElementDelimiterState, + IterativeParsingArrayFinishState, + + // Single value state + IterativeParsingValueState + }; + + enum { cIterativeParsingStateCount = IterativeParsingValueState + 1 }; + + // Tokens + enum Token { + LeftBracketToken = 0, + RightBracketToken, + + LeftCurlyBracketToken, + RightCurlyBracketToken, + + CommaToken, + ColonToken, + + StringToken, + FalseToken, + TrueToken, + NullToken, + NumberToken, + + kTokenCount + }; + + RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define N NumberToken +#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N + // Maps from ASCII to Token + static const unsigned char tokenMap[256] = { + N16, // 00~0F + N16, // 10~1F + N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F + N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F + N16, // 40~4F + N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F + N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F + N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F + N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF + }; +#undef N +#undef N16 +//!@endcond + + if (sizeof(Ch) == 1 || static_cast(c) < 256) + return static_cast(tokenMap[static_cast(c)]); + else + return NumberToken; + } + + RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { + // current state x one lookahead token -> new state + static const char G[cIterativeParsingStateCount][kTokenCount] = { + // Start + { + IterativeParsingArrayInitialState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingValueState, // String + IterativeParsingValueState, // False + IterativeParsingValueState, // True + IterativeParsingValueState, // Null + IterativeParsingValueState // Number + }, + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ObjectInitial + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberKey + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingKeyValueDelimiterState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // KeyValueDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, + // MemberValue + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingMemberDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberDelimiter + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ObjectFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ArrayInitial + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // Element + { + IterativeParsingErrorState, // Left bracket + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingElementDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ElementDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // ArrayFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Single Value (sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + } + }; // End of G + + return static_cast(G[state][token]); + } + + // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). + // May return a new state on state pop. + template + RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { + (void)token; + + switch (dst) { + case IterativeParsingErrorState: + return dst; + + case IterativeParsingObjectInitialState: + case IterativeParsingArrayInitialState: + { + // Push the state(Element or MemeberValue) if we are nested in another array or value of member. + // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. + IterativeParsingState n = src; + if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) + n = IterativeParsingElementState; + else if (src == IterativeParsingKeyValueDelimiterState) + n = IterativeParsingMemberValueState; + // Push current state. + *stack_.template Push(1) = n; + // Initialize and push the member/element count. + *stack_.template Push(1) = 0; + // Call handler + bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return dst; + } + } + + case IterativeParsingMemberKeyState: + ParseString(is, handler, true); + if (HasParseError()) + return IterativeParsingErrorState; + else + return dst; + + case IterativeParsingKeyValueDelimiterState: + RAPIDJSON_ASSERT(token == ColonToken); + is.Take(); + return dst; + + case IterativeParsingMemberValueState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingElementState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingMemberDelimiterState: + case IterativeParsingElementDelimiterState: + is.Take(); + // Update member/element count. + *stack_.template Top() = *stack_.template Top() + 1; + return dst; + + case IterativeParsingObjectFinishState: + { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); + return IterativeParsingErrorState; + } + // Get member count. + SizeType c = *stack_.template Pop(1); + // If the object is not empty, count the last member. + if (src == IterativeParsingMemberValueState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndObject(c); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + case IterativeParsingArrayFinishState: + { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); + return IterativeParsingErrorState; + } + // Get element count. + SizeType c = *stack_.template Pop(1); + // If the array is not empty, count the last element. + if (src == IterativeParsingElementState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndArray(c); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + default: + // This branch is for IterativeParsingValueState actually. + // Use `default:` rather than + // `case IterativeParsingValueState:` is for code coverage. + + // The IterativeParsingStartState is not enumerated in this switch-case. + // It is impossible for that case. And it can be caught by following assertion. + + // The IterativeParsingFinishState is not enumerated in this switch-case either. + // It is a "derivative" state which cannot triggered from Predict() directly. + // Therefore it cannot happen here. And it can be caught by following assertion. + RAPIDJSON_ASSERT(dst == IterativeParsingValueState); + + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return IterativeParsingFinishState; + } + } + + template + void HandleError(IterativeParsingState src, InputStream& is) { + if (HasParseError()) { + // Error flag has been set. + return; + } + + switch (src) { + case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; + case IterativeParsingObjectInitialState: + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; + case IterativeParsingKeyValueDelimiterState: + case IterativeParsingArrayInitialState: + case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; + default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; + } + } + + template + ParseResult IterativeParse(InputStream& is, Handler& handler) { + parseResult_.Clear(); + ClearStackOnExit scope(*this); + IterativeParsingState state = IterativeParsingStartState; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + while (is.Peek() != '\0') { + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state, t); + IterativeParsingState d = Transit(state, t, n, is, handler); + + if (d == IterativeParsingErrorState) { + HandleError(state, is); + break; + } + + state = d; + + // Do not further consume streams if a root JSON has been parsed. + if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) + break; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + + // Handle the end of file. + if (state != IterativeParsingFinishState) + HandleError(state, is); + + return parseResult_; + } + + static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. + internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. + ParseResult parseResult_; +}; // class GenericReader + +//! Reader with UTF8 encoding and default allocator. +typedef GenericReader, UTF8<> > Reader; + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_READER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/schema.h b/ext/librethinkdbxx/src/rapidjson/schema.h new file mode 100644 index 00000000..b182aa27 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/schema.h @@ -0,0 +1,2006 @@ +// Tencent is pleased to support the open source community by making RapidJSON available-> +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License-> You may obtain a copy of the License at +// +// http://opensource->org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied-> See the License for the +// specific language governing permissions and limitations under the License-> + +#ifndef RAPIDJSON_SCHEMA_H_ +#define RAPIDJSON_SCHEMA_H_ + +#include "document.h" +#include "pointer.h" +#include // abs, floor + +#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 +#endif + +#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) +#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX +#include "internal/regex.h" +#elif RAPIDJSON_SCHEMA_USE_STDREGEX +#include +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX +#define RAPIDJSON_SCHEMA_HAS_REGEX 1 +#else +#define RAPIDJSON_SCHEMA_HAS_REGEX 0 +#endif + +#ifndef RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_VERBOSE 0 +#endif + +#if RAPIDJSON_SCHEMA_VERBOSE +#include "stringbuffer.h" +#endif + +RAPIDJSON_DIAG_PUSH + +#if defined(__GNUC__) +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(weak-vtables) +RAPIDJSON_DIAG_OFF(exit-time-destructors) +RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) +RAPIDJSON_DIAG_OFF(variadic-macros) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Verbose Utilities + +#if RAPIDJSON_SCHEMA_VERBOSE + +namespace internal { + +inline void PrintInvalidKeyword(const char* keyword) { + printf("Fail keyword: %s\n", keyword); +} + +inline void PrintInvalidKeyword(const wchar_t* keyword) { + wprintf(L"Fail keyword: %ls\n", keyword); +} + +inline void PrintInvalidDocument(const char* document) { + printf("Fail document: %s\n\n", document); +} + +inline void PrintInvalidDocument(const wchar_t* document) { + wprintf(L"Fail document: %ls\n\n", document); +} + +inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { + printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +} + +inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { + wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +} + +} // namespace internal + +#endif // RAPIDJSON_SCHEMA_VERBOSE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_INVALID_KEYWORD_RETURN + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) +#else +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) +#endif + +#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + context.invalidKeyword = keyword.GetString();\ + RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ + return false;\ +RAPIDJSON_MULTILINEMACRO_END + +/////////////////////////////////////////////////////////////////////////////// +// Forward declarations + +template +class GenericSchemaDocument; + +namespace internal { + +template +class Schema; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaValidator + +class ISchemaValidator { +public: + virtual ~ISchemaValidator() {} + virtual bool IsValid() const = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaStateFactory + +template +class ISchemaStateFactory { +public: + virtual ~ISchemaStateFactory() {} + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; + virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; + virtual void* CreateHasher() = 0; + virtual uint64_t GetHashCode(void* hasher) = 0; + virtual void DestroryHasher(void* hasher) = 0; + virtual void* MallocState(size_t size) = 0; + virtual void FreeState(void* p) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Hasher + +// For comparison of compound value +template +class Hasher { +public: + typedef typename Encoding::Ch Ch; + + Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} + + bool Null() { return WriteType(kNullType); } + bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } + bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Double(double d) { + Number n; + if (d < 0) n.u.i = static_cast(d); + else n.u.u = static_cast(d); + n.d = d; + return WriteNumber(n); + } + + bool RawNumber(const Ch* str, SizeType len, bool) { + WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + + bool String(const Ch* str, SizeType len, bool) { + WriteBuffer(kStringType, str, len * sizeof(Ch)); + return true; + } + + bool StartObject() { return true; } + bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } + bool EndObject(SizeType memberCount) { + uint64_t h = Hash(0, kObjectType); + uint64_t* kv = stack_.template Pop(memberCount * 2); + for (SizeType i = 0; i < memberCount; i++) + h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + *stack_.template Push() = h; + return true; + } + + bool StartArray() { return true; } + bool EndArray(SizeType elementCount) { + uint64_t h = Hash(0, kArrayType); + uint64_t* e = stack_.template Pop(elementCount); + for (SizeType i = 0; i < elementCount; i++) + h = Hash(h, e[i]); // Use hash to achieve element order sensitive + *stack_.template Push() = h; + return true; + } + + bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } + + uint64_t GetHashCode() const { + RAPIDJSON_ASSERT(IsValid()); + return *stack_.template Top(); + } + +private: + static const size_t kDefaultSize = 256; + struct Number { + union U { + uint64_t u; + int64_t i; + }u; + double d; + }; + + bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } + + bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } + + bool WriteBuffer(Type type, const void* data, size_t len) { + // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ + uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + const unsigned char* d = static_cast(data); + for (size_t i = 0; i < len; i++) + h = Hash(h, d[i]); + *stack_.template Push() = h; + return true; + } + + static uint64_t Hash(uint64_t h, uint64_t d) { + static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); + h ^= d; + h *= kPrime; + return h; + } + + Stack stack_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidationContext + +template +struct SchemaValidationContext { + typedef Schema SchemaType; + typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef typename SchemaType::ValueType ValueType; + typedef typename ValueType::Ch Ch; + + enum PatternValidatorType { + kPatternValidatorOnly, + kPatternValidatorWithProperty, + kPatternValidatorWithAdditionalProperty + }; + + SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : + factory(f), + schema(s), + valueSchema(), + invalidKeyword(), + hasher(), + arrayElementHashCodes(), + validators(), + validatorCount(), + patternPropertiesValidators(), + patternPropertiesValidatorCount(), + patternPropertiesSchemas(), + patternPropertiesSchemaCount(), + valuePatternValidatorType(kPatternValidatorOnly), + propertyExist(), + inArray(false), + valueUniqueness(false), + arrayUniqueness(false) + { + } + + ~SchemaValidationContext() { + if (hasher) + factory.DestroryHasher(hasher); + if (validators) { + for (SizeType i = 0; i < validatorCount; i++) + factory.DestroySchemaValidator(validators[i]); + factory.FreeState(validators); + } + if (patternPropertiesValidators) { + for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + factory.FreeState(patternPropertiesValidators); + } + if (patternPropertiesSchemas) + factory.FreeState(patternPropertiesSchemas); + if (propertyExist) + factory.FreeState(propertyExist); + } + + SchemaValidatorFactoryType& factory; + const SchemaType* schema; + const SchemaType* valueSchema; + const Ch* invalidKeyword; + void* hasher; // Only validator access + void* arrayElementHashCodes; // Only validator access this + ISchemaValidator** validators; + SizeType validatorCount; + ISchemaValidator** patternPropertiesValidators; + SizeType patternPropertiesValidatorCount; + const SchemaType** patternPropertiesSchemas; + SizeType patternPropertiesSchemaCount; + PatternValidatorType valuePatternValidatorType; + PatternValidatorType objectPatternValidatorType; + SizeType arrayElementIndex; + bool* propertyExist; + bool inArray; + bool valueUniqueness; + bool arrayUniqueness; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Schema + +template +class Schema { +public: + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename SchemaDocumentType::AllocatorType AllocatorType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef SchemaValidationContext Context; + typedef Schema SchemaType; + typedef GenericValue SValue; + friend class GenericSchemaDocument; + + Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : + allocator_(allocator), + enum_(), + enumCount_(), + not_(), + type_((1 << kTotalSchemaType) - 1), // typeless + validatorCount_(), + properties_(), + additionalPropertiesSchema_(), + patternProperties_(), + patternPropertyCount_(), + propertyCount_(), + minProperties_(), + maxProperties_(SizeType(~0)), + additionalProperties_(true), + hasDependencies_(), + hasRequired_(), + hasSchemaDependencies_(), + additionalItemsSchema_(), + itemsList_(), + itemsTuple_(), + itemsTupleCount_(), + minItems_(), + maxItems_(SizeType(~0)), + additionalItems_(true), + uniqueItems_(false), + pattern_(), + minLength_(0), + maxLength_(~SizeType(0)), + exclusiveMinimum_(false), + exclusiveMaximum_(false) + { + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename ValueType::ConstValueIterator ConstValueIterator; + typedef typename ValueType::ConstMemberIterator ConstMemberIterator; + + if (!value.IsObject()) + return; + + if (const ValueType* v = GetMember(value, GetTypeString())) { + type_ = 0; + if (v->IsString()) + AddType(*v); + else if (v->IsArray()) + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + AddType(*itr); + } + + if (const ValueType* v = GetMember(value, GetEnumString())) + if (v->IsArray() && v->Size() > 0) { + enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { + typedef Hasher > EnumHasherType; + char buffer[256 + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); + enum_[enumCount_++] = h.GetHashCode(); + } + } + + if (schemaDocument) { + AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); + AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); + } + + if (const ValueType* v = GetMember(value, GetNotString())) { + schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); + notValidatorIndex_ = validatorCount_; + validatorCount_++; + } + + // Object + + const ValueType* properties = GetMember(value, GetPropertiesString()); + const ValueType* required = GetMember(value, GetRequiredString()); + const ValueType* dependencies = GetMember(value, GetDependenciesString()); + { + // Gather properties from properties/required/dependencies + SValue allProperties(kArrayType); + + if (properties && properties->IsObject()) + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + AddUniqueElement(allProperties, itr->name); + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + AddUniqueElement(allProperties, *itr); + + if (dependencies && dependencies->IsObject()) + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + AddUniqueElement(allProperties, itr->name); + if (itr->value.IsArray()) + for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) + if (i->IsString()) + AddUniqueElement(allProperties, *i); + } + + if (allProperties.Size() > 0) { + propertyCount_ = allProperties.Size(); + properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); + for (SizeType i = 0; i < propertyCount_; i++) { + new (&properties_[i]) Property(); + properties_[i].name = allProperties[i]; + properties_[i].schema = GetTypeless(); + } + } + } + + if (properties && properties->IsObject()) { + PointerType q = p.Append(GetPropertiesString(), allocator_); + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { + SizeType index; + if (FindPropertyIndex(itr->name, &index)) + schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); + } + } + + if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { + PointerType q = p.Append(GetPatternPropertiesString(), allocator_); + patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); + patternPropertyCount_ = 0; + + for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { + new (&patternProperties_[patternPropertyCount_]) PatternProperty(); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); + patternPropertyCount_++; + } + } + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) { + SizeType index; + if (FindPropertyIndex(*itr, &index)) { + properties_[index].required = true; + hasRequired_ = true; + } + } + + if (dependencies && dependencies->IsObject()) { + PointerType q = p.Append(GetDependenciesString(), allocator_); + hasDependencies_ = true; + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + SizeType sourceIndex; + if (FindPropertyIndex(itr->name, &sourceIndex)) { + if (itr->value.IsArray()) { + properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); + std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); + for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { + SizeType targetIndex; + if (FindPropertyIndex(*targetItr, &targetIndex)) + properties_[sourceIndex].dependencies[targetIndex] = true; + } + } + else if (itr->value.IsObject()) { + hasSchemaDependencies_ = true; + schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); + properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; + validatorCount_++; + } + } + } + } + + if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { + if (v->IsBool()) + additionalProperties_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); + } + + AssignIfExist(minProperties_, value, GetMinPropertiesString()); + AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); + + // Array + if (const ValueType* v = GetMember(value, GetItemsString())) { + PointerType q = p.Append(GetItemsString(), allocator_); + if (v->IsObject()) // List validation + schemaDocument->CreateSchema(&itemsList_, q, *v, document); + else if (v->IsArray()) { // Tuple validation + itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); + SizeType index = 0; + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) + schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + } + } + + AssignIfExist(minItems_, value, GetMinItemsString()); + AssignIfExist(maxItems_, value, GetMaxItemsString()); + + if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { + if (v->IsBool()) + additionalItems_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); + } + + AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); + + // String + AssignIfExist(minLength_, value, GetMinLengthString()); + AssignIfExist(maxLength_, value, GetMaxLengthString()); + + if (const ValueType* v = GetMember(value, GetPatternString())) + pattern_ = CreatePattern(*v); + + // Number + if (const ValueType* v = GetMember(value, GetMinimumString())) + if (v->IsNumber()) + minimum_.CopyFrom(*v, *allocator_); + + if (const ValueType* v = GetMember(value, GetMaximumString())) + if (v->IsNumber()) + maximum_.CopyFrom(*v, *allocator_); + + AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); + AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); + + if (const ValueType* v = GetMember(value, GetMultipleOfString())) + if (v->IsNumber() && v->GetDouble() > 0.0) + multipleOf_.CopyFrom(*v, *allocator_); + } + + ~Schema() { + if (allocator_) { + allocator_->Free(enum_); + } + if (properties_) { + for (SizeType i = 0; i < propertyCount_; i++) + properties_[i].~Property(); + AllocatorType::Free(properties_); + } + if (patternProperties_) { + for (SizeType i = 0; i < patternPropertyCount_; i++) + patternProperties_[i].~PatternProperty(); + AllocatorType::Free(patternProperties_); + } + AllocatorType::Free(itemsTuple_); +#if RAPIDJSON_SCHEMA_HAS_REGEX + if (pattern_) { + pattern_->~RegexType(); + allocator_->Free(pattern_); + } +#endif + } + + bool BeginValue(Context& context) const { + if (context.inArray) { + if (uniqueItems_) + context.valueUniqueness = true; + + if (itemsList_) + context.valueSchema = itemsList_; + else if (itemsTuple_) { + if (context.arrayElementIndex < itemsTupleCount_) + context.valueSchema = itemsTuple_[context.arrayElementIndex]; + else if (additionalItemsSchema_) + context.valueSchema = additionalItemsSchema_; + else if (additionalItems_) + context.valueSchema = GetTypeless(); + else + RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } + else + context.valueSchema = GetTypeless(); + + context.arrayElementIndex++; + } + return true; + } + + RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { + if (context.patternPropertiesValidatorCount > 0) { + bool otherValid = false; + SizeType count = context.patternPropertiesValidatorCount; + if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) + otherValid = context.patternPropertiesValidators[--count]->IsValid(); + + bool patternValid = true; + for (SizeType i = 0; i < count; i++) + if (!context.patternPropertiesValidators[i]->IsValid()) { + patternValid = false; + break; + } + + if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { + if (!patternValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { + if (!patternValid || !otherValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + + if (enum_) { + const uint64_t h = context.factory.GetHashCode(context.hasher); + for (SizeType i = 0; i < enumCount_; i++) + if (enum_[i] == h) + goto foundEnum; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); + foundEnum:; + } + + if (allOf_.schemas) + for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) + if (!context.validators[i]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + + if (anyOf_.schemas) { + for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) + if (context.validators[i]->IsValid()) + goto foundAny; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); + foundAny:; + } + + if (oneOf_.schemas) { + bool oneValid = false; + for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) + if (context.validators[i]->IsValid()) { + if (oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + else + oneValid = true; + } + if (!oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } + + if (not_ && context.validators[notValidatorIndex_]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + + return true; + } + + bool Null(Context& context) const { + if (!(type_ & (1 << kNullSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Bool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Int(Context& context, int i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint(Context& context, unsigned u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Int64(Context& context, int64_t i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint64(Context& context, uint64_t u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Double(Context& context, double d) const { + if (!(type_ & (1 << kNumberSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) + return false; + + if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) + return false; + + if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) + return false; + + return CreateParallelValidator(context); + } + + bool String(Context& context, const Ch* str, SizeType length, bool) const { + if (!(type_ & (1 << kStringSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (minLength_ != 0 || maxLength_ != SizeType(~0)) { + SizeType count; + if (internal::CountStringCodePoint(str, length, &count)) { + if (count < minLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); + if (count > maxLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } + } + + if (pattern_ && !IsPatternMatch(pattern_, str, length)) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + + return CreateParallelValidator(context); + } + + bool StartObject(Context& context) const { + if (!(type_ & (1 << kObjectSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (hasDependencies_ || hasRequired_) { + context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); + std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); + } + + if (patternProperties_) { // pre-allocate schema array + SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType + context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); + context.patternPropertiesSchemaCount = 0; + std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); + } + + return CreateParallelValidator(context); + } + + bool Key(Context& context, const Ch* str, SizeType len, bool) const { + if (patternProperties_) { + context.patternPropertiesSchemaCount = 0; + for (SizeType i = 0; i < patternPropertyCount_; i++) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + } + + SizeType index; + if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { + if (context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; + context.valueSchema = GetTypeless(); + context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; + } + else + context.valueSchema = properties_[index].schema; + + if (context.propertyExist) + context.propertyExist[index] = true; + + return true; + } + + if (additionalPropertiesSchema_) { + if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; + context.valueSchema = GetTypeless(); + context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; + } + else + context.valueSchema = additionalPropertiesSchema_; + return true; + } + else if (additionalProperties_) { + context.valueSchema = GetTypeless(); + return true; + } + + if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + + return true; + } + + bool EndObject(Context& context, SizeType memberCount) const { + if (hasRequired_) + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].required) + if (!context.propertyExist[index]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + + if (memberCount < minProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); + + if (memberCount > maxProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + + if (hasDependencies_) { + for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) + if (context.propertyExist[sourceIndex]) { + if (properties_[sourceIndex].dependencies) { + for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) + if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + else if (properties_[sourceIndex].dependenciesSchema) + if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + } + + return true; + } + + bool StartArray(Context& context) const { + if (!(type_ & (1 << kArraySchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + context.arrayElementIndex = 0; + context.inArray = true; + + return CreateParallelValidator(context); + } + + bool EndArray(Context& context, SizeType elementCount) const { + context.inArray = false; + + if (elementCount < minItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); + + if (elementCount > maxItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + + return true; + } + + // Generate functions for string literal according to Ch +#define RAPIDJSON_STRING_(name, ...) \ + static const ValueType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const ValueType v(s, sizeof(s) / sizeof(Ch) - 1);\ + return v;\ + } + + RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') + RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') + RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') + RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') + RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') + RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') + RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') + RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') + RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') + RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') + RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') + RAPIDJSON_STRING_(Not, 'n', 'o', 't') + RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') + RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') + RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') + RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + +#undef RAPIDJSON_STRING_ + +private: + enum SchemaValueType { + kNullSchemaType, + kBooleanSchemaType, + kObjectSchemaType, + kArraySchemaType, + kStringSchemaType, + kNumberSchemaType, + kIntegerSchemaType, + kTotalSchemaType + }; + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + typedef internal::GenericRegex RegexType; +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + typedef std::basic_regex RegexType; +#else + typedef char RegexType; +#endif + + struct SchemaArray { + SchemaArray() : schemas(), count() {} + ~SchemaArray() { AllocatorType::Free(schemas); } + const SchemaType** schemas; + SizeType begin; // begin index of context.validators + SizeType count; + }; + + static const SchemaType* GetTypeless() { + static SchemaType typeless(0, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); + return &typeless; + } + + template + void AddUniqueElement(V1& a, const V2& v) { + for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + if (*itr == v) + return; + V1 c(v, *allocator_); + a.PushBack(c, *allocator_); + } + + static const ValueType* GetMember(const ValueType& value, const ValueType& name) { + typename ValueType::ConstMemberIterator itr = value.FindMember(name); + return itr != value.MemberEnd() ? &(itr->value) : 0; + } + + static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsBool()) + out = v->GetBool(); + } + + static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) + out = static_cast(v->GetUint64()); + } + + void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { + if (const ValueType* v = GetMember(value, name)) { + if (v->IsArray() && v->Size() > 0) { + PointerType q = p.Append(name, allocator_); + out.count = v->Size(); + out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); + memset(out.schemas, 0, sizeof(Schema*)* out.count); + for (SizeType i = 0; i < out.count; i++) + schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); + out.begin = validatorCount_; + validatorCount_ += out.count; + } + } + } + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); + if (!r->IsValid()) { + r->~RegexType(); + AllocatorType::Free(r); + r = 0; + } + return r; + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { + return pattern->Search(str); + } +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) + try { + return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + } + catch (const std::regex_error&) { + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { + std::match_results r; + return std::regex_search(str, str + length, r, *pattern); + } +#else + template + RegexType* CreatePattern(const ValueType&) { return 0; } + + static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } +#endif // RAPIDJSON_SCHEMA_USE_STDREGEX + + void AddType(const ValueType& type) { + if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; + else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; + else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; + else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; + else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; + else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; + else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); + } + + bool CreateParallelValidator(Context& context) const { + if (enum_ || context.arrayUniqueness) + context.hasher = context.factory.CreateHasher(); + + if (validatorCount_) { + RAPIDJSON_ASSERT(context.validators == 0); + context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + context.validatorCount = validatorCount_; + + if (allOf_.schemas) + CreateSchemaValidators(context, allOf_); + + if (anyOf_.schemas) + CreateSchemaValidators(context, anyOf_); + + if (oneOf_.schemas) + CreateSchemaValidators(context, oneOf_); + + if (not_) + context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); + + if (hasSchemaDependencies_) { + for (SizeType i = 0; i < propertyCount_; i++) + if (properties_[i].dependenciesSchema) + context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); + } + } + + return true; + } + + void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { + for (SizeType i = 0; i < schemas.count; i++) + context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); + } + + // O(n) + bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { + SizeType len = name.GetStringLength(); + const Ch* str = name.GetString(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].name.GetStringLength() == len && + (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) + { + *outIndex = index; + return true; + } + return false; + } + + bool CheckInt(Context& context, int64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsInt64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsUint64()) { + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsInt64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsUint64()) + /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckUint(Context& context, uint64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsUint64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsUint64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (i % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckDoubleMinimum(Context& context, double d) const { + if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + return true; + } + + bool CheckDoubleMaximum(Context& context, double d) const { + if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + return true; + } + + bool CheckDoubleMultipleOf(Context& context, double d) const { + double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); + double q = std::floor(a / b); + double r = a - q * b; + if (r > 0.0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + return true; + } + + struct Property { + Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} + ~Property() { AllocatorType::Free(dependencies); } + SValue name; + const SchemaType* schema; + const SchemaType* dependenciesSchema; + SizeType dependenciesValidatorIndex; + bool* dependencies; + bool required; + }; + + struct PatternProperty { + PatternProperty() : schema(), pattern() {} + ~PatternProperty() { + if (pattern) { + pattern->~RegexType(); + AllocatorType::Free(pattern); + } + } + const SchemaType* schema; + RegexType* pattern; + }; + + AllocatorType* allocator_; + uint64_t* enum_; + SizeType enumCount_; + SchemaArray allOf_; + SchemaArray anyOf_; + SchemaArray oneOf_; + const SchemaType* not_; + unsigned type_; // bitmask of kSchemaType + SizeType validatorCount_; + SizeType notValidatorIndex_; + + Property* properties_; + const SchemaType* additionalPropertiesSchema_; + PatternProperty* patternProperties_; + SizeType patternPropertyCount_; + SizeType propertyCount_; + SizeType minProperties_; + SizeType maxProperties_; + bool additionalProperties_; + bool hasDependencies_; + bool hasRequired_; + bool hasSchemaDependencies_; + + const SchemaType* additionalItemsSchema_; + const SchemaType* itemsList_; + const SchemaType** itemsTuple_; + SizeType itemsTupleCount_; + SizeType minItems_; + SizeType maxItems_; + bool additionalItems_; + bool uniqueItems_; + + RegexType* pattern_; + SizeType minLength_; + SizeType maxLength_; + + SValue minimum_; + SValue maximum_; + SValue multipleOf_; + bool exclusiveMinimum_; + bool exclusiveMaximum_; +}; + +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + *documentStack.template Push() = '/'; + char buffer[21]; + size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); + for (size_t i = 0; i < length; i++) + *documentStack.template Push() = buffer[i]; + } +}; + +// Partial specialized version for char to prevent buffer copying. +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + if (sizeof(SizeType) == 4) { + char *buffer = documentStack.template Push(1 + 10); // '/' + uint + *buffer++ = '/'; + const char* end = internal::u32toa(index, buffer); + documentStack.template Pop(static_cast(10 - (end - buffer))); + } + else { + char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 + *buffer++ = '/'; + const char* end = internal::u64toa(index, buffer); + documentStack.template Pop(static_cast(20 - (end - buffer))); + } + } +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// IGenericRemoteSchemaDocumentProvider + +template +class IGenericRemoteSchemaDocumentProvider { +public: + typedef typename SchemaDocumentType::Ch Ch; + + virtual ~IGenericRemoteSchemaDocumentProvider() {} + virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaDocument + +//! JSON schema document. +/*! + A JSON schema document is a compiled version of a JSON schema. + It is basically a tree of internal::Schema. + + \note This is an immutable class (i.e. its instance cannot be modified after construction). + \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. + \tparam Allocator Allocator type for allocating memory of this document. +*/ +template +class GenericSchemaDocument { +public: + typedef ValueT ValueType; + typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; + typedef Allocator AllocatorType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef internal::Schema SchemaType; + typedef GenericPointer PointerType; + friend class internal::Schema; + template + friend class GenericSchemaValidator; + + //! Constructor. + /*! + Compile a JSON document into schema document. + + \param document A JSON document as source. + \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. + \param allocator An optional allocator instance for allocating memory. Can be null. + */ + explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + remoteProvider_(remoteProvider), + allocator_(allocator), + ownAllocator_(), + root_(), + schemaMap_(allocator, kInitialSchemaMapSize), + schemaRef_(allocator, kInitialSchemaRefSize) + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + // Generate root schema, it will call CreateSchema() to create sub-schemas, + // And call AddRefSchema() if there are $ref. + CreateSchemaRecursive(&root_, PointerType(), document, document); + + // Resolve $ref + while (!schemaRef_.Empty()) { + SchemaRefEntry* refEntry = schemaRef_.template Pop(1); + if (const SchemaType* s = GetSchema(refEntry->target)) { + if (refEntry->schema) + *refEntry->schema = s; + + // Create entry in map if not exist + if (!GetSchema(refEntry->source)) { + new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); + } + } + refEntry->~SchemaRefEntry(); + } + + RAPIDJSON_ASSERT(root_ != 0); + + schemaRef_.ShrinkToFit(); // Deallocate all memory for ref + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : + remoteProvider_(rhs.remoteProvider_), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + root_(rhs.root_), + schemaMap_(std::move(rhs.schemaMap_)), + schemaRef_(std::move(rhs.schemaRef_)) + { + rhs.remoteProvider_ = 0; + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + } +#endif + + //! Destructor + ~GenericSchemaDocument() { + while (!schemaMap_.Empty()) + schemaMap_.template Pop(1)->~SchemaEntry(); + + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Get the root schema. + const SchemaType& GetRoot() const { return *root_; } + +private: + //! Prohibit copying + GenericSchemaDocument(const GenericSchemaDocument&); + //! Prohibit assignment + GenericSchemaDocument& operator=(const GenericSchemaDocument&); + + struct SchemaRefEntry { + SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} + PointerType source; + PointerType target; + const SchemaType** schema; + }; + + struct SchemaEntry { + SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} + ~SchemaEntry() { + if (owned) { + schema->~SchemaType(); + Allocator::Free(schema); + } + } + PointerType pointer; + SchemaType* schema; + bool owned; + }; + + void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + if (schema) + *schema = SchemaType::GetTypeless(); + + if (v.GetType() == kObjectType) { + const SchemaType* s = GetSchema(pointer); + if (!s) + CreateSchema(schema, pointer, v, document); + + for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); + } + else if (v.GetType() == kArrayType) + for (SizeType i = 0; i < v.Size(); i++) + CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); + } + + void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + RAPIDJSON_ASSERT(pointer.IsValid()); + if (v.IsObject()) { + if (!HandleRefSchema(pointer, schema, v, document)) { + SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); + new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); + if (schema) + *schema = s; + } + } + } + + bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { + static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; + static const ValueType kRefValue(kRefString, 4); + + typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); + if (itr == v.MemberEnd()) + return false; + + if (itr->value.IsString()) { + SizeType len = itr->value.GetStringLength(); + if (len > 0) { + const Ch* s = itr->value.GetString(); + SizeType i = 0; + while (i < len && s[i] != '#') // Find the first # + i++; + + if (i > 0) { // Remote reference, resolve immediately + if (remoteProvider_) { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i - 1)) { + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { + if (schema) + *schema = sc; + return true; + } + } + } + } + } + else if (s[i] == '#') { // Local reference, defer resolution + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const ValueType* nv = pointer.Get(document)) + if (HandleRefSchema(source, schema, *nv, document)) + return true; + + new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); + return true; + } + } + } + } + return false; + } + + const SchemaType* GetSchema(const PointerType& pointer) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (pointer == target->pointer) + return target->schema; + return 0; + } + + PointerType GetPointer(const SchemaType* schema) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (schema == target->schema) + return target->pointer; + return PointerType(); + } + + static const size_t kInitialSchemaMapSize = 64; + static const size_t kInitialSchemaRefSize = 64; + + IRemoteSchemaDocumentProviderType* remoteProvider_; + Allocator *allocator_; + Allocator *ownAllocator_; + const SchemaType* root_; //!< Root schema. + internal::Stack schemaMap_; // Stores created Pointer -> Schemas + internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref +}; + +//! GenericSchemaDocument using Value type. +typedef GenericSchemaDocument SchemaDocument; +//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaValidator + +//! JSON Schema Validator. +/*! + A SAX style JSON schema validator. + It uses a \c GenericSchemaDocument to validate SAX events. + It delegates the incoming SAX events to an output handler. + The default output handler does nothing. + It can be reused multiple times by calling \c Reset(). + + \tparam SchemaDocumentType Type of schema document. + \tparam OutputHandler Type of output handler. Default handler does nothing. + \tparam StateAllocator Allocator for storing the internal validation states. +*/ +template < + typename SchemaDocumentType, + typename OutputHandler = BaseReaderHandler, + typename StateAllocator = CrtAllocator> +class GenericSchemaValidator : + public internal::ISchemaStateFactory, + public internal::ISchemaValidator +{ +public: + typedef typename SchemaDocumentType::SchemaType SchemaType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename SchemaType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + + //! Constructor without output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + outputHandler_(GetNullHandler()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Constructor with output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + OutputHandler& outputHandler, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + outputHandler_(outputHandler), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Destructor. + ~GenericSchemaValidator() { + Reset(); + RAPIDJSON_DELETE(ownStateAllocator_); + } + + //! Reset the internal states. + void Reset() { + while (!schemaStack_.Empty()) + PopSchema(); + documentStack_.Clear(); + valid_ = true; + } + + //! Checks whether the current state is valid. + // Implementation of ISchemaValidator + virtual bool IsValid() const { return valid_; } + + //! Gets the JSON pointer pointed to the invalid schema. + PointerType GetInvalidSchemaPointer() const { + return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); + } + + //! Gets the keyword of invalid schema. + const Ch* GetInvalidSchemaKeyword() const { + return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; + } + + //! Gets the JSON pointer pointed to the invalid value. + PointerType GetInvalidDocumentPointer() const { + return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + *documentStack_.template Push() = '\0';\ + documentStack_.template Pop(1);\ + internal::PrintInvalidDocument(documentStack_.template Bottom());\ +RAPIDJSON_MULTILINEMACRO_END +#else +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() +#endif + +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ + if (!valid_) return false; \ + if (!BeginValue() || !CurrentSchema().method arg1) {\ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ + return valid_ = false;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ + for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ + if (context->hasher)\ + static_cast(context->hasher)->method arg2;\ + if (context->validators)\ + for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ + static_cast(context->validators[i_])->method arg2;\ + if (context->patternPropertiesValidators)\ + for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ + static_cast(context->patternPropertiesValidators[i_])->method arg2;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ + return valid_ = EndValue() && outputHandler_.method arg2 + +#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ + RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) + + bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } + bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } + bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } + bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } + bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } + bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } + bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + bool String(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + + bool StartObject() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); + return valid_ = outputHandler_.StartObject(); + } + + bool Key(const Ch* str, SizeType len, bool copy) { + if (!valid_) return false; + AppendToken(str, len); + if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); + return valid_ = outputHandler_.Key(str, len, copy); + } + + bool EndObject(SizeType memberCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); + if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); + } + + bool StartArray() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); + return valid_ = outputHandler_.StartArray(); + } + + bool EndArray(SizeType elementCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); + if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); + } + +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ +#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ +#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ + + // Implementation of ISchemaStateFactory + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { + return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, +#if RAPIDJSON_SCHEMA_VERBOSE + depth_ + 1, +#endif + &GetStateAllocator()); + } + + virtual void DestroySchemaValidator(ISchemaValidator* validator) { + GenericSchemaValidator* v = static_cast(validator); + v->~GenericSchemaValidator(); + StateAllocator::Free(v); + } + + virtual void* CreateHasher() { + return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); + } + + virtual uint64_t GetHashCode(void* hasher) { + return static_cast(hasher)->GetHashCode(); + } + + virtual void DestroryHasher(void* hasher) { + HasherType* h = static_cast(hasher); + h->~HasherType(); + StateAllocator::Free(h); + } + + virtual void* MallocState(size_t size) { + return GetStateAllocator().Malloc(size); + } + + virtual void FreeState(void* p) { + return StateAllocator::Free(p); + } + +private: + typedef typename SchemaType::Context Context; + typedef GenericValue, StateAllocator> HashCodeArray; + typedef internal::Hasher HasherType; + + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + const SchemaType& root, +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth, +#endif + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(root), + outputHandler_(GetNullHandler()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(depth) +#endif + { + } + + StateAllocator& GetStateAllocator() { + if (!stateAllocator_) + stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator()); + return *stateAllocator_; + } + + bool BeginValue() { + if (schemaStack_.Empty()) + PushSchema(root_); + else { + if (CurrentContext().inArray) + internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); + + if (!CurrentSchema().BeginValue(CurrentContext())) + return false; + + SizeType count = CurrentContext().patternPropertiesSchemaCount; + const SchemaType** sa = CurrentContext().patternPropertiesSchemas; + typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; + bool valueUniqueness = CurrentContext().valueUniqueness; + if (CurrentContext().valueSchema) + PushSchema(*CurrentContext().valueSchema); + + if (count > 0) { + CurrentContext().objectPatternValidatorType = patternValidatorType; + ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; + SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; + va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + for (SizeType i = 0; i < count; i++) + va[validatorCount++] = CreateSchemaValidator(*sa[i]); + } + + CurrentContext().arrayUniqueness = valueUniqueness; + } + return true; + } + + bool EndValue() { + if (!CurrentSchema().EndValue(CurrentContext())) + return false; + +#if RAPIDJSON_SCHEMA_VERBOSE + GenericStringBuffer sb; + schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); + + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); + internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); +#endif + + uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; + + PopSchema(); + + if (!schemaStack_.Empty()) { + Context& context = CurrentContext(); + if (context.valueUniqueness) { + HashCodeArray* a = static_cast(context.arrayElementHashCodes); + if (!a) + CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); + for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) + if (itr->GetUint64() == h) + RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + a->PushBack(h, GetStateAllocator()); + } + } + + // Remove the last token of document pointer + while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') + ; + + return true; + } + + void AppendToken(const Ch* str, SizeType len) { + documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters + *documentStack_.template PushUnsafe() = '/'; + for (SizeType i = 0; i < len; i++) { + if (str[i] == '~') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '0'; + } + else if (str[i] == '/') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '1'; + } + else + *documentStack_.template PushUnsafe() = str[i]; + } + } + + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } + + RAPIDJSON_FORCEINLINE void PopSchema() { + Context* c = schemaStack_.template Pop(1); + if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { + a->~HashCodeArray(); + StateAllocator::Free(a); + } + c->~Context(); + } + + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } + Context& CurrentContext() { return *schemaStack_.template Top(); } + const Context& CurrentContext() const { return *schemaStack_.template Top(); } + + static OutputHandler& GetNullHandler() { + static OutputHandler nullHandler; + return nullHandler; + } + + static const size_t kDefaultSchemaStackCapacity = 1024; + static const size_t kDefaultDocumentStackCapacity = 256; + const SchemaDocumentType* schemaDocument_; + const SchemaType& root_; + OutputHandler& outputHandler_; + StateAllocator* stateAllocator_; + StateAllocator* ownStateAllocator_; + internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) + internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + bool valid_; +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth_; +#endif +}; + +typedef GenericSchemaValidator SchemaValidator; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidatingReader + +//! A helper class for parsing with validation. +/*! + This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). + + \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam SourceEncoding Encoding of the input stream. + \tparam SchemaDocumentType Type of schema document. + \tparam StackAllocator Allocator type for stack. +*/ +template < + unsigned parseFlags, + typename InputStream, + typename SourceEncoding, + typename SchemaDocumentType = SchemaDocument, + typename StackAllocator = CrtAllocator> +class SchemaValidatingReader { +public: + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename InputStream::Ch Ch; + + //! Constructor + /*! + \param is Input stream. + \param sd Schema document. + */ + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} + + template + bool operator()(Handler& handler) { + GenericReader reader; + GenericSchemaValidator validator(sd_, handler); + parseResult_ = reader.template Parse(is_, validator); + + isValid_ = validator.IsValid(); + if (isValid_) { + invalidSchemaPointer_ = PointerType(); + invalidSchemaKeyword_ = 0; + invalidDocumentPointer_ = PointerType(); + } + else { + invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); + invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); + invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + } + + return parseResult_; + } + + const ParseResult& GetParseResult() const { return parseResult_; } + bool IsValid() const { return isValid_; } + const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } + const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } + const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + +private: + InputStream& is_; + const SchemaDocumentType& sd_; + + ParseResult parseResult_; + PointerType invalidSchemaPointer_; + const Ch* invalidSchemaKeyword_; + PointerType invalidDocumentPointer_; + bool isValid_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + +#endif // RAPIDJSON_SCHEMA_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/stream.h b/ext/librethinkdbxx/src/rapidjson/stream.h new file mode 100644 index 00000000..fef82c25 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/stream.h @@ -0,0 +1,179 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "rapidjson.h" + +#ifndef RAPIDJSON_STREAM_H_ +#define RAPIDJSON_STREAM_H_ + +#include "encodings.h" + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Stream + +/*! \class rapidjson::Stream + \brief Concept for reading and writing characters. + + For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). + + For write-only stream, only need to implement Put() and Flush(). + +\code +concept Stream { + typename Ch; //!< Character type of the stream. + + //! Read the current character from stream without moving the read cursor. + Ch Peek() const; + + //! Read the current character from stream and moving the read cursor to next character. + Ch Take(); + + //! Get the current read cursor. + //! \return Number of characters read from start. + size_t Tell(); + + //! Begin writing operation at the current read pointer. + //! \return The begin writer pointer. + Ch* PutBegin(); + + //! Write a character. + void Put(Ch c); + + //! Flush the buffer. + void Flush(); + + //! End the writing operation. + //! \param begin The begin write pointer returned by PutBegin(). + //! \return Number of characters written. + size_t PutEnd(Ch* begin); +} +\endcode +*/ + +//! Provides additional information for stream. +/*! + By using traits pattern, this type provides a default configuration for stream. + For custom stream, this type can be specialized for other configuration. + See TEST(Reader, CustomStringStream) in readertest.cpp for example. +*/ +template +struct StreamTraits { + //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; +}; + +//! Reserve n characters for writing to a stream. +template +inline void PutReserve(Stream& stream, size_t count) { + (void)stream; + (void)count; +} + +//! Write character to a stream, presuming buffer is reserved. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { + stream.Put(c); +} + +//! Put N copies of a character to a stream. +template +inline void PutN(Stream& stream, Ch c, size_t n) { + PutReserve(stream, n); + for (size_t i = 0; i < n; i++) + PutUnsafe(stream, c); +} + +/////////////////////////////////////////////////////////////////////////////// +// StringStream + +//! Read-only string stream. +/*! \note implements Stream concept +*/ +template +struct GenericStringStream { + typedef typename Encoding::Ch Ch; + + GenericStringStream(const Ch *src) : src_(src), head_(src) {} + + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! String stream with UTF8 encoding. +typedef GenericStringStream > StringStream; + +/////////////////////////////////////////////////////////////////////////////// +// InsituStringStream + +//! A read-write string stream. +/*! This string stream is particularly designed for in-situ parsing. + \note implements Stream concept +*/ +template +struct GenericInsituStringStream { + typedef typename Encoding::Ch Ch; + + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } + + // Write + void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} + + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } + + Ch* src_; + Ch* dst_; + Ch* head_; +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! Insitu string stream with UTF8 encoding. +typedef GenericInsituStringStream > InsituStringStream; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_STREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/stringbuffer.h b/ext/librethinkdbxx/src/rapidjson/stringbuffer.h new file mode 100644 index 00000000..78f34d20 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/stringbuffer.h @@ -0,0 +1,117 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_STRINGBUFFER_H_ +#define RAPIDJSON_STRINGBUFFER_H_ + +#include "stream.h" +#include "internal/stack.h" + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + +#include "internal/stack.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory output stream. +/*! + \tparam Encoding Encoding of the stream. + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +class GenericStringBuffer { +public: + typedef typename Encoding::Ch Ch; + + GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} + GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { + if (&rhs != this) + stack_ = std::move(rhs.stack_); + return *this; + } +#endif + + void Put(Ch c) { *stack_.template Push() = c; } + void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.ShrinkToFit(); + stack_.template Pop(1); + } + + void Reserve(size_t count) { stack_.template Reserve(count); } + Ch* Push(size_t count) { return stack_.template Push(count); } + Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetString() const { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.template Pop(1); + + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; + +private: + // Prohibit copy constructor & assignment operator. + GenericStringBuffer(const GenericStringBuffer&); + GenericStringBuffer& operator=(const GenericStringBuffer&); +}; + +//! String buffer with UTF8 encoding +typedef GenericStringBuffer > StringBuffer; + +template +inline void PutReserve(GenericStringBuffer& stream, size_t count) { + stream.Reserve(count); +} + +template +inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { + stream.PutUnsafe(c); +} + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { + std::memset(stream.stack_.Push(n), c, n * sizeof(c)); +} + +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/writer.h b/ext/librethinkdbxx/src/rapidjson/writer.h new file mode 100644 index 00000000..7d0610eb --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/writer.h @@ -0,0 +1,609 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_WRITER_H_ +#define RAPIDJSON_WRITER_H_ + +#include "stream.h" +#include "internal/stack.h" +#include "internal/strfunc.h" +#include "internal/dtoa.h" +#include "internal/itoa.h" +#include "stringbuffer.h" +#include // placement new + +#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef RAPIDJSON_SSE42 +#include +#elif defined(RAPIDJSON_SSE2) +#include +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// WriteFlag + +/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS + \ingroup RAPIDJSON_CONFIG + \brief User-defined kWriteDefaultFlags definition. + + User can define this as any \c WriteFlag combinations. +*/ +#ifndef RAPIDJSON_WRITE_DEFAULT_FLAGS +#define RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags +#endif + +//! Combination of writeFlags +enum WriteFlag { + kWriteNoFlags = 0, //!< No flags are set. + kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. + kWriteNanAndInfFlag = 2, //!< Allow writing of Inf, -Inf and NaN. + kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS +}; + +//! JSON writer +/*! Writer implements the concept Handler. + It generates JSON text by events to an output os. + + User may programmatically calls the functions of a writer to generate JSON text. + + On the other side, a writer can also be passed to objects that generates events, + + for example Reader::Parse() and Document::Accept(). + + \tparam OutputStream Type of output stream. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. + \note implements Handler concept +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class Writer { +public: + typedef typename SourceEncoding::Ch Ch; + + static const int kDefaultMaxDecimalPlaces = 324; + + //! Constructor + /*! \param os Output stream. + \param stackAllocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit + Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + + explicit + Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + + //! Reset the writer with a new stream. + /*! + This function reset the writer with a new stream and default settings, + in order to make a Writer object reusable for output multiple JSONs. + + \param os New output stream. + \code + Writer writer(os1); + writer.StartObject(); + // ... + writer.EndObject(); + + writer.Reset(os2); + writer.StartObject(); + // ... + writer.EndObject(); + \endcode + */ + void Reset(OutputStream& os) { + os_ = &os; + hasRoot_ = false; + level_stack_.Clear(); + } + + //! Checks whether the output is a complete JSON. + /*! + A complete JSON has a complete root object or array. + */ + bool IsComplete() const { + return hasRoot_ && level_stack_.Empty(); + } + + int GetMaxDecimalPlaces() const { + return maxDecimalPlaces_; + } + + //! Sets the maximum number of decimal places for double output. + /*! + This setting truncates the output with specified number of decimal places. + + For example, + + \code + writer.SetMaxDecimalPlaces(3); + writer.StartArray(); + writer.Double(0.12345); // "0.123" + writer.Double(0.0001); // "0.0" + writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) + writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) + writer.EndArray(); + \endcode + + The default setting does not truncate any decimal places. You can restore to this setting by calling + \code + writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); + \endcode + */ + void SetMaxDecimalPlaces(int maxDecimalPlaces) { + maxDecimalPlaces_ = maxDecimalPlaces; + } + + /*!@name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { Prefix(kNullType); return WriteNull(); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } + bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } + bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } + bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } + + //! Writes the given \c double value to the stream + /*! + \param d The value to be written. + \return Whether it is succeed. + */ + bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + Prefix(kNumberType); + return WriteString(str, length); + } + + bool String(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + Prefix(kStringType); + return WriteString(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + bool String(const std::basic_string& str) { + return String(str.data(), SizeType(str.size())); + } +#endif + + bool StartObject() { + Prefix(kObjectType); + new (level_stack_.template Push()) Level(false); + return WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + bool ret = WriteEndObject(); + if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + os_->Flush(); + return ret; + } + + bool StartArray() { + Prefix(kArrayType); + new (level_stack_.template Push()) Level(true); + return WriteStartArray(); + } + + bool EndArray(SizeType elementCount = 0) { + (void)elementCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + bool ret = WriteEndArray(); + if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + os_->Flush(); + return ret; + } + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + */ + bool RawValue(const Ch* json, size_t length, Type type) { Prefix(type); return WriteRawValue(json, length); } + +protected: + //! Information for each nested level + struct Level { + Level(bool inArray_) : valueCount(0), inArray(inArray_) {} + size_t valueCount; //!< number of values in this level + bool inArray; //!< true if in array, otherwise in object + }; + + static const size_t kDefaultLevelDepth = 32; + + bool WriteNull() { + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; + } + + bool WriteBool(bool b) { + if (b) { + PutReserve(*os_, 4); + PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); + } + else { + PutReserve(*os_, 5); + PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); + } + return true; + } + + bool WriteInt(int i) { + char buffer[11]; + const char* end = internal::i32toa(i, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteUint(unsigned u) { + char buffer[10]; + const char* end = internal::u32toa(u, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteInt64(int64_t i64) { + char buffer[21]; + const char* end = internal::i64toa(i64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteUint64(uint64_t u64) { + char buffer[20]; + char* end = internal::u64toa(u64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + if (!(writeFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + + char buffer[25]; + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteString(const Ch* str, SizeType length) { + static const typename TargetEncoding::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const char escape[256] = { +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 + 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 + Z16, Z16, // 30~4F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF +#undef Z16 + }; + + if (TargetEncoding::supportUnicode) + PutReserve(*os_, 2 + length * 6); // "\uxxxx..." + else + PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." + + PutUnsafe(*os_, '\"'); + GenericStringStream is(str); + while (ScanWriteUnescapedString(is, length)) { + const Ch c = is.Peek(); + if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { + // Unicode escaping + unsigned codepoint; + if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) + return false; + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { + PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); + } + else { + RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); + // Surrogate pair + unsigned s = codepoint - 0x010000; + unsigned lead = (s >> 10) + 0xD800; + unsigned trail = (s & 0x3FF) + 0xDC00; + PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(lead ) & 15]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(trail ) & 15]); + } + } + else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { + is.Take(); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, static_cast(escape[static_cast(c)])); + if (escape[static_cast(c)] == 'u') { + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); + PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); + } + } + else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; + } + PutUnsafe(*os_, '\"'); + return true; + } + + bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { + return RAPIDJSON_LIKELY(is.Tell() < length); + } + + bool WriteStartObject() { os_->Put('{'); return true; } + bool WriteEndObject() { os_->Put('}'); return true; } + bool WriteStartArray() { os_->Put('['); return true; } + bool WriteEndArray() { os_->Put(']'); return true; } + + bool WriteRawValue(const Ch* json, size_t length) { + PutReserve(*os_, length); + for (size_t i = 0; i < length; i++) { + RAPIDJSON_ASSERT(json[i] != '\0'); + PutUnsafe(*os_, json[i]); + } + return true; + } + + void Prefix(Type type) { + (void)type; + if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root + Level* level = level_stack_.template Top(); + if (level->valueCount > 0) { + if (level->inArray) + os_->Put(','); // add comma if it is not the first element in array + else // in object + os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. + hasRoot_ = true; + } + } + + OutputStream* os_; + internal::Stack level_stack_; + int maxDecimalPlaces_; + bool hasRoot_; + +private: + // Prohibit copy constructor & assignment operator. + Writer(const Writer&); + Writer& operator=(const Writer&); +}; + +// Full specialization for StringStream to prevent memory copying + +template<> +inline bool Writer::WriteInt(int i) { + char *buffer = os_->Push(11); + const char* end = internal::i32toa(i, buffer); + os_->Pop(static_cast(11 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteUint(unsigned u) { + char *buffer = os_->Push(10); + const char* end = internal::u32toa(u, buffer); + os_->Pop(static_cast(10 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteInt64(int64_t i64) { + char *buffer = os_->Push(21); + const char* end = internal::i64toa(i64, buffer); + os_->Pop(static_cast(21 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteUint64(uint64_t u) { + char *buffer = os_->Push(20); + const char* end = internal::u64toa(u, buffer); + os_->Pop(static_cast(20 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). + if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + + char *buffer = os_->Push(25); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + os_->Pop(static_cast(25 - (end - buffer))); + return true; +} + +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); + + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (; p != endAligned; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType len; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + len = offset; +#else + len = static_cast(__builtin_ffs(r) - 1); +#endif + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); +} +#endif // defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) + +RAPIDJSON_NAMESPACE_END + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/term.cc b/ext/librethinkdbxx/src/term.cc new file mode 100644 index 00000000..711ef27d --- /dev/null +++ b/ext/librethinkdbxx/src/term.cc @@ -0,0 +1,285 @@ +#include +#include + +#include "term.h" +#include "json_p.h" + +namespace RethinkDB { + +using TT = Protocol::Term::TermType; + +struct { + Datum operator() (const Array& array) { + Array copy; + copy.reserve(array.size()); + for (const auto& it : array) { + copy.emplace_back(it.apply(*this)); + } + return Datum(Array{TT::MAKE_ARRAY, std::move(copy)}); + } + Datum operator() (const Object& object) { + Object copy; + for (const auto& it : object) { + copy.emplace(it.first, it.second.apply(*this)); + } + return std::move(copy); + } + template + Datum operator() (T&& atomic) { + return Datum(std::forward(atomic)); + } +} datum_to_term; + +Term::Term(Datum&& datum_) : datum(datum_.apply(datum_to_term)) { } +Term::Term(const Datum& datum_) : datum(datum_.apply(datum_to_term)) { } + +Term::Term(Term&& orig, OptArgs&& new_optargs) : datum(Nil()) { + Datum* cur = orig.datum.get_nth(2); + Object optargs; + free_vars = std::move(orig.free_vars); + if (cur) { + optargs = std::move(cur->extract_object()); + } + for (auto& it : new_optargs) { + optargs.emplace(std::move(it.first), alpha_rename(std::move(it.second))); + } + datum = Array{ std::move(orig.datum.extract_nth(0)), std::move(orig.datum.extract_nth(1)), std::move(optargs) }; +} + +Term nil() { + return Term(Nil()); +} + +Cursor Term::run(Connection& conn, OptArgs&& opts) { + if (!free_vars.empty()) { + throw Error("run: term has free variables"); + } + + return conn.start_query(this, std::move(opts)); +} + +struct { + Datum operator() (Object&& object, const std::map& subst, bool) { + Object ret; + for (auto& it : object) { + ret.emplace(std::move(it.first), std::move(it.second).apply(*this, subst, false)); + } + return ret; + } + Datum operator() (Array&& array, const std::map& subst, bool args) { + if (!args) { + double cmd = array[0].extract_number(); + if (cmd == static_cast(TT::VAR)) { + double var = array[1].extract_nth(0).extract_number(); + auto it = subst.find(static_cast(var)); + if (it != subst.end()) { + return Array{ TT::VAR, { it->second }}; + } + } + if (array.size() == 2) { + return Array{ std::move(array[0]), std::move(array[1]).apply(*this, subst, true) }; + } else { + return Array{ + std::move(array[0]), + std::move(array[1]).apply(*this, subst, true), + std::move(array[2]).apply(*this, subst, false) }; + } + } else { + Array ret; + for (auto& it : array) { + ret.emplace_back(std::move(it).apply(*this, subst, false)); + } + return ret; + } + } + template + Datum operator() (T&& a, const std::map&, bool) { + return std::move(a); + } +} alpha_renamer; + +static int new_var_id(const std::map& vars) { + while (true) { + int id = gen_var_id(); + if (vars.find(id) == vars.end()) { + return id; + } + } +} + +Datum Term::alpha_rename(Term&& term) { + if (free_vars.empty()) { + free_vars = std::move(term.free_vars); + return std::move(term.datum); + } + + std::map subst; + for (auto it = term.free_vars.begin(); it != term.free_vars.end(); ++it) { + auto var = free_vars.find(it->first); + if (var == free_vars.end()) { + free_vars.emplace(it->first, it->second); + } else if (var->second != it->second) { + int id = new_var_id(free_vars); + subst.emplace(it->first, id); + free_vars.emplace(id, it->second); + } + } + if (subst.empty()) { + return std::move(term.datum); + } else { + return term.datum.apply(alpha_renamer, subst, false); + } +} + +int gen_var_id() { + return ::random() % (1<<30); +} + +C0_IMPL(db_list, DB_LIST) +C0_IMPL(table_list, TABLE_LIST) +C0_IMPL(random, RANDOM) +C0_IMPL(now, NOW) +C0_IMPL(range, RANGE) +C0_IMPL(error, ERROR) +C0_IMPL(uuid, UUID) +C0_IMPL(literal, LITERAL) +CO0_IMPL(wait, WAIT) +C0_IMPL(rebalance, REBALANCE) +CO0_IMPL(random, RANDOM) + +Term row(TT::IMPLICIT_VAR, {}); +Term minval(TT::MINVAL, {}); +Term maxval(TT::MAXVAL, {}); + +Term binary(const std::string& data) { + return expr(Binary(data)); +} + +Term binary(std::string&& data) { + return expr(Binary(data)); +} + +Term binary(const char* data) { + return expr(Binary(data)); +} + +struct { + bool operator() (const Object& object) { + for (const auto& it : object) { + if (it.second.apply(*this)) { + return true; + } + } + return false; + } + bool operator() (const Array& array) { + int type = *array[0].get_number(); + if (type == static_cast(TT::IMPLICIT_VAR)) { + return true; + } + if (type == static_cast(TT::FUNC)) { + return false; + } + for (const auto& it : *array[1].get_array()) { + if (it.apply(*this)) { + return true; + } + } + if (array.size() == 3) { + return array[2].apply(*this); + } else { + return false; + } + } + template + bool operator() (T) { + return false; + } +} needs_func_wrap; + +Term Term::func_wrap(Term&& term) { + if (term.datum.apply(needs_func_wrap)) { + return Term(TT::FUNC, {expr(Array{new_var_id(term.free_vars)}), std::move(term)}); + } + return term; +} + +Term Term::func_wrap(const Term& term) { + if (term.datum.apply(needs_func_wrap)) { + // TODO return Term(TT::FUNC, {expr(Array{new_var_id(Term.free_vars)}), Term.copy()}); + return Term(Nil()); + } + return term; +} + +Term Term::make_object(std::vector&& args) { + if (args.size() % 2 != 0) { + return Term(TT::OBJECT, std::move(args)); + } + std::set keys; + for (auto it = args.begin(); it != args.end() && it + 1 != args.end(); it += 2) { + std::string* key = it->datum.get_string(); + if (!key || keys.count(*key)) { + return Term(TT::OBJECT, std::move(args)); + } + keys.insert(*key); + } + Term ret{Nil()}; + Object object; + for (auto it = args.begin(); it != args.end(); it += 2) { + std::string* key = it->datum.get_string(); + object.emplace(std::move(*key), ret.alpha_rename(std::move(*(it + 1)))); + } + ret.datum = std::move(object); + return ret; +} + +Term Term::make_binary(Term&& term) { + std::string* string = term.datum.get_string(); + if (string) { + return expr(Binary(std::move(*string))); + } + return Term(TT::BINARY, std::vector{term}); +} + +Term::Term(OptArgs&& optargs) : datum(Nil()) { + Object oargs; + for (auto& it : optargs) { + oargs.emplace(it.first, alpha_rename(std::move(it.second))); + } + datum = std::move(oargs); +} + +OptArgs optargs() { + return OptArgs{}; +} + +Term january(TT::JANUARY, {}); +Term february(TT::FEBRUARY, {}); +Term march(TT::MARCH, {}); +Term april(TT::APRIL, {}); +Term may(TT::MAY, {}); +Term june(TT::JUNE, {}); +Term july(TT::JULY, {}); +Term august(TT::AUGUST, {}); +Term september(TT::SEPTEMBER, {}); +Term october(TT::OCTOBER, {}); +Term november(TT::NOVEMBER, {}); +Term december(TT::DECEMBER, {}); +Term monday(TT::MONDAY, {}); +Term tuesday(TT::TUESDAY, {}); +Term wednesday(TT::WEDNESDAY, {}); +Term thursday(TT::THURSDAY, {}); +Term friday(TT::FRIDAY, {}); +Term saturday(TT::SATURDAY, {}); +Term sunday(TT::SUNDAY, {}); + +Term Term::copy() const { + return *this; +} + +Datum Term::get_datum() const { + return datum; +} + +} diff --git a/ext/librethinkdbxx/src/term.h b/ext/librethinkdbxx/src/term.h new file mode 100644 index 00000000..cdee5ec4 --- /dev/null +++ b/ext/librethinkdbxx/src/term.h @@ -0,0 +1,592 @@ +#pragma once + +#include "datum.h" +#include "connection.h" +#include "protocol_defs.h" +#include "cursor.h" + +namespace RethinkDB { + +using TT = Protocol::Term::TermType; + +class Term; +class Var; + +// An alias for the Term constructor +template +Term expr(T&&); + +int gen_var_id(); + +// Can be used as the last argument to some ReQL commands that expect named arguments +using OptArgs = std::map; + +// Represents a ReQL Term (RethinkDB Query Language) +// Designed to be used with r-value *this +class Term { +public: + Term(const Term& other) = default; + Term(Term&& other) = default; + Term& operator= (const Term& other) = default; + Term& operator= (Term&& other) = default; + + explicit Term(Datum&&); + explicit Term(const Datum&); + explicit Term(OptArgs&&); + + // Create a copy of the Term + Term copy() const; + + Term(std::function f) : datum(Nil()) { set_function>(f); } + Term(std::function f) : datum(Nil()) { set_function, 0>(f); } + Term(std::function f) : datum(Nil()) { set_function, 0, 1>(f); } + Term(std::function f) : datum(Nil()) { set_function, 0, 1, 2>(f); } + Term(Protocol::Term::TermType type, std::vector&& args) : datum(Array()) { + Array dargs; + for (auto& it : args) { + dargs.emplace_back(alpha_rename(std::move(it))); + } + datum = Datum(Array{ type, std::move(dargs) }); + } + + Term(Protocol::Term::TermType type, std::vector&& args, OptArgs&& optargs) : datum(Array()) { + Array dargs; + for (auto& it : args) { + dargs.emplace_back(alpha_rename(std::move(it))); + } + Object oargs; + for (auto& it : optargs) { + oargs.emplace(it.first, alpha_rename(std::move(it.second))); + } + datum = Array{ type, std::move(dargs), std::move(oargs) }; + } + + // Used internally to support row + static Term func_wrap(Term&&); + static Term func_wrap(const Term&); + + + // These macros are used to define most ReQL commands + // * Cn represents a method with n arguments + // * COn represents a method with n arguments and optional named arguments + // * C_ represents a method with any number of arguments + // Each method is implemented twice, once with r-value *this, and once with const *this + // The third argument, wrap, allows converting arguments into functions if they contain row + +#define C0(name, type) \ + Term name() && { return Term(TT::type, std::vector{ std::move(*this) }); } \ + Term name() const & { return Term(TT::type, std::vector{ *this }); } +#define C1(name, type, wrap) \ + template \ + Term name(T&& a) && { return Term(TT::type, std::vector{ std::move(*this), wrap(expr(std::forward(a))) }); } \ + template \ + Term name(T&& a) const & { return Term(TT::type, std::vector{ *this, wrap(expr(std::forward(a))) }); } +#define C2(name, type) \ + template Term name(T&& a, U&& b) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + expr(std::forward(a)), expr(std::forward(b)) }); } \ + template Term name(T&& a, U&& b) const & { \ + return Term(TT::type, std::vector{ *this, \ + expr(std::forward(a)), expr(std::forward(b)) }); } +#define C_(name, type, wrap) \ + template Term name(T&& ...a) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a)))... }); } \ + template Term name(T&& ...a) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a)))... }); } +#define CO0(name, type) \ + Term name(OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this) }, std::move(optarg)); } \ + Term name(OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this }, std::move(optarg)); } +#define CO1(name, type, wrap) \ + template Term name(T&& a, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))) }, std::move(optarg)); } \ + template Term name(T&& a, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))) }, std::move(optarg)); } +#define CO2(name, type, wrap) \ + template Term name(T&& a, U&& b, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))) }, std::move(optarg)); } \ + template Term name(T&& a, U&& b, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))) }, std::move(optarg)); } +#define CO3(name, type, wrap) \ + template Term name(T&& a, U&& b, V&& c, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c))) }, std::move(optarg)); } \ + template Term name(T&& a, U&& b, V&& c, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c)))}, std::move(optarg)); } +#define CO4(name, type, wrap) \ + template Term name(T&& a, U&& b, V&& c, W&& d, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c))), wrap(expr(std::forward(d))) }, std::move(optarg)); } \ + template Term name(T&& a, U&& b, V&& c, W&& d, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c))), wrap(expr(std::forward(d))) }, std::move(optarg)); } +#define CO_(name, type, wrap) \ + C_(name, type, wrap) \ + CO0(name, type) \ + CO1(name, type, wrap) \ + CO2(name, type, wrap) \ + CO3(name, type, wrap) \ + CO4(name, type, wrap) +#define no_wrap(x) x + + CO1(table_create, TABLE_CREATE, no_wrap) + C1(table_drop, TABLE_DROP, no_wrap) + C0(table_list, TABLE_LIST) + CO1(index_create, INDEX_CREATE, no_wrap) + CO2(index_create, INDEX_CREATE, func_wrap) + C1(index_drop, INDEX_DROP, no_wrap) + C0(index_list, INDEX_LIST) + CO2(index_rename, INDEX_RENAME, no_wrap) + C_(index_status, INDEX_STATUS, no_wrap) + C_(index_wait, INDEX_WAIT, no_wrap) + CO0(changes, CHANGES) + CO1(insert, INSERT, no_wrap) + CO1(update, UPDATE, func_wrap) + CO1(replace, REPLACE, func_wrap) + CO0(delete_, DELETE) + C0(sync, SYNC) + CO1(table, TABLE, no_wrap) + C1(get, GET, no_wrap) + CO_(get_all, GET_ALL, no_wrap) + CO2(between, BETWEEN, no_wrap) + CO1(filter, FILTER, func_wrap) + C2(inner_join, INNER_JOIN) + C2(outer_join, OUTER_JOIN) + CO2(eq_join, EQ_JOIN, func_wrap) + C0(zip, ZIP) + C_(map, MAP, func_wrap) + C_(with_fields, WITH_FIELDS, no_wrap) + C1(concat_map, CONCAT_MAP, func_wrap) + CO_(order_by, ORDER_BY, func_wrap) + C1(skip, SKIP, no_wrap) + C1(limit, LIMIT, no_wrap) + CO1(slice, SLICE, no_wrap) + CO2(slice, SLICE, no_wrap) + C1(nth, NTH, no_wrap) + C1(offsets_of, OFFSETS_OF, func_wrap) + C0(is_empty, IS_EMPTY) + CO_(union_, UNION, no_wrap) + C1(sample, SAMPLE, no_wrap) + CO_(group, GROUP, func_wrap) + C0(ungroup, UNGROUP) + C1(reduce, REDUCE, no_wrap) + CO2(fold, FOLD, no_wrap) + C0(count, COUNT) + C1(count, COUNT, func_wrap) + C0(sum, SUM) + C1(sum, SUM, func_wrap) + C0(avg, AVG) + C1(avg, AVG, func_wrap) + C1(min, MIN, func_wrap) + CO0(min, MIN) + C1(max, MAX, func_wrap) + CO0(max, MAX) + CO0(distinct, DISTINCT) + C_(contains, CONTAINS, func_wrap) + C_(pluck, PLUCK, no_wrap) + C_(without, WITHOUT, no_wrap) + C_(merge, MERGE, func_wrap) + C1(append, APPEND, no_wrap) + C1(prepend, PREPEND, no_wrap) + C1(difference, DIFFERENCE, no_wrap) + C1(set_insert, SET_INSERT, no_wrap) + C1(set_union, SET_UNION, no_wrap) + C1(set_intersection, SET_INTERSECTION, no_wrap) + C1(set_difference, SET_DIFFERENCE, no_wrap) + C1(operator[], BRACKET, no_wrap) + C1(get_field, GET_FIELD, no_wrap) + C_(has_fields, HAS_FIELDS, no_wrap) + C2(insert_at, INSERT_AT) + C2(splice_at, SPLICE_AT) + C1(delete_at, DELETE_AT, no_wrap) + C2(delete_at, DELETE_AT) + C2(change_at, CHANGE_AT) + C0(keys, KEYS) + C1(match, MATCH, no_wrap) + C0(split, SPLIT) + C1(split, SPLIT, no_wrap) + C2(split, SPLIT) + C0(upcase, UPCASE) + C0(downcase, DOWNCASE) + C_(add, ADD, no_wrap) + C1(operator+, ADD, no_wrap) + C_(sub, SUB, no_wrap) + C1(operator-, SUB, no_wrap) + C_(mul, MUL, no_wrap) + C1(operator*, MUL, no_wrap) + C_(div, DIV, no_wrap) + C1(operator/, DIV, no_wrap) + C1(mod, MOD, no_wrap) + C1(operator%, MOD, no_wrap) + C_(and_, AND, no_wrap) + C1(operator&&, AND, no_wrap) + C_(or_, OR, no_wrap) + C1(operator||, OR, no_wrap) + C1(eq, EQ, no_wrap) + C1(operator==, EQ, no_wrap) + C1(ne, NE, no_wrap) + C1(operator!=, NE, no_wrap) + C1(gt, GT, no_wrap) + C1(operator>, GT, no_wrap) + C1(ge, GE, no_wrap) + C1(operator>=, GE, no_wrap) + C1(lt, LT, no_wrap) + C1(operator<, LT, no_wrap) + C1(le, LE, no_wrap) + C1(operator<=, LE, no_wrap) + C0(not_, NOT) + C0(operator!, NOT) + C1(in_timezone, IN_TIMEZONE, no_wrap) + C0(timezone, TIMEZONE) + CO2(during, DURING, no_wrap) + C0(date, DATE) + C0(time_of_day, TIME_OF_DAY) + C0(year, YEAR) + C0(month, MONTH) + C0(day, DAY) + C0(day_of_week, DAY_OF_WEEK) + C0(day_of_year, DAY_OF_YEAR) + C0(hours, HOURS) + C0(minutes, MINUTES) + C0(seconds, SECONDS) + C0(to_iso8601, TO_ISO8601) + C0(to_epoch_time, TO_EPOCH_TIME) + C1(for_each, FOR_EACH, func_wrap) + C1(default_, DEFAULT, no_wrap) + CO1(js, JAVASCRIPT, no_wrap) + C1(coerce_to, COERCE_TO, no_wrap) + C0(type_of, TYPE_OF) + C0(info, INFO) + C0(to_json, TO_JSON_STRING) + C0(to_json_string, TO_JSON_STRING) + C1(distance, DISTANCE, no_wrap) + C0(fill, FILL) + C0(to_geojson, TO_GEOJSON) + CO1(get_intersecting, GET_INTERSECTING, no_wrap) + CO1(get_nearest, GET_NEAREST, no_wrap) + C1(includes, INCLUDES, no_wrap) + C1(intersects, INTERSECTS, no_wrap) + C1(polygon_sub, POLYGON_SUB, no_wrap) + C0(config, CONFIG) + C0(rebalance, REBALANCE) + CO0(reconfigure, RECONFIGURE) + C0(status, STATUS) + CO0(wait, WAIT) + C0(floor, FLOOR) + C0(ceil, CEIL) + C0(round, ROUND) + C0(values, VALUES) + + // The expansion of this macro fails to compile on some versions of GCC and Clang: + // C_(operator(), FUNCALL, no_wrap) + // The std::enable_if makes the error go away + + // $doc(do) + + template + typename std::enable_if::value, Term>::type + operator() (T&& a, U&& ...b) && { + return Term(TT::FUNCALL, std::vector{ + std::move(*this), + expr(std::forward(a)), + expr(std::forward(b))... }); + } + template + typename std::enable_if::value, Term>::type + operator() (T&& a, U&& ...b) const & { + return Term(TT::FUNCALL, std::vector{ + *this, + expr(std::forward(a)), + expr(std::forward(b))... }); + } + +#undef C0 +#undef C1 +#undef C2 +#undef C_ +#undef CO0 +#undef CO1 +#undef CO2 + + // Send the term to the server and return the results. + // Errors returned by the server are thrown. + Cursor run(Connection&, OptArgs&& args = {}); + + // $doc(do) + template + Term do_(T&& ...a) && { + auto list = { std::move(*this), Term::func_wrap(expr(std::forward(a)))... }; + std::vector args; + args.reserve(list.size() + 1); + args.emplace_back(func_wrap(std::move(*(list.end()-1)))); + for (auto it = list.begin(); it + 1 != list.end(); ++it) { + args.emplace_back(std::move(*it)); + } + return Term(TT::FUNCALL, std::move(args)); + } + + // Adds optargs to an already built term + Term opt(OptArgs&& optargs) && { + return Term(std::move(*this), std::move(optargs)); + } + + // Used internally to implement object() + static Term make_object(std::vector&&); + + // Used internally to implement array() + static Term make_binary(Term&&); + + Datum get_datum() const; + +private: + friend class Var; + friend class Connection; + friend struct Query; + + template + Var mkvar(std::vector& vars); + + template + void set_function(F); + + Datum alpha_rename(Term&&); + + Term(Term&& orig, OptArgs&& optargs); + + std::map free_vars; + Datum datum; +}; + +// A term representing null +Term nil(); + +template +Term expr(T&& a) { + return Term(std::forward(a)); +} + +// Represents a ReQL variable. +// This type is passed to functions used in ReQL queries. +class Var { +public: + // Convert to a term + Term operator*() const { + Term term(TT::VAR, std::vector{expr(*id)}); + term.free_vars = {{*id, id}}; + return term; + } + + Var(int* id_) : id(id_) { } +private: + int* id; +}; + +template +Var Term::mkvar(std::vector& vars) { + int id = gen_var_id(); + vars.push_back(id); + return Var(&*vars.rbegin()); +} + +template +void Term::set_function(F f) { + std::vector vars; + vars.reserve(sizeof...(N)); + std::vector args = { mkvar(vars)... }; + Term body = f(args[N] ...); + + int* low = &*vars.begin(); + int* high = &*(vars.end() - 1); + for (auto it = body.free_vars.begin(); it != body.free_vars.end(); ) { + if (it->second >= low && it->second <= high) { + if (it->first != *it->second) { + throw Error("Internal error: variable index mis-match"); + } + ++it; + } else { + free_vars.emplace(*it); + ++it; + } + } + datum = Array{TT::FUNC, Array{Array{TT::MAKE_ARRAY, vars}, body.datum}}; +} + +// These macros are similar to those defined above, but for top-level ReQL operations + +#define C0(name) Term name(); +#define C0_IMPL(name, type) Term name() { return Term(TT::type, std::vector{}); } +#define CO0(name) Term name(OptArgs&& optargs = {}); +#define CO0_IMPL(name, type) Term name(OptArgs&& optargs) { return Term(TT::type, std::vector{}, std::move(optargs)); } +#define C1(name, type, wrap) template Term name(T&& a) { \ + return Term(TT::type, std::vector{ wrap(expr(std::forward(a))) }); } +#define C2(name, type) template Term name(T&& a, U&& b) { \ + return Term(TT::type, std::vector{ expr(std::forward(a)), expr(std::forward(b)) }); } +#define C3(name, type) template \ + Term name(A&& a, B&& b, C&& c) { return Term(TT::type, std::vector{ \ + expr(std::forward(a)), expr(std::forward(b)), expr(std::forward(c)) }); } +#define C4(name, type) template \ + Term name(A&& a, B&& b, C&& c, D&& d) { return Term(TT::type, std::vector{ \ + expr(std::forward(a)), expr(std::forward(b)), \ + expr(std::forward(c)), expr(std::forward(d))}); } +#define C7(name, type) template \ + Term name(A&& a, B&& b, C&& c, D&& d, E&& e, F&& f, G&& g) { return Term(TT::type, std::vector{ \ + expr(std::forward(a)), expr(std::forward(b)), expr(std::forward(c)), \ + expr(std::forward(d)), expr(std::forward(e)), expr(std::forward(f)), \ + expr(std::forward(g))}); } +#define C_(name, type, wrap) template Term name(T&& ...a) { \ + return Term(TT::type, std::vector{ wrap(expr(std::forward(a)))... }); } +#define CO1(name, type, wrap) template Term name(T&& a, OptArgs&& optarg = {}) { \ + return Term(TT::type, std::vector{ wrap(expr(std::forward(a)))}, std::move(optarg)); } +#define CO2(name, type) template Term name(T&& a, U&& b, OptArgs&& optarg = {}) { \ + return Term(TT::type, std::vector{ expr(std::forward(a)), expr(std::forward(b))}, std::move(optarg)); } +#define func_wrap Term::func_wrap + +C1(db_create, DB_CREATE, no_wrap) +C1(db_drop, DB_DROP, no_wrap) +C0(db_list) +CO1(table_create, TABLE_CREATE, no_wrap) +C1(table_drop, TABLE_DROP, no_wrap) +C0(table_list) +C1(db, DB, no_wrap) +CO1(table, TABLE, no_wrap) +C_(add, ADD, no_wrap) +C2(sub, SUB) +C_(mul, MUL, no_wrap) +C_(div, DIV, no_wrap) +C2(mod, MOD) +C_(and_, AND, no_wrap) +C_(or_, OR, no_wrap) +C2(eq, EQ) +C2(ne, NE) +C2(gt, GT) +C2(ge, GE) +C2(lt, LT) +C2(le, LE) +C1(not_, NOT, no_wrap) +CO0(random) +CO1(random, RANDOM, no_wrap) +CO2(random, RANDOM) +C0(now) +C4(time, TIME) +C7(time, TIME) +C1(epoch_time, EPOCH_TIME, no_wrap) +CO1(iso8601, ISO8601, no_wrap) +CO1(js, JAVASCRIPT, no_wrap) +C1(args, ARGS, no_wrap) +C_(branch, BRANCH, no_wrap) +C0(range) +C1(range, RANGE, no_wrap) +C2(range, RANGE) +C0(error) +C1(error, ERROR, no_wrap) +C1(json, JSON, no_wrap) +CO1(http, HTTP, func_wrap) +C0(uuid) +C1(uuid, UUID, no_wrap) +CO2(circle, CIRCLE) +C1(geojson, GEOJSON, no_wrap) +C_(line, LINE, no_wrap) +C2(point, POINT) +C_(polygon, POLYGON, no_wrap) +C_(array, MAKE_ARRAY, no_wrap) +C1(desc, DESC, func_wrap) +C1(asc, ASC, func_wrap) +C0(literal) +C1(literal, LITERAL, no_wrap) +C1(type_of, TYPE_OF, no_wrap) +C_(map, MAP, func_wrap) +C1(floor, FLOOR, no_wrap) +C1(ceil, CEIL, no_wrap) +C1(round, ROUND, no_wrap) +C_(union_, UNION, no_wrap) +C_(group, GROUP, func_wrap) +C1(count, COUNT, no_wrap) +C_(count, COUNT, func_wrap) +C1(sum, SUM, no_wrap) +C_(sum, SUM, func_wrap) +C1(avg, AVG, no_wrap) +C_(avg, AVG, func_wrap) +C1(min, MIN, no_wrap) +C_(min, MIN, func_wrap) +C1(max, MAX, no_wrap) +C_(max, MAX, func_wrap) +C1(distinct, DISTINCT, no_wrap) +C1(contains, CONTAINS, no_wrap) +C_(contains, CONTAINS, func_wrap) + +#undef C0 +#undef C1 +#undef C2 +#undef C3 +#undef C4 +#undef C7 +#undef C_ +#undef CO1 +#undef CO2 +#undef func_wrap + +// $doc(do) +template +Term do_(R&& a, T&& ...b) { + return expr(std::forward(a)).do_(std::forward(b)...); +} + +// $doc(object) +template +Term object(T&& ...a) { + return Term::make_object(std::vector{ expr(std::forward(a))... }); +} + +// $doc(binary) +template +Term binary(T&& a) { + return Term::make_binary(expr(std::forward(a))); +} + +// Construct an empty optarg +OptArgs optargs(); + +// Construct an optarg made out of pairs of arguments +// For example: optargs("k1", v1, "k2", v2) +template +OptArgs optargs(const char* key, V&& val, T&& ...rest) { + OptArgs opts = optargs(rest...); + opts.emplace(key, expr(std::forward(val))); + return opts; +} + +extern Term row; +extern Term maxval; +extern Term minval; +extern Term january; +extern Term february; +extern Term march; +extern Term april; +extern Term may; +extern Term june; +extern Term july; +extern Term august; +extern Term september; +extern Term october; +extern Term november; +extern Term december; +extern Term monday; +extern Term tuesday; +extern Term wednesday; +extern Term thursday; +extern Term friday; +extern Term saturday; +extern Term sunday; +} diff --git a/ext/librethinkdbxx/src/types.cc b/ext/librethinkdbxx/src/types.cc new file mode 100644 index 00000000..ea9becaf --- /dev/null +++ b/ext/librethinkdbxx/src/types.cc @@ -0,0 +1,47 @@ +#include + +#include "types.h" +#include "error.h" + +namespace RethinkDB { + +bool Time::parse_utc_offset(const std::string& string, double* offset) { + const char *s = string.c_str(); + double sign = 1; + switch (s[0]) { + case '-': + sign = -1; + case '+': + ++s; + break; + case 0: + return false; + } + for (int i = 0; i < 5; ++i) { + if (s[i] == 0) return false; + if (i == 2) continue; + if (s[i] < '0' || s[i] > '9') return false; + } + if (s[2] != ':') return false; + *offset = sign * ((s[0] - '0') * 36000 + (s[1] - '0') * 3600 + (s[3] - '0') * 600 + (s[4] - '0') * 60); + return true; +} + +double Time::parse_utc_offset(const std::string& string) { + double out; + if (!parse_utc_offset(string, &out)) { + throw Error("invalid utc offset `%s'", string.c_str()); + } + return out; +} + +std::string Time::utc_offset_string(double offset) { + char buf[8]; + int hour = offset / 3600; + int minutes = std::abs(static_cast(offset / 60)) % 60; + int n = snprintf(buf, 7, "%+03d:%02d", hour, minutes); + buf[n] = 0; + return std::string(buf); +} + +} diff --git a/ext/librethinkdbxx/src/types.h b/ext/librethinkdbxx/src/types.h new file mode 100644 index 00000000..ac35a871 --- /dev/null +++ b/ext/librethinkdbxx/src/types.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + +namespace RethinkDB { + +class Datum; + +// Represents a null datum +struct Nil { }; + +using Array = std::vector; +using Object = std::map; + +// Represents a string of bytes. Plain std::strings are passed on to the server as utf-8 strings +struct Binary { + bool operator== (const Binary& other) const { + return data == other.data; + } + + Binary(const std::string& data_) : data(data_) { } + Binary(std::string&& data_) : data(std::move(data_)) { } + std::string data; +}; + +// Represents a point in time as +// * A floating amount of seconds since the UNIX epoch +// * And a timezone offset represented as seconds relative to UTC +struct Time { + Time(double epoch_time_, double utc_offset_ = 0) : + epoch_time(epoch_time_), utc_offset(utc_offset_) { } + + static Time now() { + return Time(time(NULL)); + } + + static bool parse_utc_offset(const std::string&, double*); + static double parse_utc_offset(const std::string&); + static std::string utc_offset_string(double); + + double epoch_time; + double utc_offset; +}; + +// Not implemented +class Point; +class Line; +class Polygon; + +} diff --git a/ext/librethinkdbxx/src/utils.cc b/ext/librethinkdbxx/src/utils.cc new file mode 100644 index 00000000..5a2c244d --- /dev/null +++ b/ext/librethinkdbxx/src/utils.cc @@ -0,0 +1,153 @@ +#include "utils.h" +#include "error.h" + +namespace RethinkDB { + +size_t utf8_encode(unsigned int code, char* buf) { + if (!(code & ~0x7F)) { + buf[0] = code; + return 1; + } else if (!(code & ~0x7FF)) { + buf[0] = 0xC0 | (code >> 6); + buf[1] = 0x80 | (code & 0x3F); + return 2; + } else if (!(code & ~0xFFFF)) { + buf[0] = 0xE0 | (code >> 12); + buf[1] = 0x80 | ((code >> 6) & 0x3F); + buf[2] = 0x80 | (code & 0x3F); + return 3; + } else if (!(code & ~0x1FFFFF)) { + buf[0] = 0xF0 | (code >> 18); + buf[1] = 0x80 | ((code >> 12) & 0x3F); + buf[2] = 0x80 | ((code >> 6) & 0x3F); + buf[3] = 0x80 | (code & 0x3F); + return 4; + } else if (!(code & ~0x3FFFFFF)) { + buf[0] = 0xF8 | (code >> 24); + buf[1] = 0x80 | ((code >> 18) & 0x3F); + buf[2] = 0x80 | ((code >> 12) & 0x3F); + buf[3] = 0x80 | ((code >> 6) & 0x3F); + buf[4] = 0x80 | (code & 0x3F); + return 5; + } else if (!(code & ~0x7FFFFFFF)) { + buf[0] = 0xFC | (code >> 30); + buf[1] = 0x80 | ((code >> 24) & 0x3F); + buf[2] = 0x80 | ((code >> 18) & 0x3F); + buf[3] = 0x80 | ((code >> 12) & 0x3F); + buf[4] = 0x80 | ((code >> 6) & 0x3F); + buf[5] = 0x80 | (code & 0x3F); + return 6; + } else { + throw Error("Invalid unicode codepoint %ud", code); + } +} + +bool base64_decode(char c, int* out) { + if (c >= 'A' && c <= 'Z') { + *out = c - 'A'; + } else if (c >= 'a' && c <= 'z') { + *out = c - ('a' - 26); + } else if (c >= '0' && c <= '9') { + *out = c - ('0' - 52); + } else if (c == '+') { + *out = 62; + } else if (c == '/') { + *out = 63; + } else { + return false; + } + return true; +} + +bool base64_decode(const std::string& in, std::string& out) { + out.clear(); + out.reserve(in.size() * 3 / 4); + auto read = in.begin(); + while (true) { + int c[4]; + int end = 4; + for (int i = 0; i < 4; i++) { + while (true) { + if (read == in.end()) { + c[i] = 0; + end = i; + i = 3; + break; + } else if (base64_decode(*read, &c[i])) { + ++read; + break; + } else { + ++read; + } + } + } + if (end == 1) return false; + int val = c[0] << 18 | c[1] << 12 | c[2] << 6 | c[3]; + if (end > 1) out.append(1, val >> 16); + if (end > 2) out.append(1, val >> 8 & 0xFF); + if (end > 3) out.append(1, val & 0xFF); + if (end != 4) break; + } + return true; +} + +char base64_encode(unsigned int c) { + if (c < 26) { + return 'A' + c; + } else if (c < 52) { + return 'a' + c - 26; + } else if (c < 62) { + return '0' + c - 52; + } else if (c == 62) { + return '+'; + } else if (c == 63) { + return '/'; + } else { + throw Error("unreachable: base64 encoding %d", c); + } +} + +void base64_encode(unsigned int* c, int n, std::string& out) { + if (n == 0) { + return; + } + out.append(1, base64_encode(c[0] >> 2)); + out.append(1, base64_encode((c[0] & 0x3) << 4 | c[1] >> 4)); + if (n == 1) { + out.append("=="); + return; + } + out.append(1, base64_encode((c[1] & 0xF) << 2 | c[2] >> 6)); + if (n == 2) { + out.append("="); + return; + } + out.append(1, base64_encode(c[2] & 0x3F)); +} + +std::string base64_encode(const std::string& in) { + std::string out; + out.reserve(in.size() * 4 / 3 + in.size() / 48 + 3); + auto read = in.begin(); + while (true) { + for (int group = 0; group < 16; ++group) { + unsigned int c[3]; + int i = 0; + for (; i < 3; ++i) { + if (read == in.end()) { + c[i] = 0; + break; + } else { + c[i] = static_cast(*read++); + } + } + base64_encode(c, i, out); + if (i != 3) { + return out; + } + } + out.append("\n"); + } +} + +} diff --git a/ext/librethinkdbxx/src/utils.h b/ext/librethinkdbxx/src/utils.h new file mode 100644 index 00000000..04496e2c --- /dev/null +++ b/ext/librethinkdbxx/src/utils.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace RethinkDB { + +// The size of the longest UTF-8 encoded unicode codepoint +const size_t max_utf8_encoded_size = 6; + +// Decode a base64 string. Returns false on failure. +bool base64_decode(const std::string& in, std::string& out); +std::string base64_encode(const std::string&); + +// Encodes a single unicode codepoint into UTF-8. Returns the number of bytes written. +// Does not add a trailing null byte +size_t utf8_encode(unsigned int, char*); + +} diff --git a/ext/librethinkdbxx/test/bench.cc b/ext/librethinkdbxx/test/bench.cc new file mode 100644 index 00000000..e2843d4c --- /dev/null +++ b/ext/librethinkdbxx/test/bench.cc @@ -0,0 +1,58 @@ +/* +#include +#include +#include +#include + +namespace R = RethinkDB; +std::unique_ptr conn; + +int main() { + signal(SIGPIPE, SIG_IGN); + try { + conn = R::connect(); + } catch(const R::Error& error) { + printf("FAILURE: could not connect to localhost:28015: %s\n", error.message.c_str()); + return 1; + } + + try { + printf("running test...\n"); + auto start = std::chrono::steady_clock::now(); + R::Datum d = R::range(1, 1000000) + .map([]() { return R::object("test", "hello", "data", "world"); }) + .run(*conn); + auto end = std::chrono::steady_clock::now(); + auto diff = end - start; + + printf("result size: %d\n", (int)d.get_array()->size()); + printf("completed in %f ms\n", std::chrono::duration(diff).count()); + } catch (const R::Error& error) { + printf("FAILURE: uncaught exception: %s\n", error.message.c_str()); + return 1; + } +} +*/ + +#include +#include + +namespace R = RethinkDB; + +int main() { + auto conn = R::connect(); + if (!conn) { + std::cerr << "Could not connect to server\n"; + return 1; + } + + std::cout << "Connected" << std::endl; + R::Cursor databases = R::db_list().run(*conn); + for (R::Datum const& db : databases) { + std::cout << *db.get_string() << '\n'; + } + + return 0; +} + + diff --git a/ext/librethinkdbxx/test/gen_index_cxx.py b/ext/librethinkdbxx/test/gen_index_cxx.py new file mode 100644 index 00000000..220bb5a2 --- /dev/null +++ b/ext/librethinkdbxx/test/gen_index_cxx.py @@ -0,0 +1,11 @@ +from sys import argv +from re import sub + +print("#include \"testlib.h\""); +print("void run_upstream_tests() {") +for path in argv[1:]: + name = sub('/', '_', path.split('.')[0]) + print(" extern void %s();" % name) + print(" clean_slate();") + print(" %s();" % name) +print("}") diff --git a/ext/librethinkdbxx/test/test.cc b/ext/librethinkdbxx/test/test.cc new file mode 100644 index 00000000..fb5ca2fc --- /dev/null +++ b/ext/librethinkdbxx/test/test.cc @@ -0,0 +1,114 @@ +#include + +#include + +#include "testlib.h" + +extern void run_upstream_tests(); + +void test_json(const char* string, const char* ret = "") { + TEST_EQ(R::Datum::from_json(string).as_json().c_str(), ret[0] ? ret : string); +} + +void test_json_parse_print() { + enter_section("json"); + test_json("-0.0", "-0.0"); + test_json("null"); + test_json("1.2"); + test_json("1.2e20", "1.2e+20"); + test_json("true"); + test_json("false"); + test_json("\"\""); + test_json("\"\\u1234\"", "\"\u1234\""); + test_json("\"\\\"\""); + test_json("\"foobar\""); + test_json("[]"); + test_json("[1]"); + test_json("[1,2,3,4]"); + test_json("{}"); + test_json("{\"a\":1}"); + test_json("{\"a\":1,\"b\":2,\"c\":3}"); + exit_section(); +} + +void test_reql() { + enter_section("reql"); + TEST_EQ((R::expr(1) + 2).run(*conn), R::Datum(3)); + TEST_EQ(R::range(4).count().run(*conn), R::Datum(4)); + TEST_EQ(R::js("Math.abs")(-1).run(*conn), 1); + exit_section(); +} + +void test_cursor() { + enter_section("cursor"); + R::Cursor cursor = R::range(10000).run(*conn); + TEST_EQ(cursor.next(), 0); + R::Array array = cursor.to_array(); + TEST_EQ(array.size(), 9999); + TEST_EQ(*array.begin(), 1); + TEST_EQ(*array.rbegin(), 9999); + int i = 0; + R::range(3).run(*conn).each([&i](R::Datum&& datum){ + TEST_EQ(datum, i++); }); + exit_section(); +} + +void test_encode(const char* str, const char* b) { + TEST_EQ(R::base64_encode(str), b); +} + +void test_decode(const char* b, const char* str) { + std::string out; + TEST_EQ(R::base64_decode(b, out), true); + TEST_EQ(out, str); +} + +#define TEST_B64(a, b) test_encode(a, b); test_decode(b, a) + +void test_binary() { + enter_section("base64"); + TEST_B64("", ""); + TEST_B64("foo", "Zm9v"); + exit_section(); +} + +void test_issue28() { + enter_section("issue #28"); + std::vector expected{ "rethinkdb", "test" }; + std::vector dbs; + R::Cursor databases = R::db_list().run(*conn); + for (R::Datum const& db : databases) { + dbs.push_back(*db.get_string()); + } + + TEST_EQ(dbs, expected); + exit_section(); +} + +int main() { + signal(SIGPIPE, SIG_IGN); + srand(time(NULL)); + try { + conn = R::connect(); + } catch(const R::Error& error) { + printf("FAILURE: could not connect to localhost:28015: %s\n", error.message.c_str()); + return 1; + } + try { + //test_binary(); + //test_json_parse_print(); + //test_reql(); + //test_cursor(); + test_issue28(); + run_upstream_tests(); + } catch (const R::Error& error) { + printf("FAILURE: uncaught expception: %s\n", error.message.c_str()); + return 1; + } + if (!failed) { + printf("SUCCESS: %d tests passed\n", count); + } else { + printf("DONE: %d of %d tests failed\n", failed, count); + return 1; + } +} diff --git a/ext/librethinkdbxx/test/testlib.cc b/ext/librethinkdbxx/test/testlib.cc new file mode 100644 index 00000000..ddc9cc27 --- /dev/null +++ b/ext/librethinkdbxx/test/testlib.cc @@ -0,0 +1,356 @@ +#include +#include +#include +#include +#include + +#include "testlib.h" + +int verbosity = 0; + +int failed = 0; +int count = 0; +std::vector> section; + +std::unique_ptr conn; + +// std::string to_string(const R::Cursor&) { +// return ""; +// } + +std::string to_string(const R::Term& query) { + return to_string(query.get_datum()); +} + +std::string to_string(const R::Datum& datum) { + return datum.as_json(); +} + +std::string to_string(const R::Object& object) { + auto it = object.find("special"); + if (it != object.end()) { + std::string type = *(it->second).get_string(); + auto bag = object.find(type); + if (bag != object.end()) { + return to_string((R::Datum)bag->second); + } + } + + return to_string((R::Datum)object); +} + +std::string to_string(const R::Error& error) { + return "Error(\"" + error.message + "\")"; +} + +void enter_section(const char* name) { + if (verbosity == 0) { + section.emplace_back(name, true); + } else { + printf("%sSection %s\n", indent(), name); + section.emplace_back(name, false); + } +} + +void section_cleanup() { + R::db("test").table_list().for_each([=](R::Var table) { + return R::db("test").table_drop(*table); + }).run(*conn); +} + +void exit_section() { + section.pop_back(); +} + +std::string to_string(const err& error) { + return "Error(\"" + error.convert_type() + ": " + error.message + "\")"; +} + +bool equal(const R::Error& a, const err& b) { + // @TODO: I think the proper solution to this proble is to in fact create + // a hierarchy of exception types. This would not only simplify these + // error cases, but could be of great use to the user. + std::string error_type = b.convert_type(); + if (error_type == "ReqlServerCompileError" && + a.message.find("ReqlCompileError") != std::string::npos) { + return true; + } + + return b.trim_message(a.message) == (error_type + ": " + b.message); +} + +bool match(const char* pattern, const char* string) { + return std::regex_match(string, std::regex(pattern)); +} + +bool equal(const R::Error& a, const err_regex& b) { + if (b.message == "Object keys must be strings" && + a.message == "runtime error: Expected type STRING but found NUMBER.") { + return true; + } + return match(b.regex().c_str(), a.message.c_str()); +} + +std::string to_string(const err_regex& error) { + return "err_regex(" + error.type + ", " + error.message + ")"; +} + +R::Object partial(R::Object&& object) { + return R::Object{{"special", "partial"}, {"partial", std::move(object)}}; +} + +R::Datum uuid() { + return R::Object{{"special", "uuid"}}; +} + +R::Object arrlen(int n, R::Datum&& datum) { + return R::Object{{"special", "arrlen"},{"len",n},{"of",datum}}; +} + +R::Object arrlen(int n) { + return R::Object{{"special", "arrlen"},{"len",n}}; +} + +std::string repeat(std::string&& s, int n) { + std::string string; + string.reserve(n * s.size()); + for (int i = 0; i < n; ++i) { + string.append(s); + } + return string; +} + +R::Term fetch(R::Cursor& cursor, int count, double timeout) { + // printf("fetch(..., %d, %lf)\n", count, timeout); + R::Array array; + int deadline = time(NULL) + int(timeout); + for (int i = 0; count == -1 || i < count; ++i) { + // printf("fetching next (%d)\n", i); + time_t now = time(NULL); + if (now > deadline) break; + + try { + array.emplace_back(cursor.next(deadline - now)); + // printf("got %s\n", write_datum(array[array.size()-1]).c_str()); + } catch (const R::Error &e) { + if (e.message != "next: No more data") { + throw e; // rethrow + } + + break; + } catch (const R::TimeoutException &e){ + // printf("fetch timeout\n"); + break; + } + } + + return expr(std::move(array)); +} + +R::Object bag(R::Array&& array) { + return R::Object{{"special", "bag"}, {"bag", std::move(array)}}; +}; + +R::Object bag(R::Datum&& d) { + return R::Object{{"special", "bag"}, {"bag", std::move(d)}}; +}; + +std::string string_key(const R::Datum& datum) { + const std::string* string = datum.get_string(); + if (string) return *string; + return datum.as_json(); +} + +bool falsey(R::Datum&& datum) { + bool* boolean = datum.get_boolean(); + if (boolean) return !*boolean; + double* number = datum.get_number(); + if (number) return *number == 0; + return false; +} + +bool equal(const R::Datum& got, const R::Datum& expected) { + const std::string* string = expected.get_string(); + if (string) { + const R::Binary* binary = got.get_binary(); + if (binary) { + return *binary == R::Binary(*string); + } + } + if (expected.get_object() && expected.get_field("$reql_type$")) { + if (!got.get_field("$reql_type$")) { + R::Datum datum = got.to_raw(); + if (datum.get_field("$reql_type$")) { + return equal(datum, expected); + } + } + } + if (got.get_object() && got.get_field("$reql_type$")) { + const std::string* type = got.get_field("$reql_type$")->get_string(); + if (type && *type == "GROUPED_DATA" && + (!expected.get_object() || !expected.get_field("$reql_type$"))) { + const R::Array* data = got.get_field("data")->get_array(); + R::Object object; + for (R::Datum it : *data) { + object.emplace(string_key(it.extract_nth(0)), it.extract_nth(1)); + } + return equal(object, expected); + } + } + do { + if (!expected.get_object()) break; + if(!expected.get_field("special")) break; + const std::string* type = expected.get_field("special")->get_string(); + if (!type) break; + if (*type == "bag") { + const R::Datum* bag_datum = expected.get_field("bag"); + if (!bag_datum || !bag_datum->get_array()) { + break; + } + R::Array bag = *bag_datum->get_array(); + const R::Array* array = got.get_array(); + if (!array) { + return false; + } + if (bag.size() != array->size()) { + return false; + } + for (const auto& it : *array) { + auto ref = std::find(bag.begin(), bag.end(), it); + if (ref == bag.end()) return false; + bag.erase(ref); + } + return true; + } else if (*type == "arrlen") { + const R::Datum* len_datum = expected.get_field("len"); + if (!len_datum) break; + const double *len = len_datum->get_number(); + if (!len) break; + const R::Array* array = got.get_array(); + if (!array) break; + return array->size() == *len; + } else if (*type == "partial") { + const R::Object* object = got.get_object(); + if (object) { + const R::Datum* partial_datum = expected.get_field("partial"); + if (!partial_datum) break; + const R::Object* partial = partial_datum->get_object(); + if (!partial) break; + for (const auto& it : *partial) { + if (!object->count(it.first) || !equal((*object).at(it.first), it.second)) { + return false; + } + return true; + } + } + const R::Array* array = got.get_array(); + if (array) { + const R::Datum* partial_datum = expected.get_field("partial"); + if (!partial_datum) break; + const R::Array* partial = partial_datum->get_array(); + if (!partial) break; + + for (const auto& want : *partial) { + bool match = false; + for (const auto& have : *array) { + if (equal(have, want)) { + match = true; + break; + } + } + if (match == false) return false; + } + return true; + } + } else if(*type == "uuid") { + const std::string* string = got.get_string(); + if (string && string->size() == 36) { + return true; + } + } else if (*type == "regex") { + const R::Datum* regex_datum = expected.get_field("regex"); + if (!regex_datum) break; + const std::string* regex = regex_datum->get_string(); + if (!regex) break; + const std::string* str = got.get_string(); + if (!str) break; + return match(regex->c_str(), str->c_str()); + } + } while(0); + const R::Object* got_object = got.get_object(); + const R::Object* expected_object = expected.get_object(); + if (got_object && expected_object) { + R::Object have = *got_object; + for (const auto& it : *expected_object) { + auto other = have.find(it.first); + if (other == have.end()) return false; + if (!equal(other->second, it.second)) return false; + have.erase(other); + } + for (auto& it : have) { + if (!falsey(std::move(it.second))) { + return false; + } + } + return true; + } + const R::Array* got_array = got.get_array(); + const R::Array* expected_array = expected.get_array(); + if (got_array && expected_array) { + if (got_array->size() != expected_array->size()) return false; + for (R::Array::const_iterator i = got_array->begin(), j = expected_array->begin(); + i < got_array->end(); + i++, j++) { + if(!equal(*i, *j)) return false; + } + return true; + } + return got == expected; +} + +R::Object partial(R::Array&& array) { + return R::Object{{"special", "partial"}, {"partial", std::move(array)}}; +} + +R::Object regex(const char* pattern) { + return R::Object{{"special", "regex"}, {"regex", pattern}}; +} + +void clean_slate() { + R::table_list().for_each([](R::Var t){ return R::table_drop(*t); }); + R::db("rethinkdb").table("_debug_scratch").delete_().run(*conn); +} + +const char* indent() { + static const char spaces[] = " "; + return spaces + sizeof(spaces) - 1 - 2 * section.size(); +} + +std::string truncate(std::string&& string) { + if (string.size() > 200) { + return string.substr(0, 197) + "..."; + } + return string; +} + +int len(const R::Datum& d) { + const R::Array* arr = d.get_array(); + if (!arr) throw ("testlib: len: expected an array but got " + to_string(d)); + return arr->size(); +} + +R::Term wait(int n) { + std::this_thread::sleep_for(std::chrono::seconds(n)); + return R::expr(n); +} + +R::Datum nil = R::Nil(); + +R::Array append(R::Array lhs, R::Array rhs) { + if (lhs.empty()) { + return rhs; + } + lhs.reserve(lhs.size() + rhs.size()); + std::move(std::begin(rhs), std::end(rhs), std::back_inserter(lhs)); + return lhs; +} diff --git a/ext/librethinkdbxx/test/testlib.h b/ext/librethinkdbxx/test/testlib.h new file mode 100644 index 00000000..5b795436 --- /dev/null +++ b/ext/librethinkdbxx/test/testlib.h @@ -0,0 +1,231 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace R = RethinkDB; + +extern std::vector> section; +extern int failed; +extern int count; +extern std::unique_ptr conn; +extern int verbosity; + +const char* indent(); + +void enter_section(const char* name); +void section_cleanup(); +void exit_section(); + +#define TEST_DO(code) \ + if (verbosity > 1) fprintf(stderr, "%sTEST: %s\n", indent(), #code); \ + code + +#define TEST_EQ(code, expected) \ + do { \ + if (verbosity > 1) fprintf(stderr, "%sTEST: %s\n", indent(), #code); \ + try { test_eq(#code, (code), (expected)); } \ + catch (const R::Error& error) { test_eq(#code, error, (expected)); } \ + } while (0) + +struct err { + err(const char* type_, std::string message_, R::Array&& backtrace_ = {}) : + type(type_), message(message_), backtrace(std::move(backtrace_)) { } + + std::string convert_type() const { + return type; + } + + static std::string trim_message(std::string msg) { + size_t i = msg.find(":\n"); + if (i != std::string::npos) { + return msg.substr(0, i + 1); + } + return msg; + } + + std::string type; + std::string message; + R::Array backtrace; +}; + +struct err_regex { + err_regex(const char* type_, const char* message_, R::Array&& backtrace_ = {}) : + type(type_), message(message_), backtrace(std::move(backtrace_)) { } + std::string type; + std::string message; + R::Array backtrace; + std::string regex() const { + return type + ": " + message; + } +}; + +R::Object regex(const char* pattern); + +bool match(const char* pattern, const char* string); + +R::Object partial(R::Object&& object); +R::Object partial(R::Array&& array); +R::Datum uuid(); +R::Object arrlen(int n, R::Datum&& datum); +R::Object arrlen(int n); +R::Term new_table(); +std::string repeat(std::string&& s, int n); +R::Term fetch(R::Cursor& cursor, int count = -1, double timeout = 1); +R::Object bag(R::Array&& array); +R::Object bag(R::Datum&& d); + +struct temp_table { + temp_table() { + char chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + char name_[15] = "temp_"; + for (unsigned int i = 5; i + 1 < sizeof name_; ++i) { + name_[i] = chars[random() % (sizeof chars - 1)]; + } + name_[14] = 0; + R::table_create(name_).run(*conn); + name = name_; + } + + ~temp_table() { + try { + R::table_drop(name).run(*conn); + } catch (const R::Error &e) { + if(!strstr(e.message.c_str(), "does not exist")){ + printf("error dropping temp_table: %s\n", e.message.c_str()); + } + } + } + + R::Term table() { return R::table(name); } + std::string name; +}; + +void clean_slate(); + +// std::string to_string(const R::Cursor&); +std::string to_string(const R::Term&); +std::string to_string(const R::Datum&); +std::string to_string(const R::Error&); +std::string to_string(const err_regex&); +std::string to_string(const err&); + +bool equal(const R::Datum&, const R::Datum&); +bool equal(const R::Error&, const err_regex&); +bool equal(const R::Error&, const err&); + +template +bool equal(const T& a, const err& b) { + return false; +} + +template +bool equal(const T& a, const err_regex& b) { + return false; +} + +template +bool equal(const R::Error& a, const T& b) { + return false; +} + +std::string truncate(std::string&&); + +template +void test_eq(const char* code, const T val, const U expected) { + + try { + count ++; + if (!equal(val, expected)) { + failed++; + for (auto& it : section) { + if (it.second) { + printf("%sSection: %s\n", indent(), it.first); + it.second = false; + } + } + try { + printf("%sFAILURE in ‘%s’:\n%s Expected: ‘%s’\n%s but got: ‘%s’\n", + indent(), code, + indent(), truncate(to_string(expected)).c_str(), + indent(), truncate(to_string(val)).c_str()); + } catch (const R::Error& e) { + printf("%sFAILURE: Failed to print failure description: %s\n", indent(), e.message.c_str()); + } catch (...) { + printf("%sFAILURE: Failed to print failure description\n", indent()); + } + } + } catch (const std::regex_error& rx_err) { + printf("%sSKIP: error with regex (likely a buggy regex implementation): %s\n", indent(), rx_err.what()); + } +} + +template +void test_eq(const char* code, const R::Cursor& val, const U expected) { + try { + R::Datum result = val.to_datum(); + test_eq(code, result, expected); + } catch (R::Error& error) { + test_eq(code, error, expected); + } +} + +int len(const R::Datum&); + +R::Term wait(int n); + +#define PacificTimeZone() (-7 * 3600) +#define UTCTimeZone() (0) + +extern R::Datum nil; + +inline R::Cursor maybe_run(R::Cursor& c, R::Connection&, R::OptArgs&& o = {}) { + return std::move(c); +} + +inline R::Cursor maybe_run(R::Term q, R::Connection& c, R::OptArgs&& o = {}) { + return q.run(c, std::move(o)); +} + +inline int operator+(R::Datum a, int b) { + return a.extract_number() + b; +} + +inline R::Array operator*(R::Array arr, int n) { + R::Array ret; + for(int i = 0; i < n; i++) { + for(const auto& it: arr) { + ret.push_back(it); + } + } + return ret; +} + +inline R::Array array_range(int x, int y) { + R::Array ret; + for(int i = x; i < y; ++i) { + ret.push_back(i); + } + return ret; +} + +template +inline R::Array array_map(F f, R::Array a){ + R::Array ret; + for(R::Datum& d: a) { + ret.push_back(f(d.extract_number())); + } + return ret; +} + +R::Array append(R::Array lhs, R::Array rhs); + +template +std::string str(T x){ + return to_string(x); +} diff --git a/ext/librethinkdbxx/test/upstream/aggregation.yaml b/ext/librethinkdbxx/test/upstream/aggregation.yaml new file mode 100644 index 00000000..1e0ec9c8 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/aggregation.yaml @@ -0,0 +1,575 @@ +desc: Tests that manipulation data in tables +table_variable_name: tbl tbl2 tbl3 tbl4 +tests: + + # Set up some data + - cd: r.range(100).for_each(tbl.insert({'id':r.row, 'a':r.row.mod(4)})) + rb: tbl.insert((0..99).map{ |i| { :id => i, :a => i % 4 } }) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100} + + - cd: r.range(100).for_each(tbl2.insert({'id':r.row, 'a':r.row.mod(4)})) + rb: tbl2.insert((0..99).map{ |i| { :id => i, :b => i % 4 } }) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100} + + - cd: r.range(100).for_each(tbl3.insert({'id':r.row, 'a':r.row.mod(4), 'b':{'c':r.row.mod(5)}})) + rb: tbl3.insert((0..99).map{ |i| { :id => i, :a => i % 4, :b => { :c => i % 5 } } }) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100} + + - def: + cd: time1 = 1375115782.24 + js: time1 = 1375115782.24 * 1000 + + - def: + cd: time2 = 1375147296.68 + js: time2 = 1375147296.68 * 1000 + + - cd: + - tbl4.insert({'id':0, 'time':r.epoch_time(time1)}) + - tbl4.insert({'id':1, 'time':r.epoch_time(time2)}) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':1} + + # GMR + + - cd: tbl.sum('a') + ot: 150 + - rb: tbl.map{|row| row['a']}.sum() + py: tbl.map(lambda row:row['a']).sum() + js: tbl.map(function(row){return row('a')}).sum() + ot: 150 + - cd: tbl.group('a').sum('id') + ot: + cd: ({0:1200, 1:1225, 2:1250, 3:1275}) + js: ([{'group':0,'reduction':1200},{'group':1,'reduction':1225},{'group':2,'reduction':1250},{'group':3,'reduction':1275}]) + - cd: tbl.avg('a') + ot: 1.5 + - rb: tbl.map{|row| row['a']}.avg() + py: tbl.map(lambda row:row['a']).avg() + js: tbl.map(function(row){return row('a')}).avg() + ot: 1.5 + - cd: tbl.group('a').avg('id') + ot: + cd: {0:48, 1:49, 2:50, 3:51} + js: [{'group':0,'reduction':48},{'group':1,'reduction':49},{'group':2,'reduction':50},{'group':3,'reduction':51}] + - cd: tbl.min('a')['a'] + js: tbl.min('a')('a') + ot: 0 + - cd: tbl.order_by('id').min('a') + ot: {'a':0, 'id':0} + - rb: tbl.map{|row| row['a']}.min() + py: tbl.map(lambda row:row['a']).min() + js: tbl.map(function(row){return row('a')}).min() + ot: 0 + - cd: tbl.group('a').min('id') + ot: + cd: {0:{'a':0, 'id':0}, 1:{'a':1, 'id':1}, 2:{'a':2, 'id':2}, 3:{'a':3, 'id':3}} + js: [{'group':0,'reduction':{'a':0, 'id':0}},{'group':1,'reduction':{'a':1, 'id':1}},{'group':2,'reduction':{'a':2, 'id':2}},{'group':3,'reduction':{'a':3, 'id':3}}] + - cd: tbl.order_by('id').max('a') + ot: {'a':3, 'id':3} + - rb: tbl.map{|row| row['a']}.max() + py: tbl.map(lambda row:row['a']).max() + js: tbl.map(function(row){return row('a')}).max() + ot: 3 + - cd: tbl.group('a').max('id') + ot: + cd: {0:{'a':0, 'id':96}, 1:{'a':1, 'id':97}, 2:{'a':2, 'id':98}, 3:{'a':3, 'id':99}} + js: [{'group':0,'reduction':{'a':0, 'id':96}},{'group':1,'reduction':{'a':1, 'id':97}},{'group':2,'reduction':{'a':2, 'id':98}},{'group':3,'reduction':{'a':3, 'id':99}}] + + - cd: tbl.min() + ot: {"a":0, "id":0} + - cd: tbl.group('a').min() + ot: + cd: {0:{"a":0, "id":0}, 1:{"a":1, "id":1}, 2:{"a":2, "id":2}, 3:{"a":3, "id":3}} + js: [{'group':0,'reduction':{"a":0,"id":0}},{'group':1,'reduction':{"a":1,"id":1}},{'group':2,'reduction':{"a":2,"id":2}},{'group':3,'reduction':{"a":3,"id":3}}] + - cd: tbl.max() + ot: {"a":3, "id":99} + - cd: tbl.group('a').max() + ot: + cd: {0:{'a':0, 'id':96}, 1:{'a':1, 'id':97}, 2:{'a':2, 'id':98}, 3:{'a':3, 'id':99}} + js: [{'group':0,'reduction':{"a":0,"id":96}},{'group':1,'reduction':{"a":1,"id":97}},{'group':2,'reduction':{"a":2,"id":98}},{'group':3,'reduction':{"a":3,"id":99}}] + + - rb: tbl.sum{|row| row['a']} + py: + - tbl.sum(lambda row:row['a']) + - tbl.sum(r.row['a']) + js: + - tbl.sum(function(row){return row('a')}) + - tbl.sum(r.row('a')) + ot: 150 + - rb: tbl.map{|row| row['a']}.sum() + py: tbl.map(lambda row:row['a']).sum() + js: tbl.map(function(row){return row('a')}).sum() + ot: 150 + - rb: tbl.group{|row| row['a']}.sum{|row| row['id']} + py: tbl.group(lambda row:row['a']).sum(lambda row:row['id']) + js: tbl.group(function(row){return row('a')}).sum(function(row){return row('id')}) + ot: + cd: {0:1200, 1:1225, 2:1250, 3:1275} + js: [{'group':0,'reduction':1200},{'group':1,'reduction':1225},{'group':2,'reduction':1250},{'group':3,'reduction':1275}] + - rb: + - tbl.avg{|row| row['a']} + py: + - tbl.avg(lambda row:row['a']) + - tbl.avg(r.row['a']) + js: + - tbl.avg(function(row){return row('a')}) + - tbl.avg(r.row('a')) + ot: 1.5 + - rb: tbl.map{|row| row['a']}.avg() + py: tbl.map(lambda row:row['a']).avg() + js: tbl.map(function(row){return row('a')}).avg() + ot: 1.5 + - rb: tbl.group{|row| row['a']}.avg{|row| row['id']} + py: tbl.group(lambda row:row['a']).avg(lambda row:row['id']) + js: tbl.group(function(row){return row('a')}).avg(function(row){return row('id')}) + ot: + cd: {0:48, 1:49, 2:50, 3:51} + js: [{'group':0,'reduction':48},{'group':1,'reduction':49},{'group':2,'reduction':50},{'group':3,'reduction':51}] + - rb: tbl.order_by(r.desc('id')).min{|row| row['a']} + py: + - tbl.order_by(r.desc('id')).min(lambda row:row['a']) + - tbl.order_by(r.desc('id')).min(r.row['a']) + js: + - tbl.order_by(r.desc('id')).min(function(row){return row('a')}) + - tbl.order_by(r.desc('id')).min(r.row('a')) + ot: {'a':0, 'id':96} + - rb: + - tbl.order_by(r.desc('id')).min{|row| row['a']}['a'] + py: + - tbl.order_by(r.desc('id')).min(lambda row:row['a'])['a'] + - tbl.order_by(r.desc('id')).min(r.row['a'])['a'] + js: + - tbl.order_by(r.desc('id')).min(function(row){return row('a')})('a') + - tbl.order_by(r.desc('id')).min(r.row('a'))('a') + ot: 0 + - rb: tbl.map{|row| row['a']}.min() + py: tbl.map(lambda row:row['a']).min() + js: tbl.map(function(row){return row('a')}).min() + ot: 0 + - rb: tbl.group{|row| row['a']}.min{|row| row['id']}['id'] + py: tbl.group(lambda row:row['a']).min(lambda row:row['id'])['id'] + js: tbl.group(function(row){return row('a')}).min(function(row){return row('id')})('id') + ot: + cd: {0:0, 1:1, 2:2, 3:3} + js: [{'group':0,'reduction':0},{'group':1,'reduction':1},{'group':2,'reduction':2},{'group':3,'reduction':3}] + - rb: + - tbl.max{|row| row['a']}['a'] + py: + - tbl.max(lambda row:row['a'])['a'] + - tbl.max(r.row['a'])['a'] + js: + - tbl.max(function(row){return row('a')})('a') + - tbl.max(r.row('a'))('a') + ot: 3 + - rb: tbl.map{|row| row['a']}.max() + py: tbl.map(lambda row:row['a']).max() + js: tbl.map(function(row){return row('a')}).max() + ot: 3 + - rb: tbl.group{|row| row['a']}.max{|row| row['id']}['id'] + py: tbl.group(lambda row:row['a']).max(lambda row:row['id'])['id'] + js: tbl.group(function(row){return row('a')}).max(function(row){return row('id')})('id') + ot: + cd: {0:96, 1:97, 2:98, 3:99} + js: [{'group':0,'reduction':96},{'group':1,'reduction':97},{'group':2,'reduction':98},{'group':3,'reduction':99}] + + - rb: tbl.group{|row| row[:a]}.map{|row| row[:id]}.reduce{|a,b| a+b} + py: tbl.group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a+b) + js: tbl.group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + ot: + cd: {0:1200, 1:1225, 2:1250, 3:1275} + js: [{'group':0,'reduction':1200},{'group':1,'reduction':1225},{'group':2,'reduction':1250},{'group':3,'reduction':1275}] + + - rb: tbl.group{|row| row[:a]}.map{|row| row[:id]}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a+b) + - tbl.group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 1200], [1, 1225], [2, 1250], [3, 1275]]} + + - cd: r.expr([{'a':1}]).filter(true).limit(1).group('a') + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[1, [{'a':1}]]]} + + # GMR + - cd: tbl.group('a').type_of() + ot: "GROUPED_STREAM" + - cd: tbl.group('a').count().type_of() + ot: "GROUPED_DATA" + - cd: tbl.group('a').coerce_to('ARRAY').type_of() + ot: "GROUPED_DATA" + + - rb: tbl.orderby(index:'id').filter{|row| row['id'].lt(10)}.group('a').map{|row| row['id']}.coerce_to('ARRAY') + py: tbl.order_by(index='id').filter(lambda row:row['id'] < 10).group('a').map(lambda row:row['id']).coerce_to('ARRAY') + js: tbl.orderBy({index:'id'}).filter(function(row){return row('id').lt(10)}).group('a').map(function(row){return row('id')}).coerce_to('ARRAY') + ot: + cd: {0:[0,4,8],1:[1,5,9],2:[2,6],3:[3,7]} + js: [{'group':0,'reduction':[0,4,8]},{'group':1,'reduction':[1,5,9]},{'group':2,'reduction':[2,6]},{'group':3,'reduction':[3,7]}] + + - rb: tbl.filter{|row| row['id'].lt(10)}.group('a').count().do{|x| x*x} + py: tbl.filter(lambda row:row['id'] < 10).group('a').count().do(lambda x:x*x) + js: tbl.filter(function(row){return row('id').lt(10)}).group('a').count().do(function(x){return x.mul(x)}) + ot: + cd: {0:9,1:9,2:4,3:4} + js: [{'group':0,'reduction':9},{'group':1,'reduction':9},{'group':2,'reduction':4},{'group':3,'reduction':4}] + + - rb: tbl.union(tbl).group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.union(tbl).group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.union(tbl).group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.union(tbl).group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.union(tbl).group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 2400], [1, 2450], [2, 2500], [3, 2550]]} + + # GMR + - rb: tbl.coerce_to("array").union(tbl).group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.coerce_to("array").union(tbl).group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.coerce_to("array").union(tbl).group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.coerce_to("array").union(tbl).group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.coerce_to("array").union(tbl).group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 2400], [1, 2450], [2, 2500], [3, 2550]]} + + # GMR + - rb: tbl.union(tbl.coerce_to("array")).group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.union(tbl.coerce_to("array")).group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.union(tbl.coerce_to("array")).group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.union(tbl.coerce_to("array")).group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.union(tbl.coerce_to("array")).group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 2400], [1, 2450], [2, 2500], [3, 2550]]} + + - py: + - tbl.group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + - tbl.group('a').map(r.row('id')).reduce(function(a,b){return a.add(b)}) + rb: tbl.group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 1200], [1, 1225], [2, 1250], [3, 1275]]} + + # undefined... + - js: + - tbl.group(function(row){}) + - tbl.map(function(row){}) + - tbl.reduce(function(row){}) + - tbl.group(r.row('a')).group(function(row){}) + - tbl.group(r.row('a')).map(function(row){}) + - tbl.group(r.row('a')).reduce(function(row){}) + - tbl.map(r.row('id')).group(function(row){}) + - tbl.map(r.row('id')).map(function(row){}) + - tbl.map(r.row('id')).reduce(function(row){}) + - tbl.reduce(function(a,b){return a+b}).group(function(row){}) + - tbl.reduce(function(a,b){return a+b}).map(function(row){}) + - tbl.reduce(function(a,b){return a+b}).reduce(function(row){}) + ot: err('ReqlDriverCompileError', 'Anonymous function returned `undefined`. Did you forget a `return`?', [0]) + + # GroupBy + + # COUNT + + - cd: tbl.group('a').count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 25], [1, 25], [2, 25], [3, 25]]} + + # SUM + - cd: tbl.group('a').sum('id') + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 1200], [1, 1225], [2, 1250], [3, 1275]]} + + # AVG + - cd: tbl.group('a').avg('id') + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 48], [1, 49], [2, 50], [3, 51]]} + + # Pattern Matching + - rb: tbl3.group{|row| row['b']['c']}.count() + py: tbl3.group(lambda row:row['b']['c']).count() + js: tbl3.group(function(row){return row('b')('c')}).count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 20], [1, 20], [2, 20], [3, 20], [4, 20]]} + + # Multiple keys + - rb: tbl.group('a', lambda {|row| row['id']%3}).count() + py: tbl.group('a', lambda row:row['id'].mod(3)).count() + js: tbl.group('a', function(row){return row('id').mod(3)}).count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, 0], 9], [[0, 1], 8], [[0, 2], 8], [[1, 0], 8], [[1, 1], 9], [[1, 2], 8], [[2, 0], 8], [[2, 1], 8], [[2, 2], 9], [[3, 0], 9], [[3, 1], 8], [[3, 2], 8]]} + + # Grouping by time + - rb: tbl4.group('time').coerce_to('array') + runopts: + time_format: 'raw' + ot: + rb: {{"$reql_type$":"TIME","epoch_time":1375115782.24,"timezone":"+00:00"}:[{"id":0,"time":{"$reql_type$":"TIME","epoch_time":1375115782.24,"timezone":"+00:00"}}],{"$reql_type$":"TIME","epoch_time":1375147296.68,"timezone":"+00:00"}:[{"id":1,"time":{"$reql_type$":"TIME","epoch_time":1375147296.68,"timezone":"+00:00"}}]} + py: {frozenset([('$reql_type$','TIME'),('timezone','+00:00'),('epoch_time',1375115782.24)]):[{'id':0,'time':{'timezone':'+00:00','$reql_type$':'TIME','epoch_time':1375115782.24}}],frozenset([('$reql_type$','TIME'),('timezone','+00:00'),('epoch_time',1375147296.68)]):[{'id':1,'time':{'timezone':'+00:00','$reql_type$':'TIME','epoch_time':1375147296.68}}]} + js: [{'group':{"$reql_type$":"TIME","epoch_time":1375115782240,"timezone":"+00:00"},'reduction':[{"id":0,"time":{"$reql_type$":"TIME","epoch_time":1375115782240,"timezone":"+00:00"}}]},{'group':{"$reql_type$":"TIME","epoch_time":1375147296680,"timezone":"+00:00"},'reduction':[{"id":1,"time":{"$reql_type$":"TIME","epoch_time":1375147296680,"timezone":"+00:00"}}]}] + + # Distinct + - py: tbl.map(lambda row:row['a']).distinct().count() + js: tbl.map(function(row) { return row('a'); }).distinct().count() + rb: tbl.map{ |row| row[:a] }.distinct.count + ot: 4 + + - cd: tbl.distinct().type_of() + ot: "STREAM" + + - cd: tbl.distinct().count() + ot: 100 + + - cd: tbl.distinct({index:'id'}).type_of() + py: tbl.distinct(index='id').type_of() + ot: "STREAM" + + - cd: tbl.distinct({index:'id'}).count() + py: tbl.distinct(index='id').count() + ot: 100 + + - cd: tbl.index_create('a') + ot: {'created':1} + + - rb: tbl.index_create('m', multi:true){|row| [row['a'], row['a']]} + ot: {'created':1} + + - rb: tbl.index_create('m2', multi:true){|row| [1, 2]} + ot: {'created':1} + + - cd: tbl.index_wait('a').pluck('index', 'ready') + ot: [{'index':'a','ready':true}] + + - rb: tbl.index_wait('m').pluck('index', 'ready') + ot: [{'index':'m','ready':true}] + + - rb: tbl.index_wait('m2').pluck('index', 'ready') + ot: [{'index':'m2','ready':true}] + + - cd: tbl.between(0, 1, {index:'a'}).distinct().count() + py: tbl.between(0, 1, index='a').distinct().count() + ot: 25 + + - cd: tbl.between(0, 1, {index:'a'}).distinct({index:'id'}).count() + py: tbl.between(0, 1, index='a').distinct(index='id').count() + ot: 25 + + - rb: tbl.between(0, 1, {index:'m'}).count() + ot: 50 + + - rb: tbl.between(0, 1, {index:'m'}).distinct().count() + ot: 25 + + - rb: tbl.orderby({index:'m'}).count() + ot: 200 + + - rb: tbl.orderby({index:'m'}).distinct().count() + ot: 100 + + - rb: tbl.orderby({index:r.desc('m')}).count() + ot: 200 + + - rb: tbl.orderby({index:r.desc('m')}).distinct().count() + ot: 100 + + - rb: tbl.between(1, 3, {index:'m2'}).count() + ot: 200 + + - rb: tbl.between(1, 3, {index:'m2'}).distinct().count() + ot: 100 + + - rb: tbl.between(1, 3, {index:'m2'}).orderby(index:r.desc('m2')).distinct().count() + ot: 100 + + - rb: tbl.between(0, 1, {index:'m'}).count() + ot: 50 + + - rb: tbl.between(0, 1, {index:'m'}).distinct().count() + ot: 25 + + - cd: tbl.distinct({index:'a'}).type_of() + py: tbl.distinct(index='a').type_of() + ot: "STREAM" + + - cd: tbl.distinct({index:'a'}).count() + py: tbl.distinct(index='a').count() + ot: 4 + + - cd: tbl.group() + ot: err('ReqlQueryLogicError', 'Cannot group by nothing.', []) + + - py: tbl.group(index='id').count() + js: tbl.group({index:'id'}).count() + cd: tbl.group(index:'id').count + runopts: + group_format: 'raw' + ot: ({'$reql_type$':'GROUPED_DATA', 'data':[[0, 1], [1, 1], [2, 1], [3, 1], [4, 1], [5, 1], [6, 1], [7, 1], [8, 1], [9, 1], [10, 1], [11, 1], [12, 1], [13, 1], [14, 1], [15, 1], [16, 1], [17, 1], [18, 1], [19, 1], [20, 1], [21, 1], [22, 1], [23, 1], [24, 1], [25, 1], [26, 1], [27, 1], [28, 1], [29, 1], [30, 1], [31, 1], [32, 1], [33, 1], [34, 1], [35, 1], [36, 1], [37, 1], [38, 1], [39, 1], [40, 1], [41, 1], [42, 1], [43, 1], [44, 1], [45, 1], [46, 1], [47, 1], [48, 1], [49, 1], [50, 1], [51, 1], [52, 1], [53, 1], [54, 1], [55, 1], [56, 1], [57, 1], [58, 1], [59, 1], [60, 1], [61, 1], [62, 1], [63, 1], [64, 1], [65, 1], [66, 1], [67, 1], [68, 1], [69, 1], [70, 1], [71, 1], [72, 1], [73, 1], [74, 1], [75, 1], [76, 1], [77, 1], [78, 1], [79, 1], [80, 1], [81, 1], [82, 1], [83, 1], [84, 1], [85, 1], [86, 1], [87, 1], [88, 1], [89, 1], [90, 1], [91, 1], [92, 1], [93, 1], [94, 1], [95, 1], [96, 1], [97, 1], [98, 1], [99, 1]]}) + + - py: tbl.group(index='a').count() + js: tbl.group({index:'a'}).count() + rb: tbl.group(index:'a').count + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 25], [1, 25], [2, 25], [3, 25]]} + + - py: tbl.group('a', index='id').count() + js: tbl.group('a', {index:'id'}).count() + rb: tbl.group('a', index:'id').count + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, 0], 1], [[0, 4], 1], [[0, 8], 1], [[0, 12], 1], [[0, 16], 1], [[0, 20], 1], [[0, 24], 1], [[0, 28], 1], [[0, 32], 1], [[0, 36], 1], [[0, 40], 1], [[0, 44], 1], [[0, 48], 1], [[0, 52], 1], [[0, 56], 1], [[0, 60], 1], [[0, 64], 1], [[0, 68], 1], [[0, 72], 1], [[0, 76], 1], [[0, 80], 1], [[0, 84], 1], [[0, 88], 1], [[0, 92], 1], [[0, 96], 1], [[1, 1], 1], [[1, 5], 1], [[1, 9], 1], [[1, 13], 1], [[1, 17], 1], [[1, 21], 1], [[1, 25], 1], [[1, 29], 1], [[1, 33], 1], [[1, 37], 1], [[1, 41], 1], [[1, 45], 1], [[1, 49], 1], [[1, 53], 1], [[1, 57], 1], [[1, 61], 1], [[1, 65], 1], [[1, 69], 1], [[1, 73], 1], [[1, 77], 1], [[1, 81], 1], [[1, 85], 1], [[1, 89], 1], [[1, 93], 1], [[1, 97], 1], [[2, 2], 1], [[2, 6], 1], [[2, 10], 1], [[2, 14], 1], [[2, 18], 1], [[2, 22], 1], [[2, 26], 1], [[2, 30], 1], [[2, 34], 1], [[2, 38], 1], [[2, 42], 1], [[2, 46], 1], [[2, 50], 1], [[2, 54], 1], [[2, 58], 1], [[2, 62], 1], [[2, 66], 1], [[2, 70], 1], [[2, 74], 1], [[2, 78], 1], [[2, 82], 1], [[2, 86], 1], [[2, 90], 1], [[2, 94], 1], [[2, 98], 1], [[3, 3], 1], [[3, 7], 1], [[3, 11], 1], [[3, 15], 1], [[3, 19], 1], [[3, 23], 1], [[3, 27], 1], [[3, 31], 1], [[3, 35], 1], [[3, 39], 1], [[3, 43], 1], [[3, 47], 1], [[3, 51], 1], [[3, 55], 1], [[3, 59], 1], [[3, 63], 1], [[3, 67], 1], [[3, 71], 1], [[3, 75], 1], [[3, 79], 1], [[3, 83], 1], [[3, 87], 1], [[3, 91], 1], [[3, 95], 1], [[3, 99], 1]]} + + - py: tbl.group('a', index='a').count() + js: tbl.group('a', {index:'a'}).count() + rb: tbl.group('a', index:'a').count + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, 0], 25], [[1, 1], 25], [[2, 2], 25], [[3, 3], 25]]} + + - rb: tbl.group('a', lambda {|row| 'f'}, lambda {|row| []}, lambda {|row| [{}, [0], null, 0]}, multi:true).count + py: tbl.group('a', lambda row:'f', lambda row:[], lambda row:[{}, [0], null, 0], multi=True).count() + js: tbl.group('a', function(row){return 'f';}, function(row){return [];}, function(row){return [{}, [0], null, 0];}, {multi:true}).count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, "f", null, [0]], 25], [[0, "f", null, null], 25], [[0, "f", null, 0], 25], [[0, "f", null, {}], 25], [[1, "f", null, [0]], 25], [[1, "f", null, null], 25], [[1, "f", null, 0], 25], [[1, "f", null, {}], 25], [[2, "f", null, [0]], 25], [[2, "f", null, null], 25], [[2, "f", null, 0], 25], [[2, "f", null, {}], 25], [[3, "f", null, [0]], 25], [[3, "f", null, null], 25], [[3, "f", null, 0], 25], [[3, "f", null, {}], 25]]} + + - cd: tbl.group('a').count().ungroup() + ot: [{'group':0, 'reduction':25}, {'group':1, 'reduction':25}, {'group':2, 'reduction':25}, {'group':3, 'reduction':25}] + + - cd: tbl.group('a').ungroup()['group'] + js: tbl.group('a').ungroup()('group') + ot: [0, 1, 2, 3] + + - py: tbl.order_by(index='id').limit(16).group('a','a').map(r.row['id']).sum().ungroup() + js: tbl.order_by({index:'id'}).limit(16).group('a','a').map(r.row('id')).sum().ungroup() + rb: tbl.order_by(index:'id').limit(16).group('a','a').map{|row| row['id']}.sum().ungroup() + ot: [{'group':[0,0],'reduction':24},{'group':[1,1],'reduction':28},{'group':[2,2],'reduction':32},{'group':[3,3],'reduction':36}] + + - cd: tbl.group('a', null).count().ungroup() + ot: [{'group':[0,null],'reduction':25},{'group':[1,null],'reduction':25},{'group':[2,null],'reduction':25},{'group':[3,null],'reduction':25}] + + - py: tbl.group('a', lambda row:[1,'two'], multi=True).count().ungroup() + js: tbl.group('a', function(row){return [1,'two']},{multi:true}).count().ungroup() + rb: tbl.group('a', lambda {|row| [1,'two']}, multi:true).count().ungroup() + ot: [{'group':[0,1],'reduction':25},{'group':[0,'two'],'reduction':25},{'group':[1,1],'reduction':25},{'group':[1,'two'],'reduction':25},{'group':[2,1],'reduction':25},{'group':[2,'two'],'reduction':25},{'group':[3,1],'reduction':25},{'group':[3,'two'],'reduction':25}] + + # proper test for seq.count() + - cd: tbl.count() + ot: 100 + + - js: tbl.filter(r.row('a').ne(1).and(r.row('id').gt(10))).update({'b':r.row('a').mul(10)}) + py: tbl.filter(r.row['a'].ne(1).and_(r.row['id'].gt(10))).update({'b':r.row['a'] * 10}) + rb: tbl.filter{|row| row['a'].ne(1).and(row['id'].gt(10))}.update{|row| {'b'=>row['a'] * 10}} + ot: partial({'errors':0, 'replaced':67}) + + - cd: tbl.group('b').count() + ot: + cd: {null:33, 0:22, 20:22, 30:23} + js: [{"group":null, "reduction":33}, {"group":0, "reduction":22}, {"group":20, "reduction":22}, {"group":30, "reduction":23}] + + - cd: tbl.group('a').sum('b') + ot: + cd: {0:0, 2:440, 3:690} + js: [{"group":0, "reduction":0}, {"group":2, "reduction":440}, {"group":3, "reduction":690}] + + - cd: tbl.group('a').avg('b') + ot: + cd: {0:0, 2:20, 3:30} + js: [{"group":0, "reduction":0}, {"group":2, "reduction":20}, {"group":3, "reduction":30}] + + - cd: tbl.order_by('id').group('a').min('b') + ot: + cd: {0:{"a":0, "b":0, "id":12}, 2:{"a":2, "b":20, "id":14}, 3:{"a":3, "b":30, "id":11}} + js: [{"group":0, "reduction":{"a":0, "b":0, "id":12}}, {"group":2, "reduction":{"a":2, "b":20, "id":14}}, {"group":3, "reduction":{"a":3, "b":30, "id":11}}] + + - cd: tbl.order_by('id').group('a').min('id') + ot: + cd: {0:{"a":0, "id":0}, 1:{"a":1, "id":1}, 2:{"a":2, "id":2}, 3:{"a":3, "id":3}} + js: [{"group":0, "reduction":{"a":0, "id":0}}, {"group":1, "reduction":{"a":1, "id":1}}, {"group":2, "reduction":{"a":2, "id":2}}, {"group":3, "reduction":{"a":3, "id":3}}] + + - cd: tbl.order_by('id').group('a').max('b') + ot: + cd: {0:{"a":0, "b":0, "id":12}, 2:{"a":2, "b":20, "id":14}, 3:{"a":3, "b":30, "id":11}} + js: [{"group":0, "reduction":{"a":0,"b":0, "id":12}}, {"group":2, "reduction":{"a":2, "b":20, "id":14}}, {"group":3, "reduction":{"a":3, "b":30, "id":11}}] + + - cd: tbl.min() + ot: {'a':0,'id':0} + - py: tbl.min(index='id') + rb: tbl.min(index:'id') + js: tbl.min({index:'id'}) + ot: {'a':0,'id':0} + - py: tbl.min(index='a') + rb: tbl.min(index:'a') + js: tbl.min({index:'a'}) + ot: {'a':0,'id':0} + + - cd: tbl.max().without('b') + ot: {'a':3,'id':99} + - py: tbl.max(index='id').without('b') + rb: tbl.max(index:'id').without('b') + js: tbl.max({index:'id'}).without('b') + ot: {'a':3,'id':99} + - py: tbl.max(index='a').without('b') + rb: tbl.max(index:'a').without('b') + js: tbl.max({index:'a'}).without('b') + ot: {'a':3,'id':99} + + + # Infix + + - cd: r.group([ 1, 1, 2 ], r.row).count().ungroup() + rb: r.group([ 1, 1, 2 ]) {|row| row}.count().ungroup() + ot: [ {'group': 1, 'reduction': 2}, {'group': 2, 'reduction': 1} ] + - cd: + - r.count([ 1, 2 ]) + - r.count([ 1, 2 ], r.row.gt(0)) + rb: + - r.count([ 1, 2 ]) + - r.count([ 1, 2 ]) {|row| row.gt(0)} + ot: 2 + - cd: + - r.sum([ 1, 2 ]) + - r.sum([ 1, 2 ], r.row) + rb: r.sum([ 1, 2 ]) + ot: 3 + - cd: + - r.avg([ 1, 2 ]) + - r.avg([ 1, 2 ], r.row) + rb: r.avg([ 1, 2 ]) + ot: 1.5 + - cd: + - r.min([ 1, 2 ]) + - r.min([ 1, 2 ], r.row) + rb: r.min([ 1, 2 ]) + ot: 1 + - cd: + - r.max([ 1, 2 ]) + - r.max([ 1, 2 ], r.row) + rb: r.max([ 1, 2 ]) + ot: 2 + - cd: r.distinct([ 1, 1 ]) + ot: [ 1 ] + - cd: + - r.contains([ 1, 2 ]) + - r.contains([ 1, 2 ], r.row.gt(0)) + rb: + - r.contains([ 1, 2 ]) + - r.contains([ 1, 2 ]) {|row| row.gt(0)} + ot: true diff --git a/ext/librethinkdbxx/test/upstream/arity.yaml b/ext/librethinkdbxx/test/upstream/arity.yaml new file mode 100644 index 00000000..8197fe44 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/arity.yaml @@ -0,0 +1,316 @@ +desc: Test the arity of every function +table_variable_name: tbl +tests: + + # TODO: add test for slice (should require one or two arguments) + + # Set up some data + - def: db = r.db('test') + - def: obj = r.expr({'a':1}) + - def: array = r.expr([1]) + + - ot: err("ReqlCompileError", "Expected 0 arguments but found 1.", []) + cd: r.db_list(1) + + - ot: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + cd: + - tbl.zip(1) + - tbl.is_empty(1) + - obj.keys(1) + + - cd: tbl.distinct(1) + ot: + cd: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + js: err("ReqlCompileError", "Expected 0 arguments (not including options) but found 1.", []) + + - cd: tbl.delete(1) + ot: + js: err("ReqlCompileError", "Expected 0 arguments (not including options) but found 1.", []) + cd: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + + - rb: db.table_list(1) + ot: err("ReqlCompileError", "Expected between 0 and 1 arguments but found 2.", []) + + - ot: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + cd: + - r.db_create() + - r.db_drop() + - r.db() + - r.floor() + - r.ceil() + - r.round() + + - cd: r.error() + ot: err("ReqlQueryLogicError", "Empty ERROR term outside a default block.", []) + + - cd: r.js() + ot: + cd: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + + - cd: r.expr() + ot: + py3.3: err_regex('TypeError', '.* missing 1 required positional argument.*', []) + py3.4: err_regex('TypeError', '.* missing 1 required positional argument.*', []) + py3.5: err_regex('TypeError', '.* missing 1 required positional argument.*', []) + py: err_regex('TypeError', ".* takes at least 1 (?:positional )?argument \(0 given\)", []) + js: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 0.", []) + rb: err("ArgumentError", 'wrong number of arguments (0 for 1)', []) + rb2: err("ArgumentError", 'wrong number of arguments (0 for 1..2)', []) + + - ot: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + cd: + - tbl.concat_map() + - tbl.skip() + - tbl.limit() + - array.append() + - array.prepend() + - array.difference() + - array.set_insert() + - array.set_union() + - array.set_intersection() + - array.set_difference() + - tbl.nth() + - tbl.for_each() + - tbl.get() + - r.expr([]).sample() + - tbl.offsets_of() + - ot: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + cd: + - r.db_create(1,2) + - r.db_drop(1,2) + - r.db(1,2) + - r.floor(1, 2) + - r.ceil(1, 2) + - r.round(1, 2) + + - cd: tbl.filter() + ot: + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + cd: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + + - cd: r.error(1, 2) + ot: err("ReqlCompileError", "Expected between 0 and 1 arguments but found 2.", []) + + - cd: db.table_drop() + ot: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + + + - cd: db.table_create() + ot: + cd: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + + - cd: r.js(1,2) + ot: + cd: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", []) + + - ot: err("ReqlCompileError", "Expected 2 arguments but found 3.", []) + cd: + - tbl.concat_map(1,2) + - tbl.skip(1,2) + - tbl.limit(1,2) + - array.append(1,2) + - array.prepend(1,2) + - array.difference([], []) + - array.set_insert(1,2) + - array.set_union([1],[2]) + - array.set_intersection([1],[2]) + - array.set_difference([1],[2]) + - tbl.nth(1,2) + - tbl.for_each(1,2) + - tbl.get(1,2) + - r.expr([]).sample(1,2) + - tbl.offsets_of(1,2) + + - cd: tbl.filter(1,2,3) + ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 4.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 3.", []) + + - cd: db.table_drop(1,2) + ot: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + + - cd: r.expr([]).delete_at() + ot: err("ReqlCompileError", "Expected between 2 and 3 arguments but found 1.", []) + + - cd: db.table_create(1,2) + ot: + cd: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", []) + + - cd: tbl.count(1,2) + ot: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + + - ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + cd: + - tbl.update() + - tbl.replace() + - tbl.insert() + + - cd: db.table() + ot: + cd: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + + - cd: tbl.reduce() + ot: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + + - cd: tbl.eq_join() + ot: + cd: err("ReqlCompileError", "Expected 3 arguments but found 1.", []) + js: err("ReqlCompileError", "Expected 2 arguments (not including options) but found 0.", []) + + - ot: err("ReqlCompileError", "Expected 3 arguments but found 1.", []) + cd: + - tbl.inner_join() + - tbl.outer_join() + - r.expr([]).insert_at() + - r.expr([]).splice_at() + - r.expr([]).change_at() + + - cd: tbl.eq_join(1) + ot: + cd: err("ReqlCompileError", "Expected 3 arguments but found 2.", []) + js: err("ReqlCompileError", "Expected 2 arguments (not including options) but found 1.", []) + + - ot: err("ReqlCompileError", "Expected 3 arguments but found 2.", []) + cd: + - tbl.inner_join(1) + - tbl.outer_join(1) + - r.expr([]).insert_at(1) + - r.expr([]).splice_at(1) + - r.expr([]).change_at(1) + + - cd: tbl.eq_join(1,2,3,4) + ot: + cd: err("ReqlCompileError", "Expected 3 arguments but found 5.", []) + js: err("ReqlCompileError", "Expected 2 arguments (not including options) but found 4.", []) + + - ot: err("ReqlCompileError", "Expected 3 arguments but found 4.", []) + cd: + - tbl.inner_join(1,2,3) + - tbl.outer_join(1,2,3) + - r.expr([]).insert_at(1, 2, 3) + - r.expr([]).splice_at(1, 2, 3) + - r.expr([]).change_at(1, 2, 3) + + - cd: tbl.map() + ot: + cd: err('ReqlCompileError', "Expected 2 or more arguments but found 1.", []) + js: err('ReqlCompileError', "Expected 1 or more arguments but found 0.", []) + + - cd: r.branch(1,2) + ot: err("ReqlCompileError", "Expected 3 or more arguments but found 2.", []) + - cd: r.branch(1,2,3,4) + ot: err("ReqlQueryLogicError", "Cannot call `branch` term with an even number of arguments.", []) + + - cd: r.expr({})[1,2] + js: r.expr({})(1,2) + ot: + js: err('ReqlCompileError', "Expected 1 argument but found 2.", []) + py: err('ReqlQueryLogicError', 'Expected NUMBER or STRING as second argument to `bracket` but found ARRAY.') + rb: err('ArgumentError', 'wrong number of arguments (2 for 1)') + + - cd: tbl.insert([{'id':0},{'id':1},{'id':2},{'id':3},{'id':4},{'id':5},{'id':6},{'id':7},{'id':8},{'id':9}]).get_field('inserted') + ot: 10 + + - cd: tbl.get_all(0, 1, 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([]), 0, 1, 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0]), 1, 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0, 1]), 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0, 1, 2])).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0]), 1, r.args([2])).get_field('id') + ot: bag([0, 1, 2]) + + # Make sure partial-evaluation still works + + - cd: r.branch(true, 1, r.error("a")) + ot: 1 + + - cd: r.branch(r.args([true, 1]), r.error("a")) + ot: 1 + + - cd: r.expr(true).branch(1, 2) + ot: 1 + + - cd: r.branch(r.args([true, 1, r.error("a")])) + ot: err("ReqlUserError", "a", []) + + # Make sure our grouped data hack still works + + - rb: tbl.group{|row| row['id'] % 2}.count({'id':0}).ungroup() + py: tbl.group(lambda row:row['id'].mod(2)).count({'id':0}).ungroup() + js: tbl.group(r.row('id').mod(2)).count({'id':0}).ungroup() + ot: ([{'group':0, 'reduction':1}]) + + - rb: tbl.group{|row| row['id'] % 2}.count(r.args([{'id':0}])).ungroup() + py: tbl.group(r.row['id'].mod(2)).count(r.args([{'id':0}])).ungroup() + js: tbl.group(r.row('id').mod(2)).count(r.args([{'id':0}])).ungroup() + ot: ([{'group':0, 'reduction':1}]) + + # Make sure `r.literal` still works + + - cd: r.expr({'a':{'b':1}}).merge(r.args([{'a':r.literal({'c':1})}])) + ot: ({'a':{'c':1}}) + + - cd: r.http("httpbin.org/get","bad_param") + ot: + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", []) + rb: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + py: err_regex('TypeError', ".*takes exactly 1 argument \(2 given\)", []) + py3.0: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.1: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.2: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3: err_regex('TypeError', ".*takes 1 positional argument but 2 were given", []) + + - cd: r.binary("1", "2") + ot: + py: err_regex('TypeError', ".*takes exactly 1 argument \(2 given\)", []) + js: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + rb: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + py3.0: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.1: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.2: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3: err_regex('TypeError', ".*takes 1 positional argument but 2 were given", []) + - cd: r.binary() + ot: + py: err_regex('TypeError', ".*takes exactly 1 argument \(0 given\)", []) + js: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + rb: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + py3.0: err_regex('TypeError', ".*takes exactly 1 positional argument \(0 given\)", []) + py3.1: err_regex('TypeError', ".*takes exactly 1 positional argument \(0 given\)", []) + py3.2: err_regex('TypeError', ".*takes exactly 1 argument \(0 given\)", []) + py3: err_regex('TypeError', ".* missing 1 required positional argument.*", []) + + # TODO: Math and logic + # TODO: Upper bound on optional arguments + # TODO: between, merge, slice + + - cd: tbl.index_rename('idx') + ot: + cd: err('ReqlCompileError','Expected 3 arguments but found 2.',[]) + js: err('ReqlCompileError','Expected 2 arguments (not including options) but found 1.',[]) + + - cd: tbl.index_rename('idx','idx2','idx3') + ot: + cd: err('ReqlCompileError','Expected 3 arguments but found 4.',[]) + js: err('ReqlCompileError','Expected 2 arguments (not including options) but found 3.',[]) + + - cd: + - r.now('foo') + - r.now(r.args([1,2,3])) + ot: err('ReqlCompileError','NOW does not accept any args.') diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml new file mode 100644 index 00000000..e66b6847 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml @@ -0,0 +1,142 @@ +desc: Test edge cases of changefeed operations +table_variable_name: tbl +tests: + + - def: common_prefix = r.expr([0,1,2,3,4,5,6,7,8]) + + - js: tbl.indexCreate('sindex', function (row) { return common_prefix.append(row('value')); }) + py: tbl.index_create('sindex', lambda row:common_prefix.append(row['value'])) + rb: tbl.index_create('sindex'){ |row| common_prefix.append(row['value']) } + ot: ({'created':1}) + - cd: tbl.index_wait('sindex') + + # create target values + - cd: pre = r.range(7).coerce_to('array').add(r.range(10,70).coerce_to('array')).append(100).map(r.row.coerce_to('string')) + rb: pre = r.range(7).coerce_to('array').add(r.range(10,70).coerce_to('array')).append(100).map{ |row| row.coerce_to('string') } + - cd: mid = r.range(2,9).coerce_to('array').add(r.range(20,90).coerce_to('array')).map(r.row.coerce_to('string')) + rb: mid = r.range(2,9).coerce_to('array').add(r.range(20,90).coerce_to('array')).map{ |row| row.coerce_to('string') } + - cd: post = r.range(3,10).coerce_to('array').add(r.range(30,100).coerce_to('array')).map(r.row.coerce_to('string')) + rb: post = r.range(3,10).coerce_to('array').add(r.range(30,100).coerce_to('array')).map{ |row| row.coerce_to('string') } + + - cd: erroredres = r.range(2).coerce_to('array').add(r.range(10, 20).coerce_to('array')).append(100).map(r.row.coerce_to('string')) + rb: erroredres = r.range(2).coerce_to('array').add(r.range(10, 20).coerce_to('array')).append(100).map{ |val| val.coerce_to('string') } + + # Start overlapping changefeeds + - js: pre_changes = tbl.between(r.minval, commonPrefix.append('7'), {index:'sindex'}).changes({squash:false}).limit(pre.length)('new_val')('value') + py: pre_changes = tbl.between(r.minval, common_prefix.append('7'), index='sindex').changes(squash=False).limit(len(pre))['new_val']['value'] + rb: pre_changes = tbl.between(r.minval, common_prefix.append('7'), index:'sindex').changes(squash:false).limit(pre.length)['new_val']['value'] + - js: mid_changes = tbl.between(commonPrefix.append('2'), common_prefix.append('9'), {index:'sindex'}).changes({squash:false}).limit(post.length)('new_val')('value') + py: mid_changes = tbl.between(common_prefix.append('2'), common_prefix.append('9'), index='sindex').changes(squash=False).limit(len(post))['new_val']['value'] + rb: mid_changes = tbl.between(common_prefix.append('2'), common_prefix.append('9'), index:'sindex').changes(squash:false).limit(post.length)['new_val']['value'] + - js: post_changes = tbl.between(commonPrefix.append('3'), r.maxval, {index:'sindex'}).changes({squash:false}).limit(mid.length)('new_val')('value') + py: post_changes = tbl.between(common_prefix.append('3'), r.maxval, index='sindex').changes(squash=False).limit(len(mid))['new_val']['value'] + rb: post_changes = tbl.between(common_prefix.append('3'), r.maxval, index:'sindex').changes(squash:false).limit(mid.length)['new_val']['value'] + + # Start changefeeds with non-existence errors + + - js: premap_changes1 = tbl.map(r.branch(r.row('value').lt('2'), r.row, r.row("dummy"))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: premap_changes1 = tbl.map(r.branch(r.row['value'].lt('2'), r.row, r.row["dummy"])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: premap_changes1 = tbl.map{ |row| r.branch(row['value'].lt('2'), row, row["dummy"]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postmap_changes1 = tbl.changes({squash:false}).map(r.branch(r.row('new_val')('value').lt('2'), r.row, r.row("dummy"))).limit(erroredres.length)('new_val')('value') + py: postmap_changes1 = tbl.changes(squash=False).map(r.branch(r.row['new_val']['value'].lt('2'), r.row, r.row["dummy"])).limit(len(erroredres))['new_val']['value'] + rb: postmap_changes1 = tbl.changes(squash:false).map{ |row| r.branch(row['new_val']['value'].lt('2'), row, row["dummy"]) }.limit(erroredres.length)['new_val']['value'] + + - js: prefilter_changes1 = tbl.filter(r.branch(r.row('value').lt('2'), true, r.row("dummy"))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: prefilter_changes1 = tbl.filter(r.branch(r.row['value'].lt('2'), True, r.row["dummy"])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: prefilter_changes1 = tbl.filter{ |row| r.branch(row['value'].lt('2'), true, row["dummy"]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postfilter_changes1 = tbl.changes({squash:false}).filter(r.branch(r.row('new'+'_'+'val')('value').lt('2'), true, r.row("dummy"))).limit(erroredres.length)('new_val')('value') + py: postfilter_changes1 = tbl.changes(squash=False).filter(r.branch(r.row['new_val']['value'].lt('2'), True, r.row["dummy"])).limit(len(erroredres))['new_val']['value'] + rb: postfilter_changes1 = tbl.changes(squash:false).filter{ |row| r.branch(row['new_val']['value'].lt('2'), true, row["dummy"]) }.limit(erroredres.length)['new_val']['value'] + + # Start changefeeds with runtime errors + + - js: premap_changes2 = tbl.map(r.branch(r.row('value').lt('2'), r.row, r.expr([]).nth(1))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: premap_changes2 = tbl.map(r.branch(r.row['value'].lt('2'), r.row, r.expr([])[1])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: premap_changes2 = tbl.map{ |row| r.branch(row['value'].lt('2'), row, r.expr([])[1]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postmap_changes2 = tbl.changes({squash:false}).map(r.branch(r.row('new'+'_'+'val')('value').lt('2'), r.row, r.expr([]).nth(1))).limit(erroredres.length)('new_val')('value') + py: postmap_changes2 = tbl.changes(squash=False).map(r.branch(r.row['new_val']['value'].lt('2'), r.row, r.expr([])[1])).limit(len(erroredres))['new_val']['value'] + rb: postmap_changes2 = tbl.changes(squash:false).map{ |row| r.branch(row['new_val']['value'].lt('2'), row, r.expr([])[1]) }.limit(erroredres.length)['new_val']['value'] + + - js: prefilter_changes2 = tbl.filter(r.branch(r.row('value').lt('2'), true, r.expr([]).nth(1))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: prefilter_changes2 = tbl.filter(r.branch(r.row['value'].lt('2'), True, r.expr([])[1])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: prefilter_changes2 = tbl.filter{ |row| r.branch(row['value'].lt('2'), true, r.expr([])[1]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postfilter_changes2 = tbl.changes({squash:false}).filter(r.branch(r.row('new'+'_'+'val')('value').lt('2'), true, r.expr([]).nth(1))).limit(erroredres.length)('new_val')('value') + py: postfilter_changes2 = tbl.changes(squash=False).filter(r.branch(r.row['new_val']['value'].lt('2'), True, r.expr([])[1])).limit(len(erroredres))['new_val']['value'] + rb: postfilter_changes2 = tbl.changes(squash:false).filter{ |row| r.branch(row['new_val']['value'].lt('2'), true, r.expr([])[1]) }.limit(erroredres.length)['new_val']['value'] + + # Start non-deterministic changefeeds - very small chance of these hanging due to not enough results + - def: + py: nondetermmap = r.branch(r.random().gt(0.5), r.row, r.error("dummy")) + js: nondetermmap = function (row) { return r.branch(r.random().gt(0.5), row, r.error("dummy")); } + rb: nondetermmap = Proc.new { |row| r.branch(r.random().gt(0.5), row, r.error("dummy")) } + - def: + py: nondetermfilter = lambda row:r.random().gt(0.5) + js: nondetermfilter = function (row) { return r.random().gt(0.5); } + rb: nondetermfilter = Proc.new { |row| r.random().gt(0.5) } + + - rb: tbl.map(nondetermmap).changes(squash:false) + js: tbl.map(nondetermmap).changes({squash:false}) + py: tbl.map(nondetermmap).changes(squash=False) + ot: err('ReqlQueryLogicError', 'Cannot call `changes` after a non-deterministic function.') + + - rb: postmap_changes3 = tbl.changes(squash:false).map(nondetermmap).limit(100) + js: postmap_changes3 = tbl.changes({squash:false}).map(nondetermmap).limit(100) + py: postmap_changes3 = tbl.changes(squash=False).map(nondetermmap).limit(100) + + - rb: tbl.filter(nondetermfilter).changes(squash:false) + js: tbl.filter(nondetermfilter).changes({squash:false}) + py: tbl.filter(nondetermfilter).changes(squash=False) + ot: err('ReqlQueryLogicError', 'Cannot call `changes` after a non-deterministic function.') + + - rb: postfilter_changes3 = tbl.changes(squash:false).filter(nondetermfilter).limit(4) + js: postfilter_changes3 = tbl.changes({squash:false}).filter(nondetermfilter).limit(4) + py: postfilter_changes3 = tbl.changes(squash=False).filter(nondetermfilter).limit(4) + + # Insert several rows that will and will not be returned + - cd: tbl.insert(r.range(101).map({'id':r.uuid().coerce_to('binary').slice(0,r.random(4,24)).coerce_to('string'),'value':r.row.coerce_to('string')})) + rb: tbl.insert(r.range(101).map{ |row| {'id'=>r.uuid().coerce_to('binary').slice(0,r.random(4,24)).coerce_to('string'),'value'=>row.coerce_to('string')}}) + ot: ({'skipped':0,'deleted':0,'unchanged':0,'errors':0,'replaced':0,'inserted':101}) + + # Check that our limited watchers have been satified + - cd: pre_changes + ot: bag(pre) + + - cd: mid_changes + ot: bag(mid) + + - cd: post_changes + ot: bag(post) + + - cd: premap_changes1 + ot: bag(erroredres) + + - cd: premap_changes2 + ot: bag(erroredres) + + - cd: postmap_changes1 + ot: err('ReqlNonExistenceError', "No attribute `dummy` in object:") + + - cd: postmap_changes2 + ot: err('ReqlNonExistenceError', "Index out of bounds:" + " 1") + + - cd: postmap_changes3 + ot: err('ReqlUserError', "dummy") + + - cd: prefilter_changes1 + ot: bag(erroredres) + + - cd: prefilter_changes2 + ot: bag(erroredres) + + - cd: postfilter_changes1 + ot: bag(erroredres) + + - cd: postfilter_changes2 + ot: bag(erroredres) + + - ot: arrlen(postfilter_changes3) + ot: 4 diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml new file mode 100644 index 00000000..7755ff96 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml @@ -0,0 +1,27 @@ +desc: Geo indexed changefeed operations +table_variable_name: tbl +tests: + - rb: tbl.index_create('L', {geo: true}) + ot: partial({'created': 1}) + + - rb: tbl.index_wait().count + ot: 1 + + - def: obj11 = {id: "11", L: r.point(1,1)} + - def: obj12 = {id: "12", L: r.point(1,2)} + - def: obj21 = {id: "21", L: r.point(2,1)} + - def: obj22 = {id: "22", L: r.point(2,2)} + + # A distance of 130,000 meters from 1,1 is enough to cover 1,2 and 2,1 (~110km + # distance) but not 2,2 (~150km distance.) + # + # This is useful because the S2LatLngRect bounding box passed to the shards contains + # 2,2 yet it should not be returned in the changefeed results. + - rb: feed = tbl.get_intersecting(r.circle(r.point(1,1), 130000), {index: "L"}).get_field("id").changes(include_initial: true) + + - rb: tbl.insert([obj11, obj12, obj21, obj22]) + ot: partial({'errors': 0, 'inserted': 4}) + + - rb: fetch(feed, 3) + ot: bag([{"new_val" => "11", "old_val" => nil}, {"new_val" => "12", "old_val" => nil}, {"new_val" => "21", "old_val" => nil}]) + diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml new file mode 100644 index 00000000..f7110ab1 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml @@ -0,0 +1,38 @@ +desc: Test duplicate indexes with squashing +table_variable_name: tbl +tests: + - cd: tbl.index_create('a') + ot: partial({'created':1}) + - cd: tbl.index_wait('a') + + - py: feed = tbl.order_by(index='a').limit(10).changes(squash=2) + rb: feed = tbl.orderby(index:'a').limit(10).changes(squash:2).limit(9) + js: feed = tbl.orderBy({index:'a'}).limit(10).changes({squash:2}).limit(9) + runopts: + # limit the number of pre-fetched rows + max_batch_rows: 1 + + - py: tbl.insert(r.range(0, 12).map({'id':r.row, 'a':5})) + rb: tbl.insert(r.range(0, 12).map{|row| {'id':row, 'a':5}}) + js: tbl.insert(r.range(0, 12).map(function(row){ return {'id':row, 'a':5}; })) + ot: partial({'inserted':12, 'errors':0}) + + - py: tbl.get_all(1, 8, 9, index='id').delete() + rb: tbl.get_all(1, 8, 9, index:'id').delete() + js: tbl.get_all(1, 8, 9, {index:'id'}).delete() + ot: partial({'deleted':3, 'errors':0}) + + # should be replaced with a noreplyWait + - cd: wait(2) + + - cd: fetch(feed) + ot: bag([ + {"new_val":{"a":5, "id":0}, "old_val":nil}, + {"new_val":{"a":5, "id":2}, "old_val":nil}, + {"new_val":{"a":5, "id":3}, "old_val":nil}, + {"new_val":{"a":5, "id":4}, "old_val":nil}, + {"new_val":{"a":5, "id":5}, "old_val":nil}, + {"new_val":{"a":5, "id":6}, "old_val":nil}, + {"new_val":{"a":5, "id":7}, "old_val":nil}, + {"new_val":{"a":5, "id":10}, "old_val":nil}, + {"new_val":{"a":5, "id":11}, "old_val":nil}]) diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml new file mode 100644 index 00000000..68dadb08 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml @@ -0,0 +1,58 @@ +desc: Test `include_states` +table_variable_name: tbl +tests: + - py: tbl.changes(squash=true, include_states=true).limit(1) + rb: tbl.changes(squash:true, include_states:true).limit(1) + js: tbl.changes({squash:true, includeStates:true}).limit(1) + ot: [{'state':'ready'}] + + - py: tbl.get(0).changes(squash=true, include_states=true, include_initial=true).limit(3) + rb: tbl.get(0).changes(squash:true, include_states:true, include_initial:true).limit(3) + js: tbl.get(0).changes({squash:true, includeStates:true, includeInitial:true}).limit(3) + ot: [{'state':'initializing'}, {'new_val':null}, {'state':'ready'}] + + - py: tbl.order_by(index='id').limit(10).changes(squash=true, include_states=true, include_initial=true).limit(2) + rb: tbl.order_by(index:'id').limit(10).changes(squash:true, include_states:true, include_initial:true).limit(2) + js: tbl.orderBy({index:'id'}).limit(10).changes({squash:true, includeStates:true, includeInitial:true}).limit(2) + ot: [{'state':'initializing'}, {'state':'ready'}] + + - cd: tbl.insert({'id':1}) + + - py: tbl.order_by(index='id').limit(10).changes(squash=true, include_states=true, include_initial=true).limit(3) + rb: tbl.order_by(index:'id').limit(10).changes(squash:true, include_states:true, include_initial:true).limit(3) + js: tbl.orderBy({index:'id'}).limit(10).changes({squash:true, includeStates:true, includeInitial:true}).limit(3) + ot: [{'state':'initializing'}, {'new_val':{'id':1}}, {'state':'ready'}] + + - py: tblchanges = tbl.changes(squash=true, include_states=true) + rb: tblchanges = tbl.changes(squash:true, include_states:true) + js: tblchanges = tbl.changes({squash:true, includeStates:true}) + + - cd: tbl.insert({'id':2}) + + - cd: fetch(tblchanges, 2) + ot: [{'state':'ready'},{'new_val':{'id':2},'old_val':null}] + + - py: getchanges = tbl.get(2).changes(include_states=true, include_initial=true) + rb: getchanges = tbl.get(2).changes(include_states:true, include_initial:true) + js: getchanges = tbl.get(2).changes({includeStates:true, includeInitial:true}) + + - cd: tbl.get(2).update({'a':1}) + + - cd: fetch(getchanges, 4) + ot: [{'state':'initializing'}, {'new_val':{'id':2}}, {'state':'ready'}, {'old_val':{'id':2},'new_val':{'id':2,'a':1}}] + + - py: limitchanges = tbl.order_by(index='id').limit(10).changes(include_states=true, include_initial=true) + rb: limitchanges = tbl.order_by(index:'id').limit(10).changes(include_states:true, include_initial:true) + js: limitchanges = tbl.orderBy({index:'id'}).limit(10).changes({includeStates:true, includeInitial:true}) + + - py: limitchangesdesc = tbl.order_by(index=r.desc('id')).limit(10).changes(include_states=true, include_initial=true) + rb: limitchangesdesc = tbl.order_by(index:r.desc('id')).limit(10).changes(include_states:true, include_initial:true) + js: limitchangesdesc = tbl.orderBy({index:r.desc('id')}).limit(10).changes({includeStates:true, includeInitial:true}) + + - cd: tbl.insert({'id':3}) + + - cd: fetch(limitchanges, 5) + ot: [{'state':'initializing'}, {'new_val':{'id':1}}, {'new_val':{'a':1, 'id':2}}, {'state':'ready'}, {'old_val':null, 'new_val':{'id':3}}] + + - cd: fetch(limitchangesdesc, 5) + ot: [{'state':'initializing'}, {'new_val':{'a':1, 'id':2}}, {'new_val':{'id':1}}, {'state':'ready'}, {'old_val':null, 'new_val':{'id':3}}] diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/point.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/point.yaml new file mode 100644 index 00000000..a444a73d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/point.yaml @@ -0,0 +1,147 @@ +desc: Test point changebasics +table_variable_name: tbl +tests: + + # -- basic + + # start a feed + + - cd: basic = tbl.get(1).changes({include_initial:true}) + py: basic = tbl.get(1).changes(include_initial=True) + + # - inital return + + - cd: fetch(basic, 1) + ot: [{'new_val':null}] + + # - inserts + + - cd: tbl.insert({'id':1}) + ot: partial({'errors':0, 'inserted':1}) + + - cd: fetch(basic, 1) + ot: [{'old_val':null, 'new_val':{'id':1}}] + + # - updates + + - cd: tbl.get(1).update({'update':1}) + ot: partial({'errors':0, 'replaced':1}) + + - cd: fetch(basic, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1,'update':1}}] + + # - deletions + + - cd: tbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + + - cd: fetch(basic, 1) + ot: [{'old_val':{'id':1,'update':1}, 'new_val':null}] + + # - closing + + - cd: basic.close() + rb: def pass; end + # the ruby test driver currently has to mangle cursors, so we can't close them properly + + # -- filter + + - py: filter = tbl.get(1).changes(squash=false,include_initial=True).filter(r.row['new_val']['update'].gt(2))['new_val']['update'] + rb: filter = tbl.get(1).changes(squash:false,include_initial:true).filter{|row| row['new_val']['update'].gt(2)}['new_val']['update'] + js: filter = tbl.get(1).changes({squash:false,include_initial:true}).filter(r.row('new_val')('update').gt(2))('new_val')('update') + + - cd: tbl.insert({'id':1, 'update':1}) + - cd: tbl.get(1).update({'update':4}) + - cd: tbl.get(1).update({'update':1}) + - cd: tbl.get(1).update({'update':7}) + + - cd: fetch(filter, 2) + ot: [4,7] + + # -- pluck on values + + - py: pluck = tbl.get(3).changes(squash=false,include_initial=True).pluck({'new_val':['red', 'blue']})['new_val'] + rb: pluck = tbl.get(3).changes(squash:false,include_initial:true).pluck({'new_val':['red', 'blue']})['new_val'] + js: pluck = tbl.get(3).changes({squash:false,include_initial:true}).pluck({'new_val':['red', 'blue']})('new_val') + + - cd: tbl.insert({'id':3, 'red':1, 'green':1}) + ot: partial({'errors':0, 'inserted':1}) + - cd: tbl.get(3).update({'blue':2, 'green':3}) + ot: partial({'errors':0, 'replaced':1}) + - cd: tbl.get(3).update({'green':4}) + ot: partial({'errors':0, 'replaced':1}) + - cd: tbl.get(3).update({'blue':4}) + ot: partial({'errors':0, 'replaced':1}) + + - cd: fetch(pluck, 4) + ot: [{'red': 1}, {'blue': 2, 'red': 1}, {'blue': 2, 'red': 1}, {'blue': 4, 'red': 1}] + + # -- virtual tables + + # - rethinkdb._debug_scratch + + - def: dtbl = r.db('rethinkdb').table('_debug_scratch') + + - cd: debug = dtbl.get(1).changes({include_initial:true}) + py: debug = dtbl.get(1).changes(include_initial=True) + + - cd: fetch(debug, 1) + ot: [{'new_val':null}] + + - cd: dtbl.insert({'id':1}) + ot: partial({'errors':0, 'inserted':1}) + - cd: fetch(debug, 1) + ot: [{'old_val':null, 'new_val':{'id':1}}] + + - cd: dtbl.get(1).update({'update':1}) + ot: partial({'errors':0, 'replaced':1}) + - cd: fetch(debug, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1,'update':1}}] + + - cd: dtbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + - cd: fetch(debug, 1) + ot: [{'old_val':{'id':1,'update':1}, 'new_val':null}] + + - cd: dtbl.insert({'id':5, 'red':1, 'green':1}) + ot: {'skipped':0, 'deleted':0, 'unchanged':0, 'errors':0, 'replaced':0, 'inserted':1} + - py: dtblPluck = dtbl.get(5).changes(include_initial=True).pluck({'new_val':['red', 'blue']})['new_val'] + rb: dtblPluck = dtbl.get(5).changes(include_initial:true).pluck({'new_val':['red', 'blue']})['new_val'] + js: dtblPluck = dtbl.get(5).changes({include_initial:true}).pluck({'new_val':['red', 'blue']})('new_val') + + # disabled because inital value is not being reported correctly, so goes missing. see #3723 + - cd: fetch(dtblPluck, 1) + ot: [{'red':1}] + + - cd: dtbl.get(5).update({'blue':2, 'green':3}) + ot: partial({'errors':0, 'replaced':1}) + + - cd: fetch(dtblPluck, 1) + ot: [{'blue':2, 'red':1}] + + # - rethinkdb.table_status bad optargs + + # disabled, re-enable once #3725 is done + # - py: r.db('rethinkdb').table('table_status').changes(squash=False) + # rb: r.db('rethinkdb').table('table_status').changes(squash:False) + # js: r.db('rethinkdb').table('table_status').changes({squash:False}) + # ot: err('ReqlRuntimeError', 'replace with error message decided in \#3725') + + # - rethinkdb.table_status + + - cd: tableId = tbl.info()['id'] + js: tableId = tbl.info()('id') + + - cd: rtblPluck = r.db('rethinkdb').table('table_status').get(tableId).changes({include_initial:true}) + py: rtblPluck = r.db('rethinkdb').table('table_status').get(tableId).changes(include_initial=True) + - cd: fetch(rtblPluck, 1) + ot: partial([{'new_val':partial({'db':'test'})}]) + + - py: tbl.reconfigure(shards=3, replicas=1) + rb: tbl.reconfigure(shards:3, replicas:1) + js: tbl.reconfigure({shards:3, replicas:1}) + - py: fetch(rtblPluck, 1, 2) + js: fetch(rtblPluck, 1, 2) + rb: fetch(rtblPluck, 1) + ot: partial([{'old_val':partial({'db':'test'}), 'new_val':partial({'db':'test'})}]) + diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml new file mode 100644 index 00000000..85ef969d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml @@ -0,0 +1,50 @@ +desc: Test basic changefeed operations +table_variable_name: tbl +tests: + + # Fill in some data + - rb: tbl.index_create('a') + ot: partial({'created':1}) + + - rb: tbl.index_wait().count + ot: 1 + + - rb: tbl.insert([{id:1, a:8}, {id:2, a:7}]) + ot: partial({'errors':0, 'inserted':2}) + + - rb: idmin = tbl.min(index:'id').changes(squash:false, include_initial:true).limit(2) + - rb: idmax = tbl.max(index:'id').changes(squash:false, include_initial:true).limit(2) + - rb: amin = tbl.min(index:'a').changes(squash:false, include_initial:true).limit(2) + - rb: amax = tbl.max(index:'a').changes(squash:false, include_initial:true).limit(2) + + - rb: idmin2 = tbl.min(index:'id').changes(squash:true, include_initial:true).limit(2) + - rb: idmax2 = tbl.max(index:'id').changes(squash:true, include_initial:true).limit(2) + - rb: amin2 = tbl.min(index:'a').changes(squash:true, include_initial:true).limit(2) + - rb: amax2 = tbl.max(index:'a').changes(squash:true, include_initial:true).limit(2) + + - rb: tbl.insert([{id:0, a:9}, {id:3, a:6}]) + ot: partial({'errors':0, 'inserted':2}) + + - rb: idmin.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) + + - rb: idmax.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amin.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amax.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) + + - rb: idmin2.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) + + - rb: idmax2.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amin2.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amax2.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml new file mode 100644 index 00000000..fa4ad7f0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml @@ -0,0 +1,62 @@ +desc: Test changefeed squashing +table_variable_name: tbl +tests: + + # Check type + + - py: tbl.changes(squash=true).type_of() + rb: tbl.changes(squash:true).type_of() + js: tbl.changes({squash:true}).typeOf() + ot: ("STREAM") + + # comparison changes + + - cd: normal_changes = tbl.changes().limit(2) + + - py: false_squash_changes = tbl.changes(squash=False).limit(2) + js: false_squash_changes = tbl.changes({squash:false}).limit(2) + rb: false_squash_changes = tbl.changes(squash:false).limit(2) + + - py: long_squash_changes = tbl.changes(squash=0.5).limit(1) + js: long_squash_changes = tbl.changes({squash:0.5}).limit(1) + rb: long_squash_changes = tbl.changes(squash:0.5).limit(1) + + - py: squash_changes = tbl.changes(squash=true).limit(1) + js: squash_changes = tbl.changes({squash:true}).limit(1) + rb: squash_changes = tbl.changes(squash:true).limit(1) + + - cd: tbl.insert({'id':100})['inserted'] + js: tbl.insert({'id':100})('inserted') + ot: 1 + + - cd: tbl.get(100).update({'a':1})['replaced'] + js: tbl.get(100).update({'a':1})('replaced') + ot: 1 + + - cd: normal_changes + ot: ([{'new_val':{'id':100}, 'old_val':null}, + {'new_val':{'a':1, 'id':100}, 'old_val':{'id':100}}]) + + - cd: false_squash_changes + ot: ([{'new_val':{'id':100}, 'old_val':null}, + {'new_val':{'a':1, 'id':100}, 'old_val':{'id':100}}]) + + - cd: long_squash_changes + ot: ([{'new_val':{'a':1, 'id':100}, 'old_val':null}]) + + - cd: squash_changes + ot: + js: ([{'new_val':{'a':1, 'id':100}, 'old_val':null}]) + cd: ([{'new_val':{'id':100}, 'old_val':null}]) + + # Bad squash values + + - py: tbl.changes(squash=null) + rb: tbl.changes(squash:null) + js: tbl.changes({squash:null}) + ot: err('ReqlQueryLogicError', 'Expected BOOL or NUMBER but found NULL.') + + - py: tbl.changes(squash=-10) + rb: tbl.changes(squash:-10) + js: tbl.changes({squash:-10}) + ot: err('ReqlQueryLogicError', 'Expected BOOL or a positive NUMBER but found a negative NUMBER.') diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/table.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/table.yaml new file mode 100644 index 00000000..c089abc9 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/table.yaml @@ -0,0 +1,101 @@ +desc: Test changefeeds on a table +table_variable_name: tbl +tests: + + # ==== regular tables + + # - start feeds + + - cd: all = tbl.changes() + + # - note: no initial values from table changefeeds + + # - inserts + + - cd: tbl.insert([{'id':1}, {'id':2}]) + ot: partial({'errors':0, 'inserted':2}) + - cd: fetch(all, 2) + ot: bag([{'old_val':null, 'new_val':{'id':1}}, {'old_val':null, 'new_val':{'id':2}}]) + + # - updates + + - cd: tbl.get(1).update({'version':1}) + ot: partial({'errors':0, 'replaced':1}) + - cd: fetch(all, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1, 'version':1}}] + + # - deletions + + - cd: tbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + - cd: fetch(all, 1) + ot: [{'old_val':{'id':1, 'version':1}, 'new_val':null}] + + # - pluck on values + + - cd: pluck = tbl.changes().pluck({'new_val':['version']}) + - cd: tbl.insert([{'id':5, 'version':5}]) + ot: partial({'errors':0, 'inserted':1}) + - cd: fetch(pluck, 1) + ot: [{'new_val':{'version':5}}] + + # - order by + + - cd: tbl.changes().order_by('id') + ot: err('ReqlQueryLogicError', "Cannot call a terminal (`reduce`, `count`, etc.) on an infinite stream (such as a changefeed).") +# +# ToDo: enable this when #4067 is done +# +# - js: orderedLimit = tbl.changes().limit(5).order_by(r.desc('id'))('new_val')('id') +# cd: orderedLimit = tbl.changes().limit(5).order_by(r.desc('id'))['new_val']['id'] +# - js: tbl.range(100, 105).map(function (row) { return {'id':row} }) +# py: tbl.range(100, 105).map({'id':r.row}) +# rb: tbl.range(100, 105).map{|row| {'id':row}} +# - cd: fetch(orderedLimit) +# ot: [104, 103, 102, 101, 100] + + # - changes overflow + + - cd: overflow = tbl.changes() + runopts: + changefeed_queue_size: 100 + # add enough entries to make sure we get the overflow error + - js: tbl.insert(r.range(200).map(function(x) { return({}); })) + py: tbl.insert(r.range(200).map(lambda x: {})) + rb: tbl.insert(r.range(200).map{|x| {}}) + - cd: fetch(overflow, 90) + ot: partial([{'error': regex('Changefeed cache over array size limit, skipped \d+ elements.')}]) + + # ==== virtual tables + + - def: vtbl = r.db('rethinkdb').table('_debug_scratch') + - cd: allVirtual = vtbl.changes() + + # - inserts + + - cd: vtbl.insert([{'id':1}, {'id':2}]) + ot: partial({'errors':0, 'inserted':2}) + - cd: fetch(allVirtual, 2) + ot: bag([{'old_val':null, 'new_val':{'id':1}}, {'old_val':null, 'new_val':{'id':2}}]) + + # - updates + + - cd: vtbl.get(1).update({'version':1}) + ot: partial({'errors':0, 'replaced':1}) + - cd: fetch(allVirtual, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1, 'version':1}}] + + # - deletions + + - cd: vtbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + - cd: fetch(allVirtual, 1) + ot: [{'old_val':{'id':1, 'version':1}, 'new_val':null}] + + # - pluck on values + + - cd: vpluck = vtbl.changes().pluck({'new_val':['version']}) + - cd: vtbl.insert([{'id':5, 'version':5}]) + ot: partial({'errors':0, 'inserted':1}) + - cd: fetch(vpluck, 1) + ot: [{'new_val':{'version':5}}] diff --git a/ext/librethinkdbxx/test/upstream/control.yaml b/ext/librethinkdbxx/test/upstream/control.yaml new file mode 100644 index 00000000..248d070a --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/control.yaml @@ -0,0 +1,297 @@ +desc: Tests RQL control flow structures +table_variable_name: tbl, tbl2 +tests: + + ## FunCall + + - py: r.expr(1).do(lambda v: v * 2) + js: r.expr(1).do(function(v) { return v.mul(2); }) + rb: r.expr(1).do{|v| v * 2 } + ot: 2 + + - py: r.expr([0, 1, 2]).do(lambda v: v.append(3)) + js: r([0, 1, 2]).do(function(v) { return v.append(3); }) + rb: r([0, 1, 2]).do{ |v| v.append(3) } + ot: [0, 1, 2, 3] + + - py: r.do(1, 2, lambda x, y: x + y) + js: r.do(1, 2, function(x, y) { return x.add(y); }) + rb: r.do(1, 2) {|x, y| x + y} + ot: 3 + + - py: r.do(lambda: 1) + js: r.do(function() { return 1; }) + rb: r.do{1} + ot: 1 + + # do error cases + - py: r.do(1, 2, lambda x: x) + js: r.do(1, 2, function(x) { return x; }) + rb: r.do(1, 2) {|x| x} + ot: err("ReqlQueryLogicError", 'Expected function with 2 arguments but found function with 1 argument.', [1]) + + - py: r.do(1, 2, 3, lambda x, y: x + y) + js: r.do(1, 2, 3, function(x, y) { return x.add(y); }) + rb: r.do(1, 2, 3) {|x, y| x + y} + ot: err("ReqlQueryLogicError", 'Expected function with 3 arguments but found function with 2 arguments.', [1]) + + - cd: r.do(1) + ot: 1 + + - js: r.do(1, function(x) {}) + ot: err("ReqlDriverCompileError", 'Anonymous function returned `undefined`. Did you forget a `return`?', [1]) + + - js: r.do(1, function(x) { return undefined; }) + ot: err("ReqlDriverCompileError", 'Anonymous function returned `undefined`. Did you forget a `return`?', [1]) + + - cd: r.do() + ot: + cd: err("ReqlCompileError", 'Expected 1 or more arguments but found 0.', [1]) + + # FunCall errors + + - py: r.expr('abc').do(lambda v: v.append(3)) + js: r('abc').do(function(v) { return v.append(3); }) + rb: r('abc').do{ |v| v.append(3) } + ot: err("ReqlQueryLogicError", "Expected type ARRAY but found STRING.", [1, 0]) + + - py: r.expr('abc').do(lambda v: v + 3) + js: r('abc').do(function(v) { return v.add(3); }) + rb: r('abc').do{ |v| v + 3 } + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [1, 1]) + + - py: r.expr('abc').do(lambda v: v + 'def') + 3 + js: r('abc').do(function(v) { return v.add('def'); }).add(3) + rb: r('abc').do{ |v| v + 'def' } + 3 + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [1]) + + - py: r.expr(0).do(lambda a,b: a + b) + js: r(0).do(function(a,b) { return a.add(b); }) + rb: r(0).do{ |a, b| a + b } + ot: err("ReqlQueryLogicError", 'Expected function with 1 argument but found function with 2 arguments.', [1]) + + - py: r.do(1, 2, lambda a: a) + js: r.do(1,2, function(a) { return a; }) + rb: r.do(1, 2) { |a| a } + ot: err("ReqlQueryLogicError", 'Expected function with 2 arguments but found function with 1 argument.', [1]) + + - cd: r.expr(5).do(r.row) + rb: r(5).do{ |row| row } + ot: 5 + + ## Branch + + - cd: r.branch(True, 1, 2) + ot: 1 + - cd: r.branch(False, 1, 2) + ot: 2 + - cd: r.branch(1, 'c', False) + ot: ("c") + - cd: r.branch(null, {}, []) + ot: ([]) + + - cd: r.branch(r.db('test'), 1, 2) + ot: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + - cd: r.branch(tbl, 1, 2) + ot: err("ReqlQueryLogicError", "Expected type DATUM but found TABLE:", []) + - cd: r.branch(r.error("a"), 1, 2) + ot: err("ReqlUserError", "a", []) + + - cd: r.branch([], 1, 2) + ot: 1 + - cd: r.branch({}, 1, 2) + ot: 1 + - cd: r.branch("a", 1, 2) + ot: 1 + - cd: r.branch(1.2, 1, 2) + ot: 1 + + - cd: r.branch(True, 1, True, 2, 3) + ot: 1 + - cd: r.branch(True, 1, False, 2, 3) + ot: 1 + - cd: r.branch(False, 1, True, 2, 3) + ot: 2 + - cd: r.branch(False, 1, False, 2, 3) + ot: 3 + + - cd: r.branch(True, 1, True, 2) + ot: err("ReqlQueryLogicError", "Cannot call `branch` term with an even number of arguments.") + + # r.error() + - cd: r.error('Hello World') + ot: err("ReqlUserError", "Hello World", [0]) + + - cd: r.error(5) + # we might want to allow this eventually + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [0]) + + # r.filter + - cd: r.expr([1, 2, 3]).filter() + ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 1.", [0]) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", [0]) + - cd: r.expr([1, 2, 3]).filter(1, 2) + ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 3.", [0]) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", [0]) + + # r.js() + - cd: r.js('1 + 1') + ot: 2 + + - cd: r.js('1 + 1; 2 + 2') + ot: 4 + + - cd: r.do(1, 2, r.js('(function(a, b) { return a + b; })')) + ot: 3 + + - cd: r.expr(1).do(r.js('(function(x) { return x + 1; })')) + ot: 2 + + - cd: r.expr('foo').do(r.js('(function(x) { return x + "bar"; })')) + ot: 'foobar' + + # js timeout optarg shouldn't be triggered + - cd: r.js('1 + 2', {timeout:1.2}) + py: r.js('1 + 2', timeout=1.2) + ot: 3 + + # js error cases + - cd: r.js('(function() { return 1; })') + ot: err("ReqlQueryLogicError", "Query result must be of type DATUM, GROUPED_DATA, or STREAM (got FUNCTION).", [0]) + + - cd: r.js('function() { return 1; }') + ot: err("ReqlQueryLogicError", "SyntaxError: Unexpected token (", [0]) + + # Play with the number of arguments in the JS function + - cd: r.do(1, 2, r.js('(function(a) { return a; })')) + ot: 1 + + - cd: r.do(1, 2, r.js('(function(a, b, c) { return a; })')) + ot: 1 + + - cd: r.do(1, 2, r.js('(function(a, b, c) { return c; })')) + ot: err("ReqlQueryLogicError", "Cannot convert javascript `undefined` to ql::datum_t.", [0]) + + - cd: r.expr([1, 2, 3]).filter(r.js('(function(a) { return a >= 2; })')) + ot: ([2, 3]) + + - cd: r.expr([1, 2, 3]).map(r.js('(function(a) { return a + 1; })')) + ot: ([2, 3, 4]) + + - cd: r.expr([1, 2, 3]).map(r.js('1')) + ot: err("ReqlQueryLogicError", "Expected type FUNCTION but found DATUM:", [0]) + + - cd: r.expr([1, 2, 3]).filter(r.js('(function(a) {})')) + ot: err("ReqlQueryLogicError", "Cannot convert javascript `undefined` to ql::datum_t.", [0]) + + # What happens if we pass static values to things that expect functions + - cd: r.expr([1, 2, 3]).map(1) + ot: err("ReqlQueryLogicError", "Expected type FUNCTION but found DATUM:", [0]) + + - cd: r.expr([1, 2, 3]).filter('foo') + ot: ([1, 2, 3]) + - cd: r.expr([1, 2, 4]).filter([]) + ot: ([1, 2, 4]) + - cd: r.expr([1, 2, 3]).filter(null) + ot: ([]) + + - cd: r.expr([1, 2, 4]).filter(False) + rb: r([1, 2, 4]).filter(false) + ot: ([]) + + # forEach + - cd: tbl.count() + ot: 0 + + # Insert three elements + - js: r([1, 2, 3]).forEach(function (row) { return tbl.insert({ id:row }) }) + py: r.expr([1, 2, 3]).for_each(lambda row:tbl.insert({ 'id':row })) + rb: r([1, 2, 3]).for_each{ |row| tbl.insert({ :id => row }) } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':3}) + + - cd: tbl.count() + ot: 3 + + # Update each row to add additional attribute + - js: r([1, 2, 3]).forEach(function (row) { return tbl.update({ foo:row }) }) + py: r.expr([1,2,3]).for_each(lambda row:tbl.update({'foo':row})) + rb: r.expr([1,2,3]).for_each{ |row| tbl.update({ :foo => row }) } + ot: ({'deleted':0.0,'replaced':9,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # Insert three more elements (and error on three) + - js: r([1, 2, 3]).forEach(function (row) { return [tbl.insert({ id:row }), tbl.insert({ id:row.mul(10) })] }) + py: r.expr([1,2,3]).for_each(lambda row:[tbl.insert({ 'id':row }), tbl.insert({ 'id':row*10 })]) + rb: r.expr([1,2,3]).for_each{ |row| [tbl.insert({ :id => row}), tbl.insert({ :id => row*10})] } + ot: {'first_error':"Duplicate primary key `id`:\n{\n\t\"foo\":\t3,\n\t\"id\":\t1\n}\n{\n\t\"id\":\t1\n}",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':3,'skipped':0.0,'inserted':3} + + - cd: tbl.count() + ot: 6 + + - cd: tableCount = tbl2.count() + - cd: r.expr([1, 2, 3]).for_each( tbl2.insert({}) ) + ot: ({'deleted':0.0,'replaced':0.0,'generated_keys':arrlen(3,uuid()),'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':3}) + # inserts only a single document per #3700 + - cd: tbl2.count() + ot: tableCount + 1 + + # We have six elements, update them 6*2*3=36 times + - js: r([1, 2, 3]).forEach(function (row) { return [tbl.update({ foo:row }), tbl.update({ bar:row })] }) + py: r.expr([1,2,3]).for_each(lambda row:[tbl.update({'foo':row}), tbl.update({'bar':row})]) + rb: r.expr([1,2,3]).for_each{ |row| [tbl.update({:foo => row}), tbl.update({:bar => row})]} + ot: ({'deleted':0.0,'replaced':36,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # forEach negative cases + - cd: r.expr([1, 2, 3]).for_each( tbl2.insert({ 'id':r.row }) ) + rb: r([1, 2, 3]).for_each{ |row| tbl2.insert({ 'id':row }) } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':3}) + + - cd: r.expr([1, 2, 3]).for_each(1) + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries. Expected type ARRAY but found NUMBER.", [0]) + + - py: r.expr([1, 2, 3]).for_each(lambda x:x) + js: r([1, 2, 3]).forEach(function (x) { return x; }) + rb: r([1, 2, 3]).for_each{ |x| x } + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries. Expected type ARRAY but found NUMBER.", [1, 1]) + + - cd: r.expr([1, 2, 3]).for_each(r.row) + rb: r([1, 2, 3]).for_each{ |row| row } + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries. Expected type ARRAY but found NUMBER.", [1, 1]) + + - js: r([1, 2, 3]).forEach(function (row) { return tbl; }) + py: r.expr([1, 2, 3]).for_each(lambda row:tbl) + rb: r([1, 2, 3]).for_each{ |row| tbl } + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries.", [1, 1]) + + # This is only relevant in JS -- what happens when we return undefined + - js: r([1, 2, 3]).forEach(function (row) {}) + ot: err("ReqlDriverCompileError", 'Anonymous function returned `undefined`. Did you forget a `return`?', [1]) + + # Make sure write queries can't be nested into stream ops + - cd: r.expr(1).do(tbl.insert({'foo':r.row})) + rb: r(1).do{ |row| tbl.insert({ :foo => row }) } + ot: ({'deleted':0.0,'replaced':0.0,'generated_keys':arrlen(1,uuid()),'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':1}) + + - py: r.expr([1, 2])[0].do(tbl.insert({'foo':r.row})) + js: r.expr([1, 2]).nth(0).do(tbl.insert({'foo':r.row})) + rb: r([1, 2])[0].do{ |row| tbl.insert({ :foo => row }) } + ot: ({'deleted':0.0,'replaced':0.0,'generated_keys':arrlen(1,uuid()),'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':1}) + + - cd: r.expr([1, 2]).map(tbl.insert({'foo':r.row})) + rb: r([1, 2]).map{ |row| tbl.insert({ :foo => row }) } + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr([1, 2]).map(r.db('test').table_create('table_create_failure')) + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr([1, 2]).map(tbl.insert({'foo':r.row}).get_field('inserted')) + rb: r.expr([1, 2]).map{|x| tbl.insert({'foo':x}).get_field('inserted')} + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr([1, 2]).map(tbl.insert({'foo':r.row}).get_field('inserted').add(5)) + rb: r.expr([1, 2]).map{|x| tbl.insert({'foo':x}).get_field('inserted').add(5)} + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr(1).do(r.db('test').table_create('table_create_success')) + ot: partial({'tables_created':1}) diff --git a/ext/librethinkdbxx/test/upstream/datum/array.yaml b/ext/librethinkdbxx/test/upstream/datum/array.yaml new file mode 100644 index 00000000..7d16f943 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/array.yaml @@ -0,0 +1,133 @@ +desc: Tests conversion to and from the RQL array type +tests: + - cd: + - r.expr([]) + - r([]) + py: r.expr([]) + ot: [] + + - py: r.expr([1]) + js: r([1]) + rb: r([1]) + ot: [1] + + - py: r.expr([1,2,3,4,5]) + js: r([1,2,3,4,5]) + rb: r.expr([1,2,3,4,5]) + ot: [1,2,3,4,5] + + - cd: r.expr([]).type_of() + ot: 'ARRAY' + + # test coercions + - cd: + - r.expr([1, 2]).coerce_to('string') + - r.expr([1, 2]).coerce_to('STRING') + ot: '[1,2]' + + - cd: r.expr([1, 2]).coerce_to('array') + ot: [1, 2] + + - cd: r.expr([1, 2]).coerce_to('number') + ot: err('ReqlQueryLogicError', 'Cannot coerce ARRAY to NUMBER.', [0]) + + - cd: r.expr([['a', 1], ['b', 2]]).coerce_to('object') + ot: {'a':1,'b':2} + + - cd: r.expr([[]]).coerce_to('object') + ot: err('ReqlQueryLogicError', 'Expected array of size 2, but got size 0.') + + - cd: r.expr([['1',2,3]]).coerce_to('object') + ot: err('ReqlQueryLogicError', 'Expected array of size 2, but got size 3.') + + # Nested expression + - cd: r.expr([r.expr(1)]) + ot: [1] + + - cd: r.expr([1,3,4]).insert_at(1, 2) + ot: [1,2,3,4] + - cd: r.expr([2,3]).insert_at(0, 1) + ot: [1,2,3] + - cd: r.expr([1,2,3]).insert_at(-1, 4) + ot: [1,2,3,4] + - cd: r.expr([1,2,3]).insert_at(3, 4) + ot: [1,2,3,4] + - py: r.expr(3).do(lambda x: r.expr([1,2,3]).insert_at(x, 4)) + - js: r.expr(3).do(function (x) { return r.expr([1,2,3]).insert_at(x, 4); }) + - rb: r.expr(3).do{|x| r.expr([1,2,3]).insert_at(x, 4)} + ot: [1,2,3,4] + - cd: r.expr([1,2,3]).insert_at(4, 5) + ot: err('ReqlNonExistenceError', 'Index `4` out of bounds for array of size: `3`.', [0]) + - cd: r.expr([1,2,3]).insert_at(-5, -1) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -5', [0]) + - cd: r.expr([1,2,3]).insert_at(1.5, 1) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).insert_at(null, 1) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) + + - cd: r.expr([1,4]).splice_at(1, [2,3]) + ot: [1,2,3,4] + - cd: r.expr([3,4]).splice_at(0, [1,2]) + ot: [1,2,3,4] + - cd: r.expr([1,2]).splice_at(2, [3,4]) + ot: [1,2,3,4] + - cd: r.expr([1,2]).splice_at(-1, [3,4]) + ot: [1,2,3,4] + - py: r.expr(2).do(lambda x: r.expr([1,2]).splice_at(x, [3,4])) + - js: r.expr(2).do(function (x) { return r.expr([1,2]).splice_at(x, [3,4]); }) + - rb: r.expr(2).do{|x| r.expr([1,2]).splice_at(x, [3,4])} + ot: [1,2,3,4] + - cd: r.expr([1,2]).splice_at(3, [3,4]) + ot: err('ReqlNonExistenceError', 'Index `3` out of bounds for array of size: `2`.', [0]) + - cd: r.expr([1,2]).splice_at(-4, [3,4]) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -4', [0]) + - cd: r.expr([1,2,3]).splice_at(1.5, [1]) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).splice_at(null, [1]) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) + - cd: r.expr([1,4]).splice_at(1, 2) + ot: err('ReqlQueryLogicError', 'Expected type ARRAY but found NUMBER.', [0]) + + - cd: r.expr([1,2,3,4]).delete_at(0) + ot: [2,3,4] + - py: r.expr(0).do(lambda x: r.expr([1,2,3,4]).delete_at(x)) + - js: r.expr(0).do(function (x) { return r.expr([1,2,3,4]).delete_at(x); }) + - rb: r.expr(0).do{|x| r.expr([1,2,3,4]).delete_at(x)} + ot: [2,3,4] + - cd: r.expr([1,2,3,4]).delete_at(-1) + ot: [1,2,3] + - cd: r.expr([1,2,3,4]).delete_at(1,3) + ot: [1,4] + - cd: r.expr([1,2,3,4]).delete_at(4,4) + ot: [1,2,3,4] + - cd: r.expr([]).delete_at(0,0) + ot: [] + - cd: r.expr([1,2,3,4]).delete_at(1,-1) + ot: [1,4] + - cd: r.expr([1,2,3,4]).delete_at(4) + ot: err('ReqlNonExistenceError', 'Index `4` out of bounds for array of size: `4`.', [0]) + - cd: r.expr([1,2,3,4]).delete_at(-5) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -5', [0]) + - cd: r.expr([1,2,3]).delete_at(1.5) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).delete_at(null) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) + + - cd: r.expr([0,2,3]).change_at(0, 1) + ot: [1,2,3] + - py: r.expr(1).do(lambda x: r.expr([0,2,3]).change_at(0,x)) + - js: r.expr(1).do(function (x) { return r.expr([0,2,3]).change_at(0,x); }) + - rb: r.expr(1).do{|x| r.expr([0,2,3]).change_at(0,x)} + ot: [1,2,3] + - cd: r.expr([1,0,3]).change_at(1, 2) + ot: [1,2,3] + - cd: r.expr([1,2,0]).change_at(2, 3) + ot: [1,2,3] + - cd: r.expr([1,2,3]).change_at(3, 4) + ot: err('ReqlNonExistenceError', 'Index `3` out of bounds for array of size: `3`.', [0]) + - cd: r.expr([1,2,3,4]).change_at(-5, 1) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -5', [0]) + - cd: r.expr([1,2,3]).change_at(1.5, 1) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).change_at(null, 1) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/datum/binary.yaml b/ext/librethinkdbxx/test/upstream/datum/binary.yaml new file mode 100644 index 00000000..6236a457 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/binary.yaml @@ -0,0 +1,363 @@ +desc: Tests of converstion to and from the RQL binary type +tests: + + # Short binary data from 0 to 12 characters + # Not fully implemented for JS as comparing Buffer objects is non-trivial + - def: + rb: s = "".force_encoding('BINARY') + py: s = b'' + js: s = Buffer("", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 0 + + - def: + rb: s = "\x00".force_encoding('BINARY') + py: s = b'\x00' + js: s = Buffer("\x00", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 1 + + - def: + rb: s = "\x00\x42".force_encoding('BINARY') + py: s = b'\x00\x42' + js: s = Buffer("\x00\x42", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 2 + + - def: + rb: s = "\x00\xfe\x7a".force_encoding('BINARY') + py: s = b'\x00\xfe\x7a' + js: s = Buffer("\x00\xfe\x7a", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 3 + + - def: + rb: s = "\xed\xfe\x00\xba".force_encoding('BINARY') + py: s = b'\xed\xfe\x00\xba' + js: s = Buffer("\xed\xfe\x00\xba", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 4 + + - def: + rb: s = "\x50\xf9\x00\x77\xf9".force_encoding('BINARY') + py: s = b'\x50\xf9\x00\x77\xf9' + js: s = Buffer("\x50\xf9\x00\x77\xf9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 5 + + - def: + rb: s = "\x2f\xe3\xb5\x57\x00\x92".force_encoding('BINARY') + py: s = b'\x2f\xe3\xb5\x57\x00\x92' + js: s = Buffer("\x2f\xe3\xb5\x57\x00\x92", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 6 + + - def: + rb: s = "\xa9\x43\x54\xe9\x00\xf8\xfb".force_encoding('BINARY') + py: s = b'\xa9\x43\x54\xe9\x00\xf8\xfb' + js: s = Buffer("\xa9\x43\x54\xe9\x00\xf8\xfb", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 7 + + - def: + rb: s = "\x57\xbb\xe5\x82\x8b\xd3\x00\xf9".force_encoding('BINARY') + py: s = b'\x57\xbb\xe5\x82\x8b\xd3\x00\xf9' + js: s = Buffer("\x57\xbb\xe5\x82\x8b\xd3\x00\xf9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 8 + + - def: + rb: s = "\x44\x1b\x3e\x00\x13\x19\x29\x2a\xbf".force_encoding('BINARY') + py: s = b'\x44\x1b\x3e\x00\x13\x19\x29\x2a\xbf' + js: s = Buffer("\x44\x1b\x3e\x00\x13\x19\x29\x2a\xbf", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 9 + + - def: + rb: s = "\x8a\x1d\x09\x00\x5d\x60\x6b\x2e\x70\xd9".force_encoding('BINARY') + py: s = b'\x8a\x1d\x09\x00\x5d\x60\x6b\x2e\x70\xd9' + js: s = Buffer("\x8a\x1d\x09\x00\x5d\x60\x6b\x2e\x70\xd9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 10 + + - def: + rb: s = "\x00\xaf\x47\x4b\x38\x99\x14\x8d\x8f\x10\x51".force_encoding('BINARY') + py: s = b'\x00\xaf\x47\x4b\x38\x99\x14\x8d\x8f\x10\x51' + js: s = Buffer("\x00\xaf\x47\x4b\x38\x99\x14\x8d\x8f\x10\x51", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 11 + + - def: + cd: s = "\x45\x39\x00\xf7\xc2\x37\xfd\xe0\x38\x82\x40\xa9".force_encoding('BINARY') + py: s = b'\x45\x39\x00\xf7\xc2\x37\xfd\xe0\x38\x82\x40\xa9' + js: s = Buffer("\x45\x39\x00\xf7\xc2\x37\xfd\xe0\x38\x82\x40\xa9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 12 + + # Test comparisons + # Binary objects to use, in order of increasing value + - def: + js: a = Buffer("\x00", 'binary') + rb: a = "\x00".force_encoding('BINARY') + py: a = b'\x00' + - def: + js: b = Buffer("\x00\x01", 'binary') + rb: b = "\x00\x01".force_encoding('BINARY') + py: b = b'\x00\x01' + - def: + js: c = Buffer("\x01", 'binary') + rb: c = "\x01".force_encoding('BINARY') + py: c = b'\x01' + - def: + js: d = Buffer("\x70\x22", 'binary') + rb: d = "\x70\x22".force_encoding('BINARY') + py: d = b'\x70\x22' + - def: + js: e = Buffer("\x80", 'binary') + rb: e = "\x80".force_encoding('BINARY') + py: e = b'\x80' + - def: + js: f = Buffer("\xFE", 'binary') + rb: f = "\xFE".force_encoding('BINARY') + py: f = b'\xFE' + + # a -> a + - cd: r.binary(a).eq(r.binary(a)) + ot: true + - cd: r.binary(a).le(r.binary(a)) + ot: true + - cd: r.binary(a).ge(r.binary(a)) + ot: true + - cd: r.binary(a).ne(r.binary(a)) + ot: false + - cd: r.binary(a).lt(r.binary(a)) + ot: false + - cd: r.binary(a).gt(r.binary(a)) + ot: false + + # a -> b + - cd: r.binary(a).ne(r.binary(b)) + ot: true + - cd: r.binary(a).lt(r.binary(b)) + ot: true + - cd: r.binary(a).le(r.binary(b)) + ot: true + - cd: r.binary(a).ge(r.binary(b)) + ot: false + - cd: r.binary(a).gt(r.binary(b)) + ot: false + - cd: r.binary(a).eq(r.binary(b)) + ot: false + + # b -> c + - cd: r.binary(b).ne(r.binary(c)) + ot: true + - cd: r.binary(b).lt(r.binary(c)) + ot: true + - cd: r.binary(b).le(r.binary(c)) + ot: true + - cd: r.binary(b).ge(r.binary(c)) + ot: false + - cd: r.binary(b).gt(r.binary(c)) + ot: false + - cd: r.binary(b).eq(r.binary(c)) + ot: false + + # c -> d + - cd: r.binary(c).ne(r.binary(d)) + ot: true + - cd: r.binary(c).lt(r.binary(d)) + ot: true + - cd: r.binary(c).le(r.binary(d)) + ot: true + - cd: r.binary(c).ge(r.binary(d)) + ot: false + - cd: r.binary(c).gt(r.binary(d)) + ot: false + - cd: r.binary(c).eq(r.binary(d)) + ot: false + + # d -> e + - cd: r.binary(d).ne(r.binary(e)) + ot: true + - cd: r.binary(d).lt(r.binary(e)) + ot: true + - cd: r.binary(d).le(r.binary(e)) + ot: true + - cd: r.binary(d).ge(r.binary(e)) + ot: false + - cd: r.binary(d).gt(r.binary(e)) + ot: false + - cd: r.binary(d).eq(r.binary(e)) + ot: false + + # e -> f + - cd: r.binary(e).ne(r.binary(f)) + ot: true + - cd: r.binary(e).lt(r.binary(f)) + ot: true + - cd: r.binary(e).le(r.binary(f)) + ot: true + - cd: r.binary(e).ge(r.binary(f)) + ot: false + - cd: r.binary(e).gt(r.binary(f)) + ot: false + - cd: r.binary(e).eq(r.binary(f)) + ot: false + + # f -> f + - cd: r.binary(f).eq(r.binary(f)) + ot: true + - cd: r.binary(f).le(r.binary(f)) + ot: true + - cd: r.binary(f).ge(r.binary(f)) + ot: true + - cd: r.binary(f).ne(r.binary(f)) + ot: false + - cd: r.binary(f).lt(r.binary(f)) + ot: false + - cd: r.binary(f).gt(r.binary(f)) + ot: false + + # Test encodings + - py: + cd: r.binary(u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム'.encode('utf-8')) + ot: u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム'.encode('utf-8') + py3: + cd: r.binary(str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム').encode('utf-8')) + ot: str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム').encode('utf-8') + - py: + cd: r.binary(u'ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ'.encode('utf-16')) + ot: u'ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ'.encode('utf-16') + py3: + cd: r.binary(str('ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ').encode('utf-16')) + ot: str('ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ').encode('utf-16') + - py: + cd: r.binary(u'lorem ipsum'.encode('ascii')) + ot: u'lorem ipsum'.encode('ascii') + py3: + cd: r.binary(str('lorem ipsum').encode('ascii')) + ot: str('lorem ipsum').encode('ascii') + + # Test coercions + - py: r.binary(b'foo').coerce_to('string') + ot: 'foo' + - py: + cd: r.binary(u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム'.encode('utf-8')).coerce_to('string') + ot: u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム' + py3: + cd: r.binary(str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム').encode('utf-8')).coerce_to('string') + ot: str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム') + - py: + cd: r.binary(u'lorem ipsum'.encode('ascii')).coerce_to('string') + ot: u'lorem ipsum' + py3: + cd: r.binary(str('lorem ipsum').encode('ascii')).coerce_to('string') + ot: str('lorem ipsum') + + - py: r.expr('foo').coerce_to('binary') + ot: b'foo' + + - cd: r.binary(a).coerce_to('bool') + ot: True + + - py: r.binary(b'foo').coerce_to('binary') + ot: b'foo' + + # Test slice + - py: r.binary(b'abcdefg').slice(-3,-1) + ot: b'ef' + - py: r.binary(b'abcdefg').slice(0, 2) + ot: b'ab' + - py: r.binary(b'abcdefg').slice(3, -1) + ot: b'def' + - py: r.binary(b'abcdefg').slice(-5, 5) + ot: b'cde' + - py: r.binary(b'abcdefg').slice(-8, 2) + ot: b'ab' + - py: r.binary(b'abcdefg').slice(5, 7) + ot: b'fg' + + # Left side out-of-bound should clamp to index 0 + - py: r.binary(b'abcdefg').slice(-9, 2) + ot: b'ab' + + # Right side out-of-bound should return the valid subset of the range + - py: r.binary(b'abcdefg').slice(5, 9) + ot: b'fg' + + # Test binary_format optarg + - cd: r.binary(b) + runopts: + binary_format: "native" + ot: b + - cd: r.binary(b) + runopts: + binary_format: "raw" + ot: {'$reql_type$':'BINARY','data':'AAE='} + + # Test r.binary of nested terms + - cd: r.binary(r.expr("data")) + ot: + js: Buffer("data", "binary") + rb: "data" + py: b"data" + + - cd: r.binary(r.expr({})) + ot: err('ReqlQueryLogicError', 'Expected type STRING but found OBJECT.', []) + + - cd: r.binary(r.expr([])) + ot: err('ReqlQueryLogicError', 'Expected type STRING but found ARRAY.', []) + + # Test errors + + # Missing 'data' field + - py: r.expr({'$reql_type$':'BINARY'}) + rb: r.expr({'$reql_type$':'BINARY'}) + ot: err('ReqlQueryLogicError','Invalid binary pseudotype:'+' lacking `data` key.',[]) + + # Invalid base64 format + - py: r.expr({'$reql_type$':'BINARY','data':'ABCDEFGH==AA'}) + ot: err('ReqlQueryLogicError','Invalid base64 format, data found after padding character \'=\'.',[]) + - py: r.expr({'$reql_type$':'BINARY','data':'ABCDEF==$'}) + ot: err('ReqlQueryLogicError','Invalid base64 format, data found after padding character \'=\'.',[]) + - py: r.expr({'$reql_type$':'BINARY','data':'A^CDEFGH'}) + ot: err('ReqlQueryLogicError','Invalid base64 character found:'+' \'^\'.',[]) + - py: r.expr({'$reql_type$':'BINARY','data':'ABCDE'}) + ot: err('ReqlQueryLogicError','Invalid base64 length:'+' 1 character remaining, cannot decode a full byte.',[]) + + # Invalid coercions + - cd: r.binary(a).coerce_to('array') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to ARRAY.',[]) + - cd: r.binary(a).coerce_to('object') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to OBJECT.',[]) + - cd: r.binary(a).coerce_to('number') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to NUMBER.',[]) + - cd: r.binary(a).coerce_to('nu'+'ll') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to NULL.',[]) diff --git a/ext/librethinkdbxx/test/upstream/datum/bool.yaml b/ext/librethinkdbxx/test/upstream/datum/bool.yaml new file mode 100644 index 00000000..0e0aa11f --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/bool.yaml @@ -0,0 +1,47 @@ +desc: Tests of conversion to and from the RQL bool type +tests: + - py: r.expr(True) + js: + - r.expr(true) + - r(true) + rb: r true + ot: true + + - py: r.expr(False) + js: + - r.expr(false) + - r(false) + rb: r false + ot: false + + - cd: r.expr(False).type_of() + ot: 'BOOL' + + # test coercions + - cd: r.expr(True).coerce_to('string') + ot: 'true' + + - cd: r.expr(True).coerce_to('bool') + ot: True + + - cd: r.expr(False).coerce_to('bool') + ot: False + + - cd: r.expr(null).coerce_to('bool') + ot: False + + - cd: r.expr(0).coerce_to('bool') + ot: True + + - cd: r.expr('false').coerce_to('bool') + ot: True + + - cd: r.expr('foo').coerce_to('bool') + ot: True + + - cd: r.expr([]).coerce_to('bool') + ot: True + + - cd: r.expr({}).coerce_to('bool') + ot: True + diff --git a/ext/librethinkdbxx/test/upstream/datum/null.yaml b/ext/librethinkdbxx/test/upstream/datum/null.yaml new file mode 100644 index 00000000..f8e95464 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/null.yaml @@ -0,0 +1,18 @@ +desc: Tests of conversion to and from the RQL null type +tests: + - cd: + - r(null) + - r.expr(null) + py: r.expr(null) + ot: (null) + + - cd: r.expr(null).type_of() + rb: r(null).type_of() + ot: 'NULL' + + # test coercions + - cd: r.expr(null).coerce_to('string') + ot: 'null' + + - cd: r.expr(null).coerce_to('null') + ot: null diff --git a/ext/librethinkdbxx/test/upstream/datum/number.yaml b/ext/librethinkdbxx/test/upstream/datum/number.yaml new file mode 100644 index 00000000..09c48d15 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/number.yaml @@ -0,0 +1,125 @@ +# desc will be included in a comment to help identify test groups +desc: Tests of conversion to and from the RQL number type +tests: + + # Simple integers + - cd: r.expr(1) + js: + - r(1) + - r.expr(1) + rb: + - r 1 + - r(1) + - r.expr(1) + ot: 1 + - cd: r.expr(-1) + js: + - r(-1) + - r.expr(-1) + rb: + - r -1 + - r(-1) + - r.expr(-1) + ot: -1 + - cd: r.expr(0) + js: + - r(0) + - r.expr(0) + rb: + - r 0 + - r(0) + - r.expr(0) + ot: 0 + + # Floats + - cd: r.expr(1.0) + js: + - r(1.0) + - r.expr(1.0) + rb: + - r 1.0 + - r(1.0) + - r.expr(1.0) + ot: 1.0 + - cd: r.expr(1.5) + js: + - r(1.5) + - r.expr(1.5) + rb: + - r 1.5 + - r(1.5) + - r.expr(1.5) + ot: 1.5 + - cd: r.expr(-0.5) + js: + - r(-0.5) + - r.expr(-0.5) + rb: + - r -0.5 + - r(-0.5) + - r.expr(-0.5) + ot: -0.5 + - cd: r.expr(67498.89278) + js: + - r(67498.89278) + - r.expr(67498.89278) + rb: + - r 67498.89278 + - r(67498.89278) + - r.expr(67498.89278) + ot: 67498.89278 + + # Big numbers + - cd: r.expr(1234567890) + js: + - r(1234567890) + - r.expr(1234567890) + rb: + - r 1234567890 + - r(1234567890) + - r.expr(1234567890) + ot: 1234567890 + + - cd: r.expr(-73850380122423) + js: + - r.expr(-73850380122423) + - r(-73850380122423) + rb: + - r -73850380122423 + - r.expr(-73850380122423) + - r(-73850380122423) + ot: -73850380122423 + + # Test that numbers round-trip correctly + - py: + cd: r.expr(1234567890123456789012345678901234567890) + ot: float(1234567890123456789012345678901234567890) + js: + cd: r.expr(1234567890123456789012345678901234567890) + ot: 1234567890123456789012345678901234567890 + - cd: r.expr(123.4567890123456789012345678901234567890) + ot: 123.4567890123456789012345678901234567890 + + - cd: r.expr(1).type_of() + ot: 'NUMBER' + + # test coercions + - cd: r.expr(1).coerce_to('string') + ot: '1' + + - cd: r.expr(1).coerce_to('number') + ot: 1 + + # The drivers now convert to an int (where relevant) if we think the result + # looks like an int (result % 1.0 == 0.0) + - py: r.expr(1.0) + rb: r 1.0 + ot: int_cmp(1) + + - py: r.expr(45) + rb: r 45 + ot: int_cmp(45) + + - py: r.expr(1.2) + rb: r 1.2 + ot: float_cmp(1.2) diff --git a/ext/librethinkdbxx/test/upstream/datum/object.yaml b/ext/librethinkdbxx/test/upstream/datum/object.yaml new file mode 100644 index 00000000..64b3cbf4 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/object.yaml @@ -0,0 +1,85 @@ +desc: Tests conversion to and from the RQL object type +tests: + - cd: + - r({}) + - r.expr({}) + py: r.expr({}) + ot: {} + - cd: + - r({a:1}) + - r.expr({'a':1}) + py: r.expr({'a':1}) + ot: {'a':1} + - cd: + - r({a:1, b:'two', c:True}) + - r.expr({'a':1, 'b':'two', 'c':True}) + py: r.expr({'a':1, 'b':'two', 'c':True}) + ot: {'a':1, 'b':'two', 'c':True} + + # Nested expressions + - cd: r.expr({'a':r.expr(1)}) + ot: {'a':1} + + - cd: r.expr({'a':{'b':[{'c':2}, 'a', 4]}}) + ot: {'a':{'b':[{'c':2}, 'a', 4]}} + + - cd: r.expr({'a':1}).type_of() + ot: 'OBJECT' + + # test coercions + - cd: r.expr({'a':1}).coerce_to('string') + ot: + cd: '{"a":1}' + + - cd: r.expr({'a':1}).coerce_to('object') + ot: {'a':1} + + - cd: r.expr({'a':1}).coerce_to('array') + ot: [['a',1]] + + # Error cases + - cd: r.expr({12:'a'}) + # JavaScript auto-converts keys for us + js: + ot: err_regex("ReqlCompileError", "Object keys must be strings.*") + + - cd: r.expr({'a':{12:'b'}}) + # JavaScript auto-converts keys for us + js: + ot: err_regex("ReqlCompileError", "Object keys must be strings.*") + + - js: r({'a':undefined}) + ot: err("ReqlCompileError", "Object field 'a' may not be undefined") + + - js: r({'a':{'b':undefined}}) + ot: err("ReqlCompileError", "Object field 'b' may not be undefined") + + - cd: r.expr({}, "foo") + ot: + cd: err("ReqlCompileError", "Second argument to `r.expr` must be a number.") + js: err("ReqlCompileError", "Second argument to `r.expr` must be a number or undefined.") + + - js: r.expr({}, NaN) + ot: err("ReqlCompileError", "Second argument to `r.expr` must be a number or undefined.") + + # r.object + - cd: r.object() + ot: {} + + - cd: r.object('a', 1, 'b', 2) + ot: {'a':1,'b':2} + + - cd: r.object('c'+'d', 3) + ot: {'cd':3} + + - cd: r.object('o','d','d') + ot: err("ReqlQueryLogicError", "OBJECT expects an even number of arguments (but found 3).", []) + + - cd: r.object(1, 1) + ot: err("ReqlQueryLogicError","Expected type STRING but found NUMBER.",[]) + + - cd: r.object('e', 4, 'e', 5) + ot: err("ReqlQueryLogicError","Duplicate key \"e\" in object. (got 4 and 5 as values)",[]) + + - cd: r.object('g', r.db('test')) + ot: err("ReqlQueryLogicError","Expected type DATUM but found DATABASE:",[]) diff --git a/ext/librethinkdbxx/test/upstream/datum/string.yaml b/ext/librethinkdbxx/test/upstream/datum/string.yaml new file mode 100644 index 00000000..c518175d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/string.yaml @@ -0,0 +1,329 @@ +desc: Tests of converstion to and from the RQL string type +tests: + + - def: + cd: japanese_hello = 'こんにちは' + # Python supports unicode strings with the u'' pattern, except 3.0-3.2 + py: japanese_hello = u'こんにちは' + py3.0: japanese_hello = 'こんにちは' + py3.1: japanese_hello = 'こんにちは' + py3.2: japanese_hello = 'こんにちは' + + # Simple strings + - cd: + - r('str') + - r.expr('str') + py: r.expr('str') + ot: "str" + - cd: + - r("str") + - r.expr("str") + py: r.expr("str") + ot: "str" + + # Unicode + + - cd: + py: + cd: r.expr(u'str') + ot: u'str' + py3.0: r.expr('str') + py3.1: r.expr('str') + py3.2: r.expr('str') + ot: 'str' + + - cd: r.expr(japanese_hello) + ot: + cd: 'こんにちは' + py: u'こんにちは' + py3.0: 'こんにちは' + py3.1: 'こんにちは' + py3.2: 'こんにちは' + + - cd: r.expr('foo').type_of() + ot: 'STRING' + + # test coercions + - cd: r.expr('foo').coerce_to('string') + ot: 'foo' + - cd: r.expr('-1.2').coerce_to('NUMBER') + ot: -1.2 + - cd: r.expr('--1.2').coerce_to('NUMBER') + ot: err("ReqlQueryLogicError", "Could not coerce `--1.2` to NUMBER.", []) + - cd: r.expr('-1.2-').coerce_to('NUMBER') + ot: err("ReqlQueryLogicError", "Could not coerce `-1.2-` to NUMBER.", []) + - cd: r.expr('0xa').coerce_to('NUMBER') + ot: 10 + - cd: r.expr('inf').coerce_to('NUMBER') + ot: err("ReqlQueryLogicError", "Non-finite number: inf", []) + + # count is defined as the number of unicode codepoints + - cd: r.expr('hello, world!').count() + ot: 13 + - cd: r.expr(japanese_hello).count() + ot: 5 + + # slice is defined on unicode codepoints + - cd: r.expr('hello').slice(1) + ot: 'ello' + - cd: r.expr('hello').slice(-1) + ot: 'o' + - cd: r.expr('hello').slice(-4,3) + ot: 'el' + - cd: r.expr('hello').slice(-99) + ot: 'hello' + - cd: r.expr('hello').slice(0) + ot: 'hello' + - cd: r.expr(japanese_hello).slice(1) + ot: + cd: 'んにちは' + py: u'んにちは' + py3.0: 'んにちは' + py3.1: 'んにちは' + py3.2: 'んにちは' + - cd: r.expr(japanese_hello).slice(1,2) + ot: + cd: 'ん' + py: u'ん' + py3.0: 'ん' + py3.1: 'ん' + py3.2: 'ん' + - cd: r.expr(japanese_hello).slice(-3) + ot: + cd: 'にちは' + py: u'にちは' + py3.0: 'にちは' + py3.1: 'にちは' + py3.2: 'にちは' + + # This is how these edge cases are handled in Python. + - cd: r.expr('').split() + ot: [] + - cd: r.expr('').split(null) + ot: [] + - cd: r.expr('').split(' ') + ot: [''] + - cd: r.expr('').split('') + ot: [] + - cd: r.expr('').split(null, 5) + ot: [] + - cd: r.expr('').split(' ', 5) + ot: [''] + - cd: r.expr('').split('', 5) + ot: [] + + - cd: r.expr('aaaa bbbb cccc ').split() + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr('aaaa bbbb cccc ').split(null) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr('aaaa bbbb cccc ').split(' ') + ot: ['aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr('aaaa bbbb cccc ').split('') + ot: ['a', 'a', 'a', 'a', ' ', 'b', 'b', 'b', 'b', ' ', ' ', 'c', 'c', 'c', 'c', ' '] + - cd: r.expr('aaaa bbbb cccc ').split('b') + ot: ['aaaa ', '', '', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb') + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ') + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb') + ot: ['aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr('aaaa bbbb cccc ').split(null, 3) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr('aaaa bbbb cccc ').split(' ', 5) + ot: ['aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr('aaaa bbbb cccc ').split('', 5) + ot: ['a', 'a', 'a', 'a', ' ', 'bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('b', 5) + ot: ['aaaa ', '', '', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb', 3) + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ', 2) + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb', 6) + ot: ['aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 3) + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr('aaaa bbbb cccc ').split(null, 2) + ot: ['aaaa', 'bbbb', 'cccc '] + - cd: r.expr("a b ").split(null, 2) + ot: ["a", "b"] + - cd: r.expr('aaaa bbbb cccc ').split(' ', 4) + ot: ['aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr('aaaa bbbb cccc ').split('', 4) + ot: ['a', 'a', 'a', 'a', ' bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('b', 4) + ot: ['aaaa ', '', '', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb', 2) + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ', 1) + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb', 5) + ot: ['aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 1) + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr('aaaa bbbb cccc ').split(null, 1) + ot: ['aaaa', 'bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' ', 2) + ot: ['aaaa', 'bbbb', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('', 2) + ot: ['a', 'a', 'aa bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('b', 2) + ot: ['aaaa ', '', 'bb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb', 2) + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ', 2) + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb', 2) + ot: ['aaaa ', '', ' cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' ').split() + ot: [] + - cd: r.expr(' ').split(null) + ot: [] + - cd: r.expr(' ').split(' ') + ot: ['', '', ''] + - cd: r.expr(' ').split(null, 5) + ot: [] + - cd: r.expr(' ').split(' ', 5) + ot: ['', '', ''] + + - cd: r.expr(' aaaa bbbb cccc ').split() + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr(' aaaa bbbb cccc ').split(null) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr(' aaaa bbbb cccc ').split(' ') + ot: ['', '', 'aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr(' aaaa bbbb cccc ').split('b') + ot: [' aaaa ', '', '', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb') + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ') + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb') + ot: [' aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' aaaa bbbb cccc ').split(null, 3) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr(' aaaa bbbb cccc ').split(' ', 5) + ot: ['', '', 'aaaa', 'bbbb', '', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('b', 5) + ot: [' aaaa ', '', '', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb', 3) + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ', 2) + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb', 6) + ot: [' aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 3) + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' aaaa bbbb cccc ').split(null, 2) + ot: ['aaaa', 'bbbb', 'cccc '] + - cd: r.expr("a b ").split(null, 2) + ot: ["a", "b"] + - cd: r.expr(' aaaa bbbb cccc ').split(' ', 4) + ot: ['', '', 'aaaa', 'bbbb', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('b', 4) + ot: [' aaaa ', '', '', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb', 2) + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ', 1) + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb', 5) + ot: [' aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 1) + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' aaaa bbbb cccc ').split(null, 1) + ot: ['aaaa', 'bbbb cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' ', 2) + ot: ['', '', 'aaaa bbbb cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('b', 2) + ot: [' aaaa ', '', 'bb cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb', 2) + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ', 2) + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb', 2) + ot: [' aaaa ', '', ' cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr("abc-dEf-GHJ").upcase() + ot: "ABC-DEF-GHJ" + - cd: r.expr("abc-dEf-GHJ").downcase() + ot: "abc-def-ghj" + + # Same 3.0-3.2 caveats + - py: + cd: r.expr(u"f\u00e9oo").split("") + ot: [u"f", u"\u00e9", u"o", u"o"] + py3.0: r.expr("f\u00e9oo").split("") + py3.1: r.expr("f\u00e9oo").split("") + py3.2: r.expr("f\u00e9oo").split("") + cd: r.expr("f\u00e9oo").split("") + ot: ["f", "\u00e9", "o", "o"] + + - py: + cd: r.expr(u"fe\u0301oo").split("") + ot: [u"f", u"e\u0301", u"o", u"o"] + py3.0: r.expr("fe\u0301oo").split("") + py3.1: r.expr("fe\u0301oo").split("") + py3.2: r.expr("fe\u0301oo").split("") + cd: r.expr("fe\u0301oo").split("") + ot: ["f", "e\u0301", "o", "o"] + + ## Unicode spacing characters. + + ## original set from previous work: + - cd: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + py: + cd: r.expr(u"foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + ot: ["foo", "bar", "baz", "quux", "fred", "barney", "wilma"] + py3.0: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + py3.1: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + py3.2: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + ot: ["foo", "bar", "baz", "quux", "fred", "barney", "wilma"] + + ## some specialized Unicode horrors: + ## - U+00A0 is nonbreaking space and is in the Zs category + ## - U+0085 is the next line character and is not in the Zs category but is considered whitespace + ## - U+2001 is em quad space and is in the Zs category + ## - U+200B is a zero width space and is not in the Zs category and is not considered whitespace + ## - U+2060 is a word joining zero width nonbreaking space and is NOT in any of the Z categories + ## - U+2028 is a line separator and is in the Zl category + ## - U+2029 is a paragraph separator and is in the Zp category + - py: + cd: r.expr(u"foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + ot: ["foo", "bar", u"baz\u2060quux", "fred", "barney", "wilma", u"betty\u200b"] + py3.0: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + py3.1: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + py3.2: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + cd: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + ot: ["foo", "bar", "baz\u2060quux", "fred", "barney", "wilma", "betty\u200b"] diff --git a/ext/librethinkdbxx/test/upstream/datum/typeof.yaml b/ext/librethinkdbxx/test/upstream/datum/typeof.yaml new file mode 100644 index 00000000..1f03f53e --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/typeof.yaml @@ -0,0 +1,14 @@ +desc: These tests test the type of command +tests: + + # Method form + - cd: r.expr(null).type_of() + ot: 'NULL' + + # Prefix form + - cd: r.type_of(null) + ot: 'NULL' + + # Error cases + - js: r(null).typeOf(1) + ot: err('ReqlCompileError', 'Expected 1 argument but found 2.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/datum/uuid.yaml b/ext/librethinkdbxx/test/upstream/datum/uuid.yaml new file mode 100644 index 00000000..003e8604 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/uuid.yaml @@ -0,0 +1,20 @@ +desc: Test that UUIDs work +tests: + - cd: r.uuid() + ot: uuid() + - cd: r.expr(r.uuid()) + ot: uuid() + - cd: r.type_of(r.uuid()) + ot: 'STRING' + - cd: r.uuid().ne(r.uuid()) + ot: true + - cd: r.uuid('magic') + ot: ('97dd10a5-4fc4-554f-86c5-0d2c2e3d5330') + - cd: r.uuid('magic').eq(r.uuid('magic')) + ot: true + - cd: r.uuid('magic').ne(r.uuid('beans')) + ot: true + - py: r.expr([1,2,3,4,5,6,7,8,9,10]).map(lambda u:r.uuid()).distinct().count() + js: r([1,2,3,4,5,6,7,8,9,10]).map(function(u) {return r.uuid();}).distinct().count() + rb: r.expr([1,2,3,4,5,6,7,8,9,10]).map {|u| r.uuid()}.distinct().count() + ot: 10 diff --git a/ext/librethinkdbxx/test/upstream/default.yaml b/ext/librethinkdbxx/test/upstream/default.yaml new file mode 100644 index 00000000..e7ba44c0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/default.yaml @@ -0,0 +1,270 @@ +desc: Tests r.default +tests: + - cd: r.expr(1).default(2) + ot: 1 + - cd: r.expr(null).default(2) + ot: 2 + - cd: r.expr({})['b'].default(2) + js: r.expr({})('b').default(2) + ot: 2 + - cd: r.expr(r.expr('a')['b']).default(2) + js: r.expr(r.expr('a')('b')).default(2) + ot: err("ReqlQueryLogicError", "Cannot perform bracket on a non-object non-sequence `\"a\"`.", []) + - rb: r.expr([]).reduce{|a,b| a+b}.default(2) + py: r.expr([]).reduce(lambda a,b:a+b).default(2) + js: r.expr([]).reduce(function(a,b){return a+b}).default(2) + ot: 2 + - rb: r.expr([]).union([]).reduce{|a,b| a+b}.default(2) + py: r.expr([]).union([]).reduce(lambda a,b:a+b).default(2) + js: r.expr([]).union([]).reduce(function(a,b){return a+b}).default(2) + ot: 2 + - rb: r.expr('a').reduce{|a,b| a+b}.default(2) + py: r.expr('a').reduce(lambda a,b:a+b).default(2) + js: r.expr('a').reduce(function(a,b){return a+b}).default(2) + ot: err("ReqlQueryLogicError", "Cannot convert STRING to SEQUENCE", []) + - cd: (r.expr(null) + 5).default(2) + js: (r.expr(null).add(5)).default(2) + ot: 2 + - cd: (5 + r.expr(null)).default(2) + js: (r.expr(5).add(null)).default(2) + ot: 2 + - cd: (5 - r.expr(null)).default(2) + js: (r.expr(5).sub(null)).default(2) + ot: 2 + - cd: (r.expr(null) - 5).default(2) + js: (r.expr(null).sub(5)).default(2) + ot: 2 + - cd: (r.expr('a') + 5).default(2) + js: (r.expr('a').add(5)).default(2) + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", []) + - cd: (5 + r.expr('a')).default(2) + js: (r.expr(5).add('a')).default(2) + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + - cd: (r.expr('a') - 5).default(2) + js: (r.expr('a').sub(5)).default(2) + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + - cd: (5 - r.expr('a')).default(2) + js: (r.expr(5).sub('a')).default(2) + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + - cd: r.expr(1).default(r.error()) + ot: 1 + - cd: r.expr(null).default(r.error()) + ot: (null) + - cd: r.expr({})['b'].default(r.error()) + js: r.expr({})('b').default(r.error()) + ot: err("ReqlNonExistenceError", "No attribute `b` in object:", []) + - rb: r.expr([]).reduce{|a,b| a+b}.default(r.error) + py: r.expr([]).reduce(lambda a,b:a+b).default(r.error) + js: r.expr([]).reduce(function(a,b){return a+b}).default(r.error) + ot: err("ReqlNonExistenceError", "Cannot reduce over an empty stream.", []) + - rb: r.expr([]).union([]).reduce{|a,b| a+b}.default(r.error) + py: r.expr([]).union([]).reduce(lambda a,b:a+b).default(r.error) + js: r.expr([]).union([]).reduce(function(a,b){return a+b}).default(r.error) + ot: err("ReqlNonExistenceError", "Cannot reduce over an empty stream.", []) + - cd: (r.expr(null) + 5).default(r.error) + js: (r.expr(null).add(5)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + - cd: (5 + r.expr(null)).default(r.error) + js: (r.expr(5).add(null)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + - cd: (5 - r.expr(null)).default(r.error) + js: (r.expr(5).sub(null)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + - cd: (r.expr(null) - 5).default(r.error) + js: (r.expr(null).sub(5)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + + - rb: r.expr(1).default{|e| e} + py: r.expr(1).default(lambda e:e) + js: r.expr(1).default(function(e){return e}) + ot: 1 + - cd: r.expr(null).default{|e| e} + py: r.expr(null).default(lambda e:e) + js: r.expr(null).default(function(e){return e}) + ot: (null) + - cd: r.expr({})['b'].default{|e| e} + py: r.expr({})['b'].default(lambda e:e) + js: r.expr({})('b').default(function(e){return e}) + ot: "No attribute `b` in object:\n{}" + - cd: r.expr([]).reduce{|a,b| a+b}.default{|e| e} + py: r.expr([]).reduce(lambda a,b:a+b).default(lambda e:e) + js: r.expr([]).reduce(function(a,b){return a+b}).default(function(e){return e}) + ot: ("Cannot reduce over an empty stream.") + - cd: r.expr([]).union([]).reduce{|a,b| a+b}.default{|e| e} + py: r.expr([]).union([]).reduce(lambda a,b:a+b).default(lambda e:e) + js: r.expr([]).union([]).reduce(function(a,b){return a+b}).default(function(e){return e}) + ot: ("Cannot reduce over an empty stream.") + - cd: (r.expr(null) + 5).default{|e| e} + py: (r.expr(null) + 5).default(lambda e:e) + js: (r.expr(null).add(5)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + - cd: (5 + r.expr(null)).default{|e| e} + py: (5 + r.expr(null)).default(lambda e:e) + js: (r.expr(5).add(null)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + - cd: (5 - r.expr(null)).default{|e| e} + py: (5 - r.expr(null)).default(lambda e:e) + js: (r.expr(5).sub(null)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + - cd: (r.expr(null) - 5).default{|e| e} + py: (r.expr(null) - 5).default(lambda e:e) + js: (r.expr(null).sub(5)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + + - def: arr = r.expr([{'a':1},{'a':null},{}]).order_by('a') + + - cd: arr.filter{|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1)) + js: arr.filter(function(x){return x('a').eq(1)}) + ot: [{'a':1}] + - cd: arr.filter(:default => false){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=False) + js: arr.filter(function(x){return x('a').eq(1)}, {'default':false}) + ot: [{'a':1}] + - cd: arr.filter(:default => true){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=True) + js: arr.filter(function(x){return x('a').eq(1)}, {'default':true}) + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + - cd: arr.filter(:default => r.js('true')){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=r.js('true')) + js: arr.filter(function(x) { return x('a').eq(1) }, { 'default':r.js('true') }) + ot: [{}, {'a':1}] + - cd: arr.filter(:default => r.js('false')){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=r.js('false')) + js: arr.filter(function(x) { return x('a').eq(1) }, { 'default':r.js('false') }) + ot: [{'a':1}] + - cd: arr.filter(:default => r.error){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=r.error()) + js: arr.filter(function(x){return x('a').eq(1)}, {'default':r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.expr(false).do{|d| arr.filter(:default => d){|x| x['a'].eq(1)}} + py: r.expr(False).do(lambda d:arr.filter(lambda x:x['a'].eq(1), default=d)) + js: r.expr(false).do(function(d){return arr.filter(function(x){return x('a').eq(1)}, {default:d})}) + ot: [{'a':1}] + - cd: r.expr(true).do{|d| arr.filter(:default => d){|x| x['a'].eq(1)}}.orderby('a') + py: r.expr(True).do(lambda d:arr.filter(lambda x:x['a'].eq(1), default=d)).order_by('a') + js: r.expr(true).do(function(d){return arr.filter(function(x){return x('a').eq(1)}, {default:d})}).orderBy('a') + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + + - cd: arr.filter{|x| x['a'].default(0).eq(1)} + py: arr.filter(lambda x:x['a'].default(0).eq(1)) + js: arr.filter(function(x){return x('a').default(0).eq(1)}) + ot: [{'a':1}] + - cd: arr.filter{|x| x['a'].default(1).eq(1)}.orderby('a') + py: arr.filter(lambda x:x['a'].default(1).eq(1)).order_by('a') + js: arr.filter(function(x){return x('a').default(1).eq(1)}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: arr.filter{|x| x['a'].default(r.error).eq(1)} + py: arr.filter(lambda x:x['a'].default(r.error()).eq(1)) + js: arr.filter(function(x){return x('a').default(r.error()).eq(1)}) + ot: [{'a':1}] + # gets caught by `filter` default + + - cd: r.expr(0).do{|i| arr.filter{|x| x['a'].default(i).eq(1)}} + py: r.expr(0).do(lambda i:arr.filter(lambda x:x['a'].default(i).eq(1))) + js: r.expr(0).do(function(i){return arr.filter(function(x){return x('a').default(i).eq(1)})}) + ot: [{'a':1}] + - cd: r.expr(1).do{|i| arr.filter{|x| x['a'].default(i).eq(1)}}.orderby('a') + py: r.expr(1).do(lambda i:arr.filter(lambda x:x['a'].default(i).eq(1))).order_by('a') + js: r.expr(1).do(function(i){return arr.filter(function(x){return x('a').default(i).eq(1)})}).orderBy('a') + ot: ([{},{'a':null},{'a':1}]) + + - cd: arr.filter{|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2))) + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}) + ot: [{'a':1}] + - cd: arr.filter(:default => false){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=False) + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:false}) + ot: [{'a':1}] + - cd: arr.filter(:default => true){|x| x['a'].eq(1).or(x['a']['b'].eq(2))}.orderby('a') + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=True).order_by('a') + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:true}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: arr.filter(:default => r.error){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=r.error()) + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.table_create('default_test') + ot: partial({'tables_created':1}) + + - cd: r.table('default_test').insert(arr) + ot: ({'deleted':0,'replaced':0,'generated_keys':arrlen(3,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':3}) + + - def: tbl = r.table('default_test').order_by('a').pluck('a') + + - cd: tbl.filter{|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1)) + js: tbl.filter(function(x){return x('a').eq(1)}) + ot: [{'a':1}] + - cd: tbl.filter(:default => false){|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1), default=False) + js: tbl.filter(function(x){return x('a').eq(1)}, {'default':false}) + ot: [{'a':1}] + - cd: tbl.filter(:default => true){|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1), default=True) + js: tbl.filter(function(x){return x('a').eq(1)}, {'default':true}) + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + - cd: tbl.filter(:default => r.error){|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1), default=r.error()) + js: tbl.filter(function(x){return x('a').eq(1)}, {'default':r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.expr(false).do{|d| tbl.filter(:default => d){|x| x['a'].eq(1)}} + py: r.expr(False).do(lambda d:tbl.filter(lambda x:x['a'].eq(1), default=d)) + js: r.expr(false).do(function(d){return tbl.filter(function(x){return x('a').eq(1)}, {default:d})}) + ot: [{'a':1}] + - cd: r.expr(true).do{|d| tbl.filter(:default => d){|x| x['a'].eq(1)}}.orderby('a') + py: r.expr(True).do(lambda d:tbl.filter(lambda x:x['a'].eq(1), default=d)).order_by('a') + js: r.expr(true).do(function(d){return tbl.filter(function(x){return x('a').eq(1)}, {default:d})}).orderBy('a') + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + + - cd: tbl.filter{|x| x['a'].default(0).eq(1)} + py: tbl.filter(lambda x:x['a'].default(0).eq(1)) + js: tbl.filter(function(x){return x('a').default(0).eq(1)}) + ot: [{'a':1}] + - cd: tbl.filter{|x| x['a'].default(1).eq(1)}.orderby('a') + py: tbl.filter(lambda x:x['a'].default(1).eq(1)).order_by('a') + js: tbl.filter(function(x){return x('a').default(1).eq(1)}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: tbl.filter{|x| x['a'].default(r.error).eq(1)} + py: tbl.filter(lambda x:x['a'].default(r.error()).eq(1)) + js: tbl.filter(function(x){return x('a').default(r.error()).eq(1)}) + ot: [{'a':1}] + # gets caught by `filter` default + + - cd: r.expr(0).do{|i| tbl.filter{|x| x['a'].default(i).eq(1)}} + py: r.expr(0).do(lambda i:tbl.filter(lambda x:x['a'].default(i).eq(1))) + js: r.expr(0).do(function(i){return tbl.filter(function(x){return x('a').default(i).eq(1)})}) + ot: [{'a':1}] + - cd: r.expr(1).do{|i| tbl.filter{|x| x['a'].default(i).eq(1)}}.orderby('a') + py: r.expr(1).do(lambda i:tbl.filter(lambda x:x['a'].default(i).eq(1))).order_by('a') + js: r.expr(1).do(function(i){return tbl.filter(function(x){return x('a').default(i).eq(1)})}).orderBy('a') + ot: ([{},{'a':null},{'a':1}]) + + - cd: tbl.filter{|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2))) + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}) + ot: [{'a':1}] + - cd: tbl.filter(:default => false){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=False) + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:false}) + ot: [{'a':1}] + - cd: tbl.filter(:default => true){|x| x['a'].eq(1).or(x['a']['b'].eq(2))}.orderby('a') + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=True).order_by('a') + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:true}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: tbl.filter(:default => r.error){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=r.error()) + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.table_drop('default_test') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/geo/constructors.yaml b/ext/librethinkdbxx/test/upstream/geo/constructors.yaml new file mode 100644 index 00000000..9991c582 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/constructors.yaml @@ -0,0 +1,64 @@ +desc: Test geo constructors +tests: + # Point + - cd: r.point(0, 0) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 0], 'type':'Point'}) + - cd: r.point(0, -90) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, -90], 'type':'Point'}) + - cd: r.point(0, 90) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 90], 'type':'Point'}) + - cd: r.point(-180, 0) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[-180, 0], 'type':'Point'}) + - cd: r.point(180, 0) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[180, 0], 'type':'Point'}) + - cd: r.point(0, -91) + ot: err('ReqlQueryLogicError', 'Latitude must be between -90 and 90. Got -91.', [0]) + - cd: r.point(0, 91) + ot: err('ReqlQueryLogicError', 'Latitude must be between -90 and 90. Got 91.', [0]) + - cd: r.point(-181, 0) + ot: err('ReqlQueryLogicError', 'Longitude must be between -180 and 180. Got -181.', [0]) + - cd: r.point(181, 0) + ot: err('ReqlQueryLogicError', 'Longitude must be between -180 and 180. Got 181.', [0]) + + # Line + - cd: r.line() + ot: err('ReqlCompileError', 'Expected 2 or more arguments but found 0.', [0]) + - cd: r.line([0,0]) + ot: err('ReqlCompileError', 'Expected 2 or more arguments but found 1.', [0]) + - cd: r.line([0,0], [0,0]) + ot: err('ReqlQueryLogicError', 'Invalid LineString. Are there antipodal or duplicate vertices?', [0]) + - cd: r.line([0,0], [0,1]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1]], 'type':'LineString'}) + - cd: r.line([0,0], [1]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 1 element array instead of a 2 element one.', [0]) + - cd: r.line([0,0], [1,0,0]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 3 element array instead of a 2 element one.', [0]) + - cd: r.line([0,0], [0,1], [0,0]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1], [0,0]], 'type':'LineString'}) + - cd: r.line(r.point(0,0), r.point(0,1), r.point(0,0)) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1], [0,0]], 'type':'LineString'}) + - cd: r.line(r.point(0,0), r.point(1,0), r.line([0,0], [1,0])) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Point` but found `LineString`.', [0]) + + # Polygon + - cd: r.polygon() + ot: err('ReqlCompileError', 'Expected 3 or more arguments but found 0.', [0]) + - cd: r.polygon([0,0]) + ot: err('ReqlCompileError', 'Expected 3 or more arguments but found 1.', [0]) + - cd: r.polygon([0,0], [0,0]) + ot: err('ReqlCompileError', 'Expected 3 or more arguments but found 2.', [0]) + - cd: r.polygon([0,0], [0,0], [0,0], [0,0]) + ot: err('ReqlQueryLogicError', 'Invalid LinearRing. Are there antipodal or duplicate vertices? Is it self-intersecting?', [0]) + - cd: r.polygon([0,0], [0,1], [1,0]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [0,1], [1,0], [0,0]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [0,1], [1,0], [-1,0.5]) + ot: err('ReqlQueryLogicError', 'Invalid LinearRing. Are there antipodal or duplicate vertices? Is it self-intersecting?', [0]) + - cd: r.polygon([0,0], [0,1], [0]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 1 element array instead of a 2 element one.', [0]) + - cd: r.polygon([0,0], [0,1], [0,1,0]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 3 element array instead of a 2 element one.', [0]) + - cd: r.polygon(r.point(0,0), r.point(0,1), r.line([0,0], [0,1])) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Point` but found `LineString`.', [0]) + diff --git a/ext/librethinkdbxx/test/upstream/geo/geojson.yaml b/ext/librethinkdbxx/test/upstream/geo/geojson.yaml new file mode 100644 index 00000000..cc0be877 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/geojson.yaml @@ -0,0 +1,31 @@ +desc: Test geoJSON conversion +tests: + # Basic conversion + - cd: r.geojson({'coordinates':[0, 0], 'type':'Point'}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 0], 'type':'Point'}) + - cd: r.geojson({'coordinates':[[0,0], [0,1]], 'type':'LineString'}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1]], 'type':'LineString'}) + - cd: r.geojson({'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + + # Wrong / missing fields + - cd: r.geojson({'coordinates':[[], 0], 'type':'Point'}) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found ARRAY.', [0]) + - cd: r.geojson({'coordinates':true, 'type':'Point'}) + ot: err('ReqlQueryLogicError', 'Expected type ARRAY but found BOOL.', [0]) + - cd: r.geojson({'type':'Point'}) + ot: err('ReqlNonExistenceError', 'No attribute `coordinates` in object:', [0]) + - cd: r.geojson({'coordinates':[0, 0]}) + ot: err('ReqlNonExistenceError', 'No attribute `type` in object:', [0]) + - cd: r.geojson({'coordinates':[0, 0], 'type':'foo'}) + ot: err('ReqlQueryLogicError', 'Unrecognized GeoJSON type `foo`.', [0]) + - cd: r.geojson({'coordinates':[0, 0], 'type':'Point', 'foo':'wrong'}) + ot: err('ReqlQueryLogicError', 'Unrecognized field `foo` found in geometry object.', [0]) + + # Unsupported features + - cd: r.geojson({'coordinates':[0, 0], 'type':'Point', 'crs':null}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 0], 'type':'Point', 'crs':null}) + - js: r.geojson({'coordinates':[0, 0], 'type':'Point', 'crs':{'type':'name', 'properties':{'name':'test'}}}) + ot: err('ReqlQueryLogicError', 'Non-default coordinate reference systems are not supported in GeoJSON objects. Make sure the `crs` field of the geometry is null or non-existent.', [0]) + - cd: r.geojson({'coordinates':[0, 0], 'type':'MultiPoint'}) + ot: err('ReqlQueryLogicError', 'GeoJSON type `MultiPoint` is not supported.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/geo/indexing.yaml b/ext/librethinkdbxx/test/upstream/geo/indexing.yaml new file mode 100644 index 00000000..059f2929 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/indexing.yaml @@ -0,0 +1,208 @@ +desc: Test ReQL interface to geo indexes +table_variable_name: tbl +tests: + - def: rows = [{'id':0, 'g':r.point(10,10), 'm':[r.point(0,0),r.point(1,0),r.point(2,0)]}, + {'id':1, 'g':r.polygon([0,0], [0,1], [1,1], [1,0])}, + {'id':2, 'g':r.line([0.000002,-1], [-0.000001,1])}] + + - cd: tbl.insert(rows) + ot: ({'deleted':0,'inserted':3,'skipped':0,'errors':0,'replaced':0,'unchanged':0}) + + - rb: tbl.index_create('g', :geo=>true) + py: tbl.index_create('g', geo=true) + js: tbl.indexCreate('g', {'geo':true}) + ot: {'created':1} + - rb: tbl.index_create('m', :geo=>true, :multi=>true) + py: tbl.index_create('m', geo=true, multi=true) + js: tbl.indexCreate('m', {'geo':true, 'multi':true}) + ot: {'created':1} + - cd: tbl.index_create('other') + ot: {'created':1} + # r.point is deterministic and can be used in an index function + - rb: tbl.index_create('point_det'){ |x| r.point(x, x) } + py: tbl.index_create('point_det', lambda x: r.point(x, x) ) + js: tbl.indexCreate('point_det', function(x) {return r.point(x, x);} ) + ot: {'created':1} + + - cd: tbl.index_wait() + + # r.line (and friends) are non-deterministic across servers and should be disallowed + # in index functions + - rb: tbl.index_create('point_det'){ |x| r.line(x, x) } + py: tbl.index_create('point_det', lambda x: r.line(x, x) ) + js: tbl.indexCreate('point_det', function(x) {return r.line(x, x);} ) + ot: err('ReqlQueryLogicError', 'Could not prove function deterministic. Index functions must be deterministic.') + + - js: tbl.get_intersecting(r.point(0,0), {'index':'other'}).count() + py: tbl.get_intersecting(r.point(0,0), index='other').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'other').count() + ot: err('ReqlQueryLogicError', 'Index `other` is not a geospatial index. get_intersecting can only be used with a geospatial index.', [0]) + - js: tbl.get_intersecting(r.point(0,0), {'index':'missing'}).count() + py: tbl.get_intersecting(r.point(0,0), index='missing').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'missing').count() + ot: err_regex('ReqlOpFailedError', 'Index `missing` was not found on table `[a-zA-Z0-9_]+.[a-zA-Z0-9_]+`[.]', [0]) + - cd: tbl.get_intersecting(r.point(0,0)).count() + ot: err('ReqlQueryLogicError', 'get_intersecting requires an index argument.', [0]) + - js: tbl.get_all(0, {'index':'g'}).count() + py: tbl.get_all(0, index='g').count() + rb: tbl.get_all(0, :index=>'g').count() + ot: err('ReqlQueryLogicError', 'Index `g` is a geospatial index. Only get_nearest and get_intersecting can use a geospatial index.', [0]) + - js: tbl.between(0, 1, {'index':'g'}).count() + py: tbl.between(0, 1, index='g').count() + rb: tbl.between(0, 1, :index=>'g').count() + ot: err('ReqlQueryLogicError', 'Index `g` is a geospatial index. Only get_nearest and get_intersecting can use a geospatial index.', [0]) + - js: tbl.order_by({'index':'g'}).count() + py: tbl.order_by(index='g').count() + rb: tbl.order_by(:index=>'g').count() + ot: err('ReqlQueryLogicError', 'Index `g` is a geospatial index. Only get_nearest and get_intersecting can use a geospatial index.', [0]) + - js: tbl.between(0, 1).get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.between(0, 1).get_intersecting(r.point(0,0), index='g').count() + rb: tbl.between(0, 1).get_intersecting(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'Between' object has no attribute 'get_intersecting'") + - js: tbl.get_all(0).get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.get_all(0).get_intersecting(r.point(0,0), index='g').count() + rb: tbl.get_all(0).get_intersecting(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found SELECTION:', [0]) + py: err('AttributeError', "'GetAll' object has no attribute 'get_intersecting'") + - js: tbl.order_by({'index':'id'}).get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.order_by(index='id').get_intersecting(r.point(0,0), index='g').count() + rb: tbl.order_by(:index=>'id').get_intersecting(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'OrderBy' object has no attribute 'get_intersecting'") + - js: tbl.get_intersecting(r.point(0,0), {'index':'id'}).count() + py: tbl.get_intersecting(r.point(0,0), index='id').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'id').count() + ot: err('ReqlQueryLogicError', 'get_intersecting cannot use the primary index.', [0]) + + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(0,0), index='g').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').count() + ot: 1 + - js: tbl.get_intersecting(r.point(10,10), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(10,10), index='g').count() + rb: tbl.get_intersecting(r.point(10,10), :index=>'g').count() + ot: 1 + - js: tbl.get_intersecting(r.point(0.5,0.5), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(0.5,0.5), index='g').count() + rb: tbl.get_intersecting(r.point(0.5,0.5), :index=>'g').count() + ot: 1 + - js: tbl.get_intersecting(r.point(20,20), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(20,20), index='g').count() + rb: tbl.get_intersecting(r.point(20,20), :index=>'g').count() + ot: 0 + - js: tbl.get_intersecting(r.polygon([0,0], [1,0], [1,1], [0,1]), {'index':'g'}).count() + py: tbl.get_intersecting(r.polygon([0,0], [1,0], [1,1], [0,1]), index='g').count() + rb: tbl.get_intersecting(r.polygon([0,0], [1,0], [1,1], [0,1]), :index=>'g').count() + ot: 2 + - js: tbl.get_intersecting(r.line([0,0], [10,10]), {'index':'g'}).count() + py: tbl.get_intersecting(r.line([0,0], [10,10]), index='g').count() + rb: tbl.get_intersecting(r.line([0,0], [10,10]), :index=>'g').count() + ot: 3 + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).type_of() + py: tbl.get_intersecting(r.point(0,0), index='g').type_of() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').type_of() + ot: ("SELECTION") + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).filter(true).type_of() + py: tbl.get_intersecting(r.point(0,0), index='g').filter(true).type_of() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').filter(true).type_of() + ot: ("SELECTION") + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).map(r.row).type_of() + py: tbl.get_intersecting(r.point(0,0), index='g').map(r.row).type_of() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').map{|x|x}.type_of() + ot: ("STREAM") + + - js: tbl.get_intersecting(r.point(0,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(0,0), index='m').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'m').count() + ot: 1 + - js: tbl.get_intersecting(r.point(1,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(1,0), index='m').count() + rb: tbl.get_intersecting(r.point(1,0), :index=>'m').count() + ot: 1 + - js: tbl.get_intersecting(r.point(2,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(2,0), index='m').count() + rb: tbl.get_intersecting(r.point(2,0), :index=>'m').count() + ot: 1 + - js: tbl.get_intersecting(r.point(3,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(3,0), index='m').count() + rb: tbl.get_intersecting(r.point(3,0), :index=>'m').count() + ot: 0 + # The document is emitted once for each match. + - js: tbl.get_intersecting(r.polygon([0,0], [0,1], [1,1], [1,0]), {'index':'m'}).count() + py: tbl.get_intersecting(r.polygon([0,0], [0,1], [1,1], [1,0]), index='m').count() + rb: tbl.get_intersecting(r.polygon([0,0], [0,1], [1,1], [1,0]), :index=>'m').count() + ot: 2 + + + - js: tbl.get_nearest(r.point(0,0), {'index':'other'}) + py: tbl.get_nearest(r.point(0,0), index='other') + rb: tbl.get_nearest(r.point(0,0), :index=>'other') + ot: err('ReqlQueryLogicError', 'Index `other` is not a geospatial index. get_nearest can only be used with a geospatial index.', [0]) + - js: tbl.get_nearest(r.point(0,0), {'index':'missing'}) + py: tbl.get_nearest(r.point(0,0), index='missing') + rb: tbl.get_nearest(r.point(0,0), :index=>'missing') + ot: err_regex('ReqlOpFailedError', 'Index `missing` was not found on table `[a-zA-Z0-9_]+.[a-zA-Z0-9_]+`[.]', [0]) + - cd: tbl.get_nearest(r.point(0,0)) + ot: err('ReqlQueryLogicError', 'get_nearest requires an index argument.', [0]) + - js: tbl.between(0, 1).get_nearest(r.point(0,0), {'index':'g'}).count() + py: tbl.between(0, 1).get_nearest(r.point(0,0), index='g').count() + rb: tbl.between(0, 1).get_nearest(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'Between' object has no attribute 'get_nearest'") + - js: tbl.get_all(0).get_nearest(r.point(0,0), {'index':'g'}).count() + py: tbl.get_all(0).get_nearest(r.point(0,0), index='g').count() + rb: tbl.get_all(0).get_nearest(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found SELECTION:', [0]) + py: err('AttributeError', "'GetAll' object has no attribute 'get_nearest'") + - js: tbl.order_by({'index':'id'}).get_nearest(r.point(0,0), {'index':'g'}).count() + py: tbl.order_by(index='id').get_nearest(r.point(0,0), index='g').count() + rb: tbl.order_by(:index=>'id').get_nearest(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'OrderBy' object has no attribute 'get_nearest'") + - js: tbl.get_nearest(r.point(0,0), {'index':'id'}).count() + py: tbl.get_nearest(r.point(0,0), index='id').count() + rb: tbl.get_nearest(r.point(0,0), :index=>'id').count() + ot: err('ReqlQueryLogicError', 'get_nearest cannot use the primary index.', [0]) + + - js: tbl.get_nearest(r.point(0,0), {'index':'g'}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.055659745396754216,'doc':{'id':2}}]) + - js: tbl.get_nearest(r.point(-0.000001,1), {'index':'g'}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(-0.000001,1), index='g').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(-0.000001,1), :index=>'g').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':2}},{'dist':0.11130264976984369,'doc':{'id':1}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':1565110}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1565110).pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1565110).pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.055659745396754216,'doc':{'id':2}},{'dist':1565109.0992178896,'doc':{'id':0}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':1565110, 'max_results':2}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1565110, max_results=2).pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1565110, :max_results=>2).pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.055659745396754216,'doc':{'id':2}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':10000000}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=10000000).pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>10000000).pluck('dist', {'doc':'id'}) + ot: err('ReqlQueryLogicError', 'The distance has become too large for continuing the indexed nearest traversal. Consider specifying a smaller `max_dist` parameter. (Radius must be smaller than a quarter of the circumference along the minor axis of the reference ellipsoid. Got 10968937.995244588703m, but must be smaller than 9985163.1855612862855m.)', [0]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':1566, 'unit':'km'}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1566, unit='km').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1566, :unit=>'km').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.00005565974539675422,'doc':{'id':2}},{'dist':1565.1090992178897,'doc':{'id':0}}]) + - py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1, geo_system='unit_sphere').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1, :geo_system=>'unit_sphere').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0, 'doc':{'id':1}}, {'dist':8.726646259990191e-09, 'doc':{'id':2}}, {'dist':0.24619691677893205, 'doc':{'id':0}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g'}).type_of() + py: tbl.get_nearest(r.point(0,0), index='g').type_of() + rb: tbl.get_nearest(r.point(0,0), :index=>'g').type_of() + ot: ("ARRAY") + - js: tbl.get_nearest(r.point(0,0), {'index':'g'}).map(r.row).type_of() + py: tbl.get_nearest(r.point(0,0), index='g').map(r.row).type_of() + rb: tbl.get_nearest(r.point(0,0), :index=>'g').map{|x|x}.type_of() + ot: ("ARRAY") diff --git a/ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml b/ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml new file mode 100644 index 00000000..18a643a0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml @@ -0,0 +1,119 @@ +desc: Test intersects and includes semantics +tests: + # Intersects + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(1.5,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(2.5,2.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.5,1.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.05,1.05)) + ot: true + # Our current semantics: we define polygons as closed, so points that are exactly *on* the outline of a polygon do intersect + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(2,2)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(2,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([1.5,1.5], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([1.5,1.5], [2,1.5])) + ot: true + # (...with holes in the polygon being closed with respect to the polygon, i.e. the set cut out is open) + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.1,1.1)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.5,1.1)) + ot: true + # ... lines are interpreted as closed sets as well, so even if they meet only at their end points, we consider them as intersecting. + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([2,2], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([2,1.5], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([1.5,1.5], [3,3])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([1.5,1.5], [2.5,1.5], [2.5,2.5], [1.5,2.5])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])) + ot: false + # Polygons behave like lines in that respect + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([2,1.1], [3,1.1], [3,1.9], [2,1.9])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([2,2], [3,2], [3,3], [2,3])) + ot: false + - cd: r.point(1,1).intersects(r.point(1.5,1.5)) + ot: false + - cd: r.point(1,1).intersects(r.point(1,1)) + ot: true + - cd: r.line([1,1], [2,1]).intersects(r.point(1,1)) + ot: true + # This one currently fails due to numeric precision problems. + #- cd: r.line([1,0], [2,0]).intersects(r.point(1.5,0)) + # ot: true + - cd: r.line([1,1], [1,2]).intersects(r.point(1,1.8)) + ot: true + - cd: r.line([1,0], [2,0]).intersects(r.point(1.8,0)) + ot: true + - cd: r.line([1,1], [2,1]).intersects(r.point(1.5,1.5)) + ot: false + - cd: r.line([1,1], [2,1]).intersects(r.line([2,1], [3,1])) + ot: true + # intersects on an array/stream + - cd: r.expr([r.point(1, 0), r.point(3,0), r.point(2, 0)]).intersects(r.line([0,0], [2, 0])).count() + ot: 2 + + # Includes + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(1.5,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(2.5,2.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.5,1.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.05,1.05)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(2,2)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(2,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([1.5,1.5], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([1.5,1.5], [2,1.5])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.1,1.1)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.5,1.1)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([2,2], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([2,1.5], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([2,1], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([1.5,1.5], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1,1], [2,1], [2,2], [1,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1.5,1.5], [2,1.5], [2,2], [1.5,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1.5,1.5], [2.5,1.5], [2.5,2.5], [1.5,2.5])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.polygon([1.1,1.1], [2,1.1], [2,2], [1.1,2])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([2,1.1], [3,1.1], [3,1.9], [2,1.9])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([2,2], [3,2], [3,3], [2,3])) + ot: false + # includes on an array/stream + - cd: r.expr([r.polygon([0,0], [1,1], [1,0]), r.polygon([0,1], [1,2], [1,1])]).includes(r.point(0,0)).count() + ot: 1 + # Wrong geometry type arguments (the first one must be a polygon) + - cd: r.point(0,0).includes(r.point(0,0)) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Polygon` but found `Point`.') + - cd: r.line([0,0], [0,1]).includes(r.point(0,0)) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Polygon` but found `LineString`.') diff --git a/ext/librethinkdbxx/test/upstream/geo/operations.yaml b/ext/librethinkdbxx/test/upstream/geo/operations.yaml new file mode 100644 index 00000000..85e07e93 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/operations.yaml @@ -0,0 +1,97 @@ +desc: Test basic geometry operators +tests: + # Distance + # coerce_to('STRING') because the test utility has some issues with rounding and I'm too lazy to investigate that now. + - cd: r.distance(r.point(-122, 37), r.point(-123, 37)).coerce_to('STRING') + ot: ("89011.26253835332") + - cd: r.distance(r.point(-122, 37), r.point(-122, 36)).coerce_to('STRING') + ot: ("110968.30443995494") + - cd: r.distance(r.point(-122, 37), r.point(-122, 36)).eq(r.distance(r.point(-122, 36), r.point(-122, 37))) + ot: true + - cd: r.point(-122, 37).distance(r.point(-123, 37)).coerce_to('STRING') + ot: ("89011.26253835332") + - def: someDist = r.distance(r.point(-122, 37), r.point(-123, 37)) + js: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'m'})) + py: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='m')) + rb: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'m')) + ot: true + - js: someDist.mul(1.0/1000.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'km'})) + py: someDist.mul(1.0/1000.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='km')) + rb: someDist.mul(1.0/1000.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'km')) + ot: true + - js: someDist.mul(1.0/1609.344).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'mi'})) + py: someDist.mul(1.0/1609.344).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='mi')) + rb: someDist.mul(1.0/1609.344).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'mi')) + ot: true + - js: someDist.mul(1.0/0.3048).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'ft'})) + py: someDist.mul(1.0/0.3048).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='ft')) + rb: someDist.mul(1.0/0.3048).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'ft')) + ot: true + - js: someDist.mul(1.0/1852.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'nm'})) + py: someDist.mul(1.0/1852.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='nm')) + rb: someDist.mul(1.0/1852.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'nm')) + ot: true + - js: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), {'geo_system':'WGS84'})) + py: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), geo_system='WGS84')) + rb: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), :geo_system=>'WGS84')) + ot: true + # Mearth is a small planet, just 1/10th of earth's size. + - js: someDist.div(10).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {'geo_system':{'a':637813.7, 'f':(1.0/298.257223563)}})) + py: someDist.div(10).eq(r.distance(r.point(-122, 37), r.point(-123, 37), geo_system={'a':637813.7, 'f':(1.0/298.257223563)})) + rb: someDist.div(10).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :geo_system=>{'a':637813.7, 'f':(1.0/298.257223563)})) + ot: true + - py: r.distance(r.point(-122, 37), r.point(-123, 37), geo_system='unit_sphere').coerce_to('STRING') + rb: r.distance(r.point(-122, 37), r.point(-123, 37), :geo_system=>'unit_sphere').coerce_to('STRING') + js: r.distance(r.point(-122, 37), r.point(-123, 37), {'geo_system':'unit_sphere'}).coerce_to('STRING') + ot: ("0.01393875509649327") + - cd: r.distance(r.point(0, 0), r.point(0, 0)).coerce_to('STRING') + ot: ("0") + # These two give the earth's circumference through the poles + - cd: r.distance(r.point(0, 0), r.point(180, 0)).mul(2).coerce_to('STRING') + ot: ("40007862.917250897") + - cd: r.distance(r.point(0, -90), r.point(0, 90)).mul(2).coerce_to('STRING') + ot: ("40007862.917250897") + - cd: r.distance(r.point(0, 0), r.line([0,0], [0,1])).coerce_to('STRING') + ot: ("0") + - cd: r.distance(r.line([0,0], [0,1]), r.point(0, 0)).coerce_to('STRING') + ot: ("0") + - cd: r.distance(r.point(0, 0), r.line([0.1,0], [1,0])).eq(r.distance(r.point(0, 0), r.point(0.1, 0))) + ot: true + - cd: r.distance(r.point(0, 0), r.line([5,-1], [4,2])).coerce_to('STRING') + ot: ("492471.4990055255") + - cd: r.distance(r.point(0, 0), r.polygon([5,-1], [4,2], [10,10])).coerce_to('STRING') + ot: ("492471.4990055255") + - cd: r.distance(r.point(0, 0), r.polygon([0,-1], [0,1], [10,10])).coerce_to('STRING') + ot: ("0") + - cd: r.distance(r.point(0.5, 0.5), r.polygon([0,-1], [0,1], [10,10])).coerce_to('STRING') + ot: ("0") + + # Fill + - js: r.circle([0,0], 1, {fill:false}).eq(r.circle([0,0], 1, {fill:true})) + py: r.circle([0,0], 1, fill=false).eq(r.circle([0,0], 1, fill=true)) + rb: r.circle([0,0], 1, :fill=>false).eq(r.circle([0,0], 1, :fill=>true)) + ot: false + - js: r.circle([0,0], 1, {fill:false}).fill().eq(r.circle([0,0], 1, {fill:true})) + py: r.circle([0,0], 1, fill=false).fill().eq(r.circle([0,0], 1, fill=true)) + rb: r.circle([0,0], 1, :fill=>false).fill().eq(r.circle([0,0], 1, :fill=>true)) + ot: true + + # Subtraction + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0.1,0.1], [0.9,0.1], [0.9,0.9], [0.1,0.9])) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0],[1,0],[1,1],[0,1],[0,0]],[[0.1,0.1],[0.9,0.1],[0.9,0.9],[0.1,0.9],[0.1,0.1]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0.1,0.9], [0.9,0.0], [0.9,0.9], [0.1,0.9])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,0], [2,0], [2,2], [0,2])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,-2], [1,-2], [-1,1], [0,-1])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,-1], [1,-1], [1,0], [0,0])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0.1,-1], [0.9,-1], [0.9,0.5], [0.1,0.5])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,0],[0.1,0.9],[0.9,0.9],[0.9,0.1])) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0],[1,0],[1,1],[0,1],[0,0]],[[0,0],[0.1,0.9],[0.9,0.9],[0.9,0.1],[0,0]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,0],[0.1,0.9],[0.9,0.9],[0.9,0.1]).polygon_sub(r.polygon([0.2,0.2],[0.5,0.8],[0.8,0.2]))) + ot: err('ReqlQueryLogicError', 'Expected a Polygon with only an outer shell. This one has holes.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.line([0,0],[0.9,0.1],[0.9,0.9],[0.1,0.9])) + ot: err('ReqlQueryLogicError', 'Expected a Polygon but found a LineString.', []) diff --git a/ext/librethinkdbxx/test/upstream/geo/primitives.yaml b/ext/librethinkdbxx/test/upstream/geo/primitives.yaml new file mode 100644 index 00000000..1fd2c0e0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/primitives.yaml @@ -0,0 +1,50 @@ +desc: Test geometric primitive constructors +tests: + # Circle + - js: r.circle([0,0], 1, {num_vertices:3}) + py: r.circle([0,0], 1, num_vertices=3) + rb: r.circle([0,0], 1, :num_vertices=>3) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle(r.point(0,0), 1, {num_vertices:3}) + py: r.circle(r.point(0,0), 1, num_vertices=3) + rb: r.circle(r.point(0,0), 1, :num_vertices=>3) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle([0,0], 1, {num_vertices:3, fill:false}) + py: r.circle([0,0], 1, num_vertices=3, fill=false) + rb: r.circle([0,0], 1, :num_vertices=>3, :fill=>false) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]], 'type':'LineString'}) + + - js: r.circle([0,0], 14000000, {num_vertices:3}) + py: r.circle([0,0], 14000000, num_vertices=3) + rb: r.circle([0,0], 14000000, :num_vertices=>3) + ot: err('ReqlQueryLogicError', 'Radius must be smaller than a quarter of the circumference along the minor axis of the reference ellipsoid. Got 14000000m, but must be smaller than 9985163.1855612862855m.', [0]) + + - js: r.circle([0,0], 1, {num_vertices:3, geo_system:'WGS84'}) + py: r.circle([0,0], 1, num_vertices=3, geo_system='WGS84') + rb: r.circle([0,0], 1, :num_vertices=>3, :geo_system=>'WGS84') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle([0,0], 2, {num_vertices:3, geo_system:'unit_'+'sphere'}) + py: r.circle([0,0], 2, num_vertices=3, geo_system='unit_sphere') + rb: r.circle([0,0], 2, :num_vertices=>3, :geo_system=>'unit_sphere') + ot: err('ReqlQueryLogicError', 'Radius must be smaller than a quarter of the circumference along the minor axis of the reference ellipsoid. Got 2m, but must be smaller than 1.570796326794896558m.', [0]) + + - js: r.circle([0,0], 0.1, {num_vertices:3, geo_system:'unit_'+'sphere'}) + py: r.circle([0,0], 0.1, num_vertices=3, geo_system='unit_sphere') + rb: r.circle([0,0], 0.1, :num_vertices=>3, :geo_system=>'unit_sphere') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -5.729577951308232], [-4.966092947444857, 2.861205754495701], [4.966092947444857, 2.861205754495701], [0, -5.729577951308232]]], 'type':'Polygon'}) + testopts: + precision: 0.0000000000001 + + - js: r.circle([0,0], 1.0/1000.0, {num_vertices:3, unit:'km'}) + py: r.circle([0,0], 1.0/1000.0, num_vertices=3, unit='km') + rb: r.circle([0,0], 1.0/1000.0, :num_vertices=>3, :unit=>'km') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle([0,0], 1.0/1609.344, {num_vertices:3, unit:'mi'}) + py: r.circle([0,0], 1.0/1609.344, num_vertices=3, unit='mi') + rb: r.circle([0,0], 1.0/1609.344, :num_vertices=>3, :unit=>'mi') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + diff --git a/ext/librethinkdbxx/test/upstream/joins.yaml b/ext/librethinkdbxx/test/upstream/joins.yaml new file mode 100644 index 00000000..80ffed50 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/joins.yaml @@ -0,0 +1,133 @@ +desc: Tests that manipulation data in tables +table_variable_name: tbl, tbl2, senders, receivers, messages, otbl, otbl2 +tests: + + # Setup some more tables + + - py: r.db('test').table_create('test3', primary_key='foo') + rb: r.db('test').table_create('test3', {:primary_key=>'foo'}) + js: r.db('test').tableCreate('test3', {'primaryKey':'foo'}) + ot: partial({'tables_created':1}) + - def: tbl3 = r.db('test').table('test3') + + - py: tbl.insert(r.range(0, 100).map({'id':r.row, 'a':r.row % 4})) + rb: tbl.insert(r.range(0, 100).map{|row| {'id':row, a:row % 4}}) + js: tbl.insert(r.range(0, 100).map(function (row) { return {'id':row, 'a':row.mod(4)}; })) + ot: partial({'errors':0, 'inserted':100}) + + - py: tbl2.insert(r.range(0, 100).map({'id':r.row, 'b':r.row % 4})) + rb: tbl2.insert(r.range(0, 100).map{|row| {'id':row, b:row % 4}}) + js: tbl2.insert(r.range(0, 100).map(function (row) { return {'id':row, 'b':row.mod(4)}; })) + ot: partial({'errors':0, 'inserted':100}) + + - py: tbl3.insert(r.range(0, 100).map({'foo':r.row, 'b':r.row % 4})) + rb: tbl3.insert(r.range(0, 100).map{|row| {'foo':row, b:row % 4}}) + js: tbl3.insert(r.range(0, 100).map(function (row) { return {'foo':row, 'b':row.mod(4)}; })) + ot: partial({'errors':0, 'inserted':100}) + + - py: otbl.insert(r.range(1,100).map({'id': r.row, 'a': r.row})) + - py: otbl2.insert(r.range(1,100).map({'id': r.row, 'b': 2 * r.row})) + + # Inner-Join + + - def: + py: ij = tbl.inner_join(tbl2, lambda x,y:x['a'] == y['b']).zip() + js: ij = tbl.innerJoin(tbl2, function(x, y) { return x('a').eq(y('b')); }).zip() + rb: ij = tbl.inner_join(tbl2){ |x, y| x[:a].eq y[:b] }.zip + - cd: ij.count() + ot: 2500 + - py: ij.filter(lambda row:row['a'] != row['b']).count() + js: ij.filter(function(row) { return row('a').ne(row('b')); }).count() + rb: ij.filter{ |row| row[:a].ne row[:b] }.count + ot: 0 + + # Outer-Join + - def: + py: oj = tbl.outer_join(tbl2, lambda x,y:x['a'] == y['b']).zip() + js: oj = tbl.outerJoin(tbl2, function(x, y) { return x('a').eq(y('b')); }).zip() + rb: oj = tbl.outer_join(tbl2){ |x, y| x[:a].eq y[:b] }.zip + - cd: oj.count() + ot: 2500 + - py: oj.filter(lambda row:row['a'] != row['b']).count() + js: oj.filter(function(row) { return row('a').ne(row('b')); }).count() + rb: oj.filter{ |row| row[:a].ne row[:b] }.count + ot: 0 + + # Ordered eq_join + - py: blah = otbl.order_by("id").eq_join(r.row['id'], otbl2, ordered=True).zip() + ot: [{'id': i, 'a': i, 'b': i * 2} for i in range(1, 100)] + - py: blah = otbl.order_by(r.desc("id")).eq_join(r.row['id'], otbl2, ordered=True).zip() + ot: [{'id': i, 'a': i, 'b': i * 2} for i in range(99, 0, -1)] + - py: blah = otbl.order_by("id").eq_join(r.row['a'], otbl2, ordered=True).zip() + ot: [{'id': i, 'a': i, 'b': i * 2} for i in range(1, 100)] + + # Eq-Join + - cd: tbl.eq_join('a', tbl2).zip().count() + ot: 100 + + - cd: tbl.eq_join('fake', tbl2).zip().count() + ot: 0 + + - py: tbl.eq_join(lambda x:x['a'], tbl2).zip().count() + rb: tbl.eq_join(lambda{|x| x['a']}, tbl2).zip().count() + js: tbl.eq_join(function(x) { return x('a'); }, tbl2).zip().count() + ot: 100 + + - py: tbl.eq_join(lambda x:x['fake'], tbl2).zip().count() + rb: tbl.eq_join(lambda{|x| x['fake']}, tbl2).zip().count() + js: tbl.eq_join(function(x) { return x('fake'); }, tbl2).zip().count() + ot: 0 + + - py: tbl.eq_join(lambda x:null, tbl2).zip().count() + rb: tbl.eq_join(lambda{|x| null}, tbl2).zip().count() + js: tbl.eq_join(function(x) { return null; }, tbl2).zip().count() + ot: 0 + + - py: tbl.eq_join(lambda x:x['a'], tbl2).count() + rb: tbl.eq_join(lambda {|x| x[:a]}, tbl2).count() + js: tbl.eq_join(function(x) { return x('a'); }, tbl2).count() + ot: 100 + + # eqjoin where id isn't a primary key + - cd: tbl.eq_join('a', tbl3).zip().count() + ot: 100 + + - py: tbl.eq_join(lambda x:x['a'], tbl3).count() + rb: tbl.eq_join(lambda {|x| x[:a]}, tbl3).count() + js: tbl.eq_join(function(x) { return x('a'); }, tbl3).count() + ot: 100 + + # eq_join with r.row + - py: tbl.eq_join(r.row['a'], tbl2).count() + js: tbl.eq_join(r.row('a'), tbl2).count() + ot: 100 + + # test an inner-join condition where inner-join differs from outer-join + - def: left = r.expr([{'a':1},{'a':2},{'a':3}]) + - def: right = r.expr([{'b':2},{'b':3}]) + + - py: left.inner_join(right, lambda l, r:l['a'] == r['b']).zip() + js: left.innerJoin(right, function(l, r) { return l('a').eq(r('b')); }).zip() + rb: left.inner_join(right){ |lt, rt| lt[:a].eq(rt[:b]) }.zip + ot: [{'a':2,'b':2},{'a':3,'b':3}] + + # test an outer-join condition where outer-join differs from inner-join + - py: left.outer_join(right, lambda l, r:l['a'] == r['b']).zip() + js: left.outerJoin(right, function(l, r) { return l('a').eq(r('b')); }).zip() + rb: left.outer_join(right){ |lt, rt| lt[:a].eq(rt[:b]) }.zip + ot: [{'a':1},{'a':2,'b':2},{'a':3,'b':3}] + + - rb: senders.insert({id:1, sender:'Sender One'})['inserted'] + ot: 1 + - rb: receivers.insert({id:1, receiver:'Receiver One'})['inserted'] + ot: 1 + - rb: messages.insert([{id:10, sender_id:1, receiver_id:1, msg:'Message One'}, {id:20, sender_id:1, receiver_id:1, msg:'Message Two'}, {id:30, sender_id:1, receiver_id:1, msg:'Message Three'}])['inserted'] + ot: 3 + + - rb: messages.orderby(index:'id').eq_join('sender_id', senders).without({right:{id:true}}).zip.eq_join('receiver_id', receivers).without({right:{id:true}}).zip + ot: [{'id':10,'msg':'Message One','receiver':'Receiver One','receiver_id':1,'sender':'Sender One','sender_id':1},{'id':20,'msg':'Message Two','receiver':'Receiver One','receiver_id':1,'sender':'Sender One','sender_id':1},{'id':30,'msg':'Message Three','receiver':'Receiver One','receiver_id':1,'sender':'Sender One','sender_id':1}] + + # Clean up + + - cd: r.db('test').table_drop('test3') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/json.yaml b/ext/librethinkdbxx/test/upstream/json.yaml new file mode 100644 index 00000000..2a896cd4 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/json.yaml @@ -0,0 +1,74 @@ +desc: Tests RQL json parsing +tests: + + - cd: r.json("[1,2,3]") + ot: [1,2,3] + + - cd: r.json("1") + ot: 1 + + - cd: r.json("{}") + ot: {} + + - cd: r.json('"foo"') + ot: "foo" + + - cd: r.json("[1,2") + ot: err("ReqlQueryLogicError", 'Failed to parse "[1,2" as JSON:' + ' Missing a comma or \']\' after an array element.', [0]) + + - cd: r.json("[1,2,3]").to_json_string() + ot: '[1,2,3]' + + - js: r.json("[1,2,3]").toJSON() + py: r.json("[1,2,3]").to_json() + ot: '[1,2,3]' + + - cd: r.json("{\"foo\":4}").to_json_string() + ot: '{"foo":4}' + + - js: r.json("{\"foo\":4}").toJSON() + py: r.json("{\"foo\":4}").to_json() + ot: '{"foo":4}' + + # stress test: data is from http://www.mockaroo.com/ + - def: text = '[{"id":1,"first_name":"Harry","last_name":"Riley","email":"hriley0@usgs.gov","country":"Andorra","ip_address":"221.25.65.136"},{"id":2,"first_name":"Bonnie","last_name":"Anderson","email":"banderson1@list-manage.com","country":"Tuvalu","ip_address":"116.162.43.150"},{"id":3,"first_name":"Marie","last_name":"Schmidt","email":"mschmidt2@diigo.com","country":"Iraq","ip_address":"181.105.59.57"},{"id":4,"first_name":"Phillip","last_name":"Willis","email":"pwillis3@com.com","country":"Montenegro","ip_address":"24.223.139.156"}]' + - def: sorted = '[{"country":"Andorra","email":"hriley0@usgs.gov","first_name":"Harry","id":1,"ip_address":"221.25.65.136","last_name":"Riley"},{"country":"Tuvalu","email":"banderson1@list-manage.com","first_name":"Bonnie","id":2,"ip_address":"116.162.43.150","last_name":"Anderson"},{"country":"Iraq","email":"mschmidt2@diigo.com","first_name":"Marie","id":3,"ip_address":"181.105.59.57","last_name":"Schmidt"},{"country":"Montenegro","email":"pwillis3@com.com","first_name":"Phillip","id":4,"ip_address":"24.223.139.156","last_name":"Willis"}]' + + - cd: r.json(text).to_json_string() + ot: sorted + + - cd: r.expr(r.minval).to_json_string() + ot: err('ReqlQueryLogicError', 'Cannot convert `r.minval` to JSON.') + + - cd: r.expr(r.maxval).to_json_string() + ot: err('ReqlQueryLogicError', 'Cannot convert `r.maxval` to JSON.') + + - cd: r.expr(r.minval).coerce_to('string') + ot: err('ReqlQueryLogicError', 'Cannot convert `r.minval` to JSON.') + + - cd: r.expr(r.maxval).coerce_to('string') + ot: err('ReqlQueryLogicError', 'Cannot convert `r.maxval` to JSON.') + + - cd: r.time(2014,9,11, 'Z') + runopts: + time_format: 'raw' + ot: {'timezone':'+00:00','$reql_type$':'TIME','epoch_time':1410393600} + + - cd: r.time(2014,9,11, 'Z').to_json_string() + ot: '{"$reql_type$":"TIME","epoch_time":1410393600,"timezone":"+00:00"}' + + - cd: r.point(0,0) + ot: {'$reql_type$':'GEOMETRY','coordinates':[0,0],'type':'Point'} + + - cd: r.point(0,0).to_json_string() + ot: '{"$reql_type$":"GEOMETRY","coordinates":[0,0],"type":"Point"}' + + - def: + rb: s = "\x66\x6f\x6f".force_encoding('BINARY') + py: s = b'\x66\x6f\x6f' + js: s = Buffer("\x66\x6f\x6f", 'binary') + - cd: r.binary(s) + ot: s + + - cd: r.expr("foo").coerce_to("binary").to_json_string() + ot: '{"$reql_type$":"BINARY","data":"Zm9v"}' diff --git a/ext/librethinkdbxx/test/upstream/limits.yaml b/ext/librethinkdbxx/test/upstream/limits.yaml new file mode 100644 index 00000000..41316df2 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/limits.yaml @@ -0,0 +1,128 @@ +desc: Tests array limit variations +table_variable_name: tbl +tests: + + # test simplistic array limits + - cd: r.expr([1,1,1,1]).union([1, 1, 1, 1]) + runopts: + array_limit: 8 + ot: [1,1,1,1,1,1,1,1] + - cd: r.expr([1,2,3,4]).union([5, 6, 7, 8]) + runopts: + array_limit: 4 + ot: err("ReqlResourceLimitError", "Array over size limit `4`.", [0]) + + # test array limits on query creation + - cd: r.expr([1,2,3,4,5,6,7,8]) + runopts: + array_limit: 4 + ot: err("ReqlResourceLimitError", "Array over size limit `4`.", [0]) + + # test bizarre array limits + - cd: r.expr([1,2,3,4,5,6,7,8]) + runopts: + array_limit: -1 + ot: err("ReqlQueryLogicError", "Illegal array size limit `-1`. (Must be >= 1.)", []) + + - cd: r.expr([1,2,3,4,5,6,7,8]) + runopts: + array_limit: 0 + ot: err("ReqlQueryLogicError", "Illegal array size limit `0`. (Must be >= 1.)", []) + + # make enormous > 100,000 element array + - def: ten_l = r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - def: + js: ten_f = function(l) { return ten_l } + py: ten_f = lambda l:list(range(1,11)) + - def: + js: huge_l = r.expr(ten_l).concatMap(ten_f).concatMap(ten_f).concatMap(ten_f).concatMap(ten_f) + py: huge_l = r.expr(ten_l).concat_map(ten_f).concat_map(ten_f).concat_map(ten_f).concat_map(ten_f) + rb: huge_l = r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l} + - cd: huge_l.append(1).count() + runopts: + array_limit: 100001 + ot: 100001 + + # attempt to insert enormous array + - cd: tbl.insert({'id':0, 'array':huge_l.append(1)}) + runopts: + array_limit: 100001 + ot: partial({'errors':1, 'first_error':"Array too large for disk writes (limit 100,000 elements)."}) + + - cd: tbl.get(0) + runopts: + array_limit: 100001 + ot: (null) + + # attempt to read array that violates limit from disk + - cd: tbl.insert({'id':1, 'array':ten_l}) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1}) + - cd: tbl.get(1) + runopts: + array_limit: 4 + ot: ({'array':[1,2,3,4,5,6,7,8,9,10],'id':1}) + + + # Test that the changefeed queue size actually causes changes to be sent early. + - cd: tbl.delete().get_field('deleted') + ot: 1 + + - cd: c = tbl.changes({squash:1000000, changefeed_queue_size:10}) + py: c = tbl.changes(squash=1000000, changefeed_queue_size=10) + + - cd: tbl.insert([{'id':0}, {'id':1}, {'id':2}, {'id':3}, {'id':4}, {'id':5}, {'id':6}]).get_field('inserted') + ot: 7 + - py: fetch(c, 7) + rb: fetch(c, 7) + ot: bag([{'old_val':null, 'new_val':{'id':0}}, + {'old_val':null, 'new_val':{'id':1}}, + {'old_val':null, 'new_val':{'id':2}}, + {'old_val':null, 'new_val':{'id':3}}, + {'old_val':null, 'new_val':{'id':4}}, + {'old_val':null, 'new_val':{'id':5}}, + {'old_val':null, 'new_val':{'id':6}}]) + + - cd: tbl.insert([{'id':7}, {'id':8}, {'id':9}, {'id':10}, {'id':11}, {'id':12}, {'id':13}]).get_field('inserted') + ot: 7 + - py: fetch(c, 7) + rb: fetch(c, 7) + ot: bag([{'old_val':null, 'new_val':{'id':7}}, + {'old_val':null, 'new_val':{'id':8}}, + {'old_val':null, 'new_val':{'id':9}}, + {'old_val':null, 'new_val':{'id':10}}, + {'old_val':null, 'new_val':{'id':11}}, + {'old_val':null, 'new_val':{'id':12}}, + {'old_val':null, 'new_val':{'id':13}}]) + + - cd: tbl.delete().get_field('deleted') + ot: 14 + + - cd: c2 = tbl.changes({squash:1000000}) + py: c2 = tbl.changes(squash=1000000) + runopts: + changefeed_queue_size: 10 + + + - cd: tbl.insert([{'id':0}, {'id':1}, {'id':2}, {'id':3}, {'id':4}, {'id':5}, {'id':6}]).get_field('inserted') + ot: 7 + - py: fetch(c2, 7) + rb: fetch(c2, 7) + ot: bag([{'old_val':null, 'new_val':{'id':0}}, + {'old_val':null, 'new_val':{'id':1}}, + {'old_val':null, 'new_val':{'id':2}}, + {'old_val':null, 'new_val':{'id':3}}, + {'old_val':null, 'new_val':{'id':4}}, + {'old_val':null, 'new_val':{'id':5}}, + {'old_val':null, 'new_val':{'id':6}}]) + + - cd: tbl.insert([{'id':7}, {'id':8}, {'id':9}, {'id':10}, {'id':11}, {'id':12}, {'id':13}]).get_field('inserted') + ot: 7 + - py: fetch(c2, 7) + rb: fetch(c2, 7) + ot: bag([{'old_val':null, 'new_val':{'id':7}}, + {'old_val':null, 'new_val':{'id':8}}, + {'old_val':null, 'new_val':{'id':9}}, + {'old_val':null, 'new_val':{'id':10}}, + {'old_val':null, 'new_val':{'id':11}}, + {'old_val':null, 'new_val':{'id':12}}, + {'old_val':null, 'new_val':{'id':13}}]) diff --git a/ext/librethinkdbxx/test/upstream/match.yaml b/ext/librethinkdbxx/test/upstream/match.yaml new file mode 100644 index 00000000..55e33801 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/match.yaml @@ -0,0 +1,38 @@ +desc: Tests for match +table_variable_name: tbl +tests: + - cd: r.expr("abcdefg").match("a(b.e)|b(c.e)") + ot: ({'str':'bcde','groups':[null,{'start':2,'str':'cde','end':5}],'start':1,'end':5}) + - cd: r.expr("abcdefg").match("a(b.e)|B(c.e)") + ot: (null) + - cd: r.expr("abcdefg").match("(?i)a(b.e)|B(c.e)") + ot: ({'str':'bcde','groups':[null,{'start':2,'str':'cde','end':5}],'start':1,'end':5}) + + - cd: r.expr(["aba", "aca", "ada", "aea"]).filter{|row| row.match("a(.)a")[:groups][0][:str].match("[cd]")} + py: r.expr(["aba", "aca", "ada", "aea"]).filter(lambda row:row.match("a(.)a")['groups'][0]['str'].match("[cd]")) + js: r.expr(["aba", "aca", "ada", "aea"]).filter(function(row){return row.match("a(.)a")('groups').nth(0)('str').match("[cd]")}) + ot: (["aca", "ada"]) + + - cd: tbl.insert([{'id':0,'a':'abc'},{'id':1,'a':'ab'},{'id':2,'a':'bc'}]) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':3}) + + - cd: tbl.filter{|row| row['a'].match('b')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('b')).order_by('id') + js: tbl.filter(function(row){return row('a').match('b')}).order_by('id') + ot: ([{'id':0,'a':'abc'},{'id':1,'a':'ab'},{'id':2,'a':'bc'}]) + - cd: tbl.filter{|row| row['a'].match('ab')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('ab')).order_by('id') + js: tbl.filter(function(row){return row('a').match('ab')}).order_by('id') + ot: ([{'id':0,'a':'abc'},{'id':1,'a':'ab'}]) + - cd: tbl.filter{|row| row['a'].match('ab$')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('ab$')).order_by('id') + js: tbl.filter(function(row){return row('a').match('ab$')}).order_by('id') + ot: ([{'id':1,'a':'ab'}]) + - cd: tbl.filter{|row| row['a'].match('^b$')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('^b$')).order_by('id') + js: tbl.filter(function(row){return row('a').match('^b$')}).order_by('id') + ot: ([]) + + - cd: r.expr("").match("ab\\9") + ot: | + err("ReqlQueryLogicError", "Error in regexp `ab\\9` (portion `\\9`): invalid escape sequence: \\9", []) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/add.yaml b/ext/librethinkdbxx/test/upstream/math_logic/add.yaml new file mode 100644 index 00000000..e956aaa7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/add.yaml @@ -0,0 +1,65 @@ +desc: Tests for basic usage of the add operation +tests: + - cd: r.add(1, 1) + ot: 2 + + - js: r(1).add(1) + py: + - r.expr(1) + 1 + - 1 + r.expr(1) + - r.expr(1).add(1) + rb: + - r(1) + 1 + - r(1).add(1) + ot: 2 + + - py: r.expr(-1) + 1 + js: r(-1).add(1) + rb: (r -1) + 1 + ot: 0 + + - py: r.expr(1.75) + 8.5 + js: r(1.75).add(8.5) + rb: (r 1.75) + 8.5 + ot: 10.25 + + # Add is polymorphic on strings + - py: r.expr('') + '' + js: r('').add('') + rb: (r '') + '' + ot: '' + + - py: r.expr('abc') + 'def' + js: r('abc').add('def') + rb: (r 'abc') + 'def' + ot: 'abcdef' + + # Add is polymorphic on arrays + - cd: r.expr([1,2]) + [3] + [4,5] + [6,7,8] + js: r([1,2]).add([3]).add([4,5]).add([6,7,8]) + ot: [1,2,3,4,5,6,7,8] + + # All arithmetic operations (except mod) actually support arbitrary arguments + # but this feature can't be accessed in Python because it's operators are binary + - js: r(1).add(2,3,4,5) + ot: 15 + + - js: r('a').add('b', 'c', 'd') + ot: 'abcd' + + # Type errors + - cd: r(1).add('a') + py: r.expr(1) + 'a' + rb: r(1) + 'a' + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", [1]) + + - cd: r('a').add(1) + py: r.expr('a') + 1 + rb: r('a') + 1 + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [1]) + + - cd: r([]).add(1) + py: r.expr([]) + 1 + rb: r([]) + 1 + ot: err("ReqlQueryLogicError", "Expected type ARRAY but found NUMBER.", [1]) + diff --git a/ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml b/ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml new file mode 100644 index 00000000..f1181f87 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml @@ -0,0 +1,46 @@ +desc: Test named aliases for math and logic operators +tests: + + - cd: + - r.expr(0).add(1) + - r.add(0, 1) + - r.expr(2).sub(1) + - r.sub(2, 1) + - r.expr(2).div(2) + - r.div(2, 2) + - r.expr(1).mul(1) + - r.mul(1, 1) + - r.expr(1).mod(2) + - r.mod(1, 2) + ot: 1 + + - cd: + - r.expr(True).and(True) + - r.expr(True).or(True) + - r.and(True, True) + - r.or(True, True) + - r.expr(False).not() + - r.not(False) + py: + - r.expr(True).and_(True) + - r.expr(True).or_(True) + - r.and_(True, True) + - r.or_(True, True) + - r.expr(False).not_() + - r.not_(False) + ot: True + + - cd: + - r.expr(1).eq(1) + - r.expr(1).ne(2) + - r.expr(1).lt(2) + - r.expr(1).gt(0) + - r.expr(1).le(1) + - r.expr(1).ge(1) + - r.eq(1, 1) + - r.ne(1, 2) + - r.lt(1, 2) + - r.gt(1, 0) + - r.le(1, 1) + - r.ge(1, 1) + ot: True diff --git a/ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml b/ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml new file mode 100644 index 00000000..3fed54a9 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml @@ -0,0 +1,477 @@ +desc: Tests of comparison operators +tests: + + ### Numeric comparisons + + ## basic < + + - cd: r(1).lt(2) + py: + - r.expr(1) < 2 + - 1 < r.expr(2) + - r.expr(1).lt(2) + rb: + - r(1) < 2 + - r(1).lt(2) + - 1 < r(2) + ot: true + - cd: r(3).lt(2) + py: r.expr(3) < 2 + rb: r(3) < 2 + ot: false + - py: r.expr(2) < 2 + js: r(2).lt(2) + rb: r(2) < 2 + ot: false + + # All Comparisons can take an arbitrary number of arguments though + # the functionality is only available in JS at the moment + - js: r(1).lt(2, 3, 4) + ot: true + - js: r(1).lt(2, 3, 2) + ot: false + + ## basic > + + - cd: r(1).gt(2) + py: + - r.expr(1) > 2 + - 1 > r.expr(2) + - r.expr(1).gt(2) + rb: + - r(1) > 2 + - r(1).gt(2) + ot: false + - py: r.expr(3) > 2 + js: r(3).gt(2) + rb: r(3) > 2 + ot: true + - py: r.expr(2) > 2 + js: r(2).gt(2) + rb: r(2) > 2 + ot: false + + - js: r(4).gt(3, 2, 1) + ot: true + - js: r(4).gt(3, 2, 3) + ot: false + + ## basic == + + - cd: r(1).eq(2) + py: + - r.expr(1) == 2 + - 1 == r.expr(2) + - r.expr(1).eq(2) + rb: r(1).eq 2 + ot: false + - py: r.expr(3) == 2 + js: r(3).eq(2) + rb: r(3).eq 2 + ot: false + - py: r.expr(2) == 2 + js: r(2).eq(2) + rb: r(2).eq 2 + ot: true + + - js: r(1).eq(1, 1, 1) + ot: true + - js: r(1).eq(1, 2, 1) + ot: false + + ## basic != + + - cd: r(1).ne(2) + py: + - r.expr(1) != 2 + - 1 != r.expr(2) + - r.expr(1).ne(2) + rb: r(1).ne 2 + ot: true + - py: r.expr(3) != 2 + js: r(3).ne(2) + rb: r(3).ne 2 + ot: true + - py: r.expr(2) != 2 + js: r(2).ne(2) + rb: r(2).ne 2 + ot: false + + - js: r(1).ne(3, 2, 4) + ot: true + - js: r(1).ne(3, 2, 3) + ot: true + + ## basic <= + + - js: r(1).le(2) + py: + - r.expr(1) <= 2 + - 1 <= r.expr(2) + - r.expr(1).le(2) + rb: + - r(1) <= 2 + - r(1).le(2) + ot: true + - py: r.expr(3) <= 2 + js: r(3).le(2) + rb: r(3) <= 2 + ot: false + - py: r.expr(2) <= 2 + js: r(2).le(2) + rb: r(2) <= 2 + ot: true + + - js: r(1).le(1, 2, 2) + ot: true + - js: r(1).le(1, 3, 2) + ot: false + + ## basic >= + + - cd: r(1).ge(2) + py: + - r.expr(1) >= 2 + - 1 >= r.expr(2) + - r.expr(1).ge(2) + rb: + - r(1) >= 2 + - r(1).ge(2) + ot: false + - py: r.expr(3) >= 2 + js: r(3).ge(2) + rb: r(3) >= 2 + ot: true + - py: r.expr(2) >= 2 + js: r(2).ge(2) + rb: r(2) >= 2 + ot: true + + - js: r(4).ge(4, 2, 2) + ot: true + - js: r(4).ge(4, 2, 3) + ot: false + + # Comparisons for NULL + - cd: r(null).eq(null) + py: + - r.expr(null) == null + - null == r.expr(null) + ot: true + + - cd: r(null).lt(null) + py: + - r.expr(null) < null + - null < r.expr(null) + - r.expr(null).lt(null) + rb: r(null) < null + ot: false + + - cd: r(null).gt(null) + py: + - r.expr(null) > null + - null > r.expr(null) + - r.expr(null).gt(null) + rb: r(null) > null + ot: false + + # Comparisons for STRING + # STRING comparison should be lexicagraphical + - py: r.expr('a') == 'a' + cd: r('a').eq('a') + ot: true + + - py: r.expr('a') == 'aa' + cd: r('a').eq('aa') + ot: false + + - py: r.expr('a') < 'aa' + cd: r('a').lt('aa') + ot: true + + - py: r.expr('a') < 'bb' + cd: r('a').lt('bb') + ot: true + + - py: r.expr('bb') > 'a' + cd: r('bb').gt('a') + ot: true + + - py: r.expr('abcdef') < 'abcdeg' + cd: r('abcdef').lt('abcdeg') + ot: true + + - py: r.expr('abcdefg') > 'abcdeg' + cd: r('abcdefg').gt('abcdeg') + ot: false + + - py: r.expr('A quick brown fox') > 'A quick brawn fox' + js: r('A quick brown fox').gt('A quick brawn fox') + rb: r('A quick brown fox') > 'A quick brawn fox' + ot: true + + # Comparisons for ARRAY + # Also lexicographical + + - py: r.expr([1]) < [2] + js: r([1]).lt([2]) + rb: r([1]) < [2] + ot: true + + - py: r.expr([1]) > [2] + js: r([1]).gt([2]) + rb: r([1]) > [2] + ot: false + + - py: r.expr([1, 0]) < [2] + js: r([1, 0]).lt([2]) + rb: r([1, 0]) < [2] + ot: true + + - py: r.expr([1, 0]) < [1] + js: r([1, 0]).lt([1]) + rb: r([1, 0]) < [1] + ot: false + + - py: r.expr([1, 0]) > [0] + js: r([1, 0]).gt([0]) + rb: r([1, 0]) > [0] + ot: true + + - py: r.expr([1, 'a']) < [1, 'b'] + js: r([1, 'a']).lt([1, 'b']) + rb: r([1, 'a']) < [1, 'b'] + ot: true + + - py: r.expr([0, 'z']) < [1, 'b'] + js: r([0, 'z']).lt([1, 'b']) + rb: r([0, 'z']) < [1, 'b'] + ot: true + + - py: r.expr([1, 1, 1]) < [1, 0, 2] + js: r([1, 1, 1]).lt([1, 0, 2]) + rb: r([1, 1, 1]) < [1, 0, 2] + ot: false + + - py: r.expr([1, 0, 2]) < [1, 1, 1] + js: r([1, 0, 2]).lt([1, 1, 1]) + rb: r([1, 0, 2]) < [1, 1, 1] + ot: true + + # Comparisons for OBJECT + + - py: r.expr({'a':0}) == {'a':0} + cd: r({'a':0}).eq({'a':0}) + ot: true + + - py: r.expr({'a':0, 'b':1}) == {'b':1, 'a':0} + cd: r({'a':0, 'b':1}).eq({'b':1, 'a':0}) + ot: true + + - py: r.expr({'a':0, 'b':1, 'c':2}) == {'b':1, 'a':0} + cd: r({'a':0, 'b':1, 'c':2}).eq({'b':1, 'a':0}) + ot: false + + - py: r.expr({'a':0, 'b':1}) == {'b':1, 'a':0, 'c':2} + cd: r({'a':0, 'b':1}).eq({'b':1, 'a':0, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'd':2}) == {'b':1, 'a':0, 'c':2} + cd: r({'a':0, 'b':1, 'd':2}).eq({'b':1, 'a':0, 'c':2}) + ot: false + + - py: r.expr({'a':0}) < {'b':0} + cd: r({'a':0}).lt({'b':0}) + ot: true + + - py: r.expr({'a':1}) < {'b':0} + cd: r({'a':1}).lt({'b':0}) + ot: true + + - py: r.expr({'b':1}) < {'b':0} + cd: r({'b':1}).lt({'b':0}) + ot: false + + - py: r.expr({'b':1}) < {'a':0} + cd: r({'b':1}).lt({'a':0}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'c':2}) < {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'b':1, 'c':2}).lt({'a':0, 'b':1, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'c':2, 'd':3}) < {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'b':1, 'c':2, 'd':3}).lt({'a':0, 'b':1, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'c':2}) < {'a':0, 'b':1, 'c':2, 'd':3} + cd: r({'a':0, 'b':1, 'c':2}).lt({'a':0, 'b':1, 'c':2, 'd':3}) + ot: true + + - py: r.expr({'a':0, 'c':2}) < {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'c':2}).lt({'a':0, 'b':1, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'c':2}) > {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'c':2}).gt({'a':0, 'b':1, 'c':2}) + ot: true + + # Comparisons across types + # RQL primitive types compare as if mapped to the following numbers + # MINVAL: 0 + # ARRAY: 1 + # BINARY: 2 + # BOOLEAN: 3 + # NULL: 4 + # NUMBER: 5 + # OBJECT: 6 + # STRING: 7 + # MAXVAL: 8 + - def: + py: everything = r.expr([[],r.now(),r.binary(b"\x00"),false,null,-5,{},"a",r.maxval]) + js: everything = r.expr([[],r.now(),r.binary(Buffer("\x00")),false,null,-5,{},"a",r.maxval]) + rb: everything = r.expr([[],r.now(),r.binary("\x00"),false,null,-5,{},"a",r.maxval]) + + - js: r.and(r.args(everything.map(r.lt(r.minval, r.row)))) + py: r.and_(r.args(everything.map(r.lt(r.minval, r.row)))) + rb: r.and(r.args(everything.map{|x| r.lt(r.minval, x)})) + ot: true + + - js: r.or(r.args(everything.map(r.gt(r.minval, r.row)))) + py: r.or_(r.args(everything.map(r.gt(r.minval, r.row)))) + rb: r.or(r.args(everything.map{|x| r.gt(r.minval, x)})) + ot: false + + - cd: r.eq(r.minval, r.minval) + ot: true + + - py: r.expr([]) < True + js: r([]).lt(true) + rb: r([]) < true + ot: true + + - py: r.expr([1,2]) < False + js: r([1,2]).lt(false) + rb: r([1,2]) < false + ot: true + + - py: r.expr(False) < [] + js: r(false).lt([]) + rb: r(false) < [] + ot: false + + - py: r.expr([]) < r.binary(b"\xAE") + js: r([]).lt(r.binary(Buffer("\x00"))) + rb: r([]) < r.binary("") + ot: true + + - py: r.expr([1,2]) < r.binary(b"\xAE") + js: r([1,2]).lt(r.binary(Buffer("\x00"))) + rb: r([1,2]) < r.binary("") + ot: true + + - py: True < r.expr(null) + js: r(true).lt(null) + rb: r(true) < null + ot: true + + - py: r.expr(null) > [] + js: r(null).gt([]) + rb: r(null) > [] + ot: true + + - py: r.expr(null) < 12 + js: r(null).lt(12) + rb: r(null) < 12 + ot: true + + - py: r.expr(null) < -2 + js: r(null).lt(-2) + rb: r(null) < -2 + ot: true + + - py: r.expr(-12) < {} + js: r(-12).lt({}) + rb: r(-12) < {} + ot: true + + - py: r.expr(100) < {'a':-12} + js: r(100).lt({a:-12}) + rb: r(100) < { :a => 12 } + ot: true + + - py: r.expr(r.binary(b"\xAE")) < 12 + js: r(r.binary(Buffer("\x00"))).lt(12) + rb: r(r.binary("")) < 12 + ot: false + + - py: r.binary(b"0xAE") < 'abc' + js: r.binary(Buffer("0x00")).lt('abc') + rb: r.binary("") < 'abc' + ot: true + + - py: r.binary(b"0xAE") > r.now() + js: r.binary(Buffer("0x00")).gt(r.now()) + rb: r.binary("") > r.now() + ot: false + + - cd: r.now() > 12 + js: r.now().gt(12) + ot: true + + - cd: r.now() > 'abc' + js: r.now().gt('abc') + ot: false + + - py: r.expr("abc") > {'a':-12} + js: r('abc').gt({a:-12}) + rb: r('abc') > { :a => 12 } + ot: true + + - py: r.expr("abc") > {'abc':'abc'} + js: r('abc').gt({abc:'abc'}) + rb: r('abc') > { :abc => 'abc' } + ot: true + + - py: r.expr('zzz') > 128 + js: r('zzz').gt(128) + rb: r('zzz') > 128 + ot: true + + - py: r.expr('zzz') > {} + js: r('zzz').gt({}) + rb: r('zzz') > {} + ot: true + + - py: 'zzz' > r.expr(-152) + js: r('zzz').gt(-152) + rb: r('zzz') > -152 + ot: true + + - py: 'zzz' > r.expr(null) + js: r('zzz').gt(null) + rb: r('zzz') > null + ot: true + + - py: 'zzz' > r.expr([]) + js: r('zzz').gt([]) + rb: r('zzz') > [] + ot: true + + - def: + rb: everything2 = r.expr([r.minval,[],r.now(),r.binary("\x00"),false,null,-5,{},"a"]) + py: everything2 = r.expr([r.minval,[],r.now(),r.binary(b"\x00"),false,null,-5,{},"a"]) + js: everything2 = r.expr([r.minval,[],r.now(),r.binary(Buffer("\x00")),false,null,-5,{},"a"]) + + - js: r.and(r.args(everything2.map(r.gt(r.maxval, r.row)))) + py: r.and_(r.args(everything2.map(r.gt(r.maxval, r.row)))) + rb: r.and(r.args(everything2.map{|x| r.gt(r.maxval, x)})) + ot: true + + - js: r.or(r.args(everything2.map(r.lt(r.maxval, r.row)))) + py: r.or_(r.args(everything2.map(r.lt(r.maxval, r.row)))) + rb: r.or(r.args(everything2.map{|x| r.lt(r.maxval, x)})) + ot: false + + - cd: r.eq(r.maxval, r.maxval) + ot: true diff --git a/ext/librethinkdbxx/test/upstream/math_logic/div.yaml b/ext/librethinkdbxx/test/upstream/math_logic/div.yaml new file mode 100644 index 00000000..ffca4156 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/div.yaml @@ -0,0 +1,52 @@ +desc: Tests for the basic usage of the division operation +tests: + + - cd: r(4).div(2) + py: + - r.expr(4) / 2 + - 4 / r.expr(2) + - r.expr(4).div(2) + rb: + - (r 4) / 2 + - r(4).div 2 + - 4 / r(2) + ot: 2 + + - py: r.expr(-1) / -2 + js: r(-1).div(-2) + rb: (r -1) / -2 + ot: 0.5 + + - py: r.expr(4.9) / 0.7 + js: r(4.9).div(0.7) + rb: (r 4.9) / 0.7 + ot: 4.9 / 0.7 + + - cd: r.expr(1).div(2,3,4,5) + ot: 1.0/120 + + # Divide by zero test + - cd: + - r(1).div(0) + - r(2.0).div(0) + - r(3).div(0.0) + - r(4.0).div(0.0) + - r(0).div(0) + - r(0.0).div(0.0) + py: + - r.expr(1) / 0 + - r.expr(2.0) / 0 + - r.expr(3) / 0.0 + - r.expr(4.0) / 0.0 + - r.expr(0) / 0 + - r.expr(0.0) / 0.0 + ot: err('ReqlQueryLogicError', 'Cannot divide by zero.', [1]) + + # Type errors + - py: r.expr('a') / 0.8 + cd: r('a').div(0.8) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - py: r.expr(1) / 'a' + cd: r(1).div('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml b/ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml new file mode 100644 index 00000000..d32526f5 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml @@ -0,0 +1,114 @@ +desc: tests for `floor`, `ceil`, and `round`, tests inspired by the Python test suite +tests: + - cd: r.floor(1.0).type_of() + ot: "NUMBER" + - cd: r.floor(1.0) + ot: 1.0 + - cd: r.expr(1.0).floor() + ot: 1.0 + + - cd: r.floor(0.5) + ot: 0.0 + - cd: r.floor(1.0) + ot: 1.0 + - cd: r.floor(1.5) + ot: 1.0 + - cd: r.floor(-0.5) + ot: -1.0 + - cd: r.floor(-1.0) + ot: -1.0 + - cd: r.floor(-1.5) + ot: -2.0 + + - cd: r.expr('X').floor() + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + + - cd: r.ceil(1.0).type_of() + ot: "NUMBER" + - cd: r.ceil(1.0) + ot: 1.0 + - cd: r.expr(1.0).ceil() + ot: 1.0 + + - cd: r.ceil(0.5) + ot: 1.0 + - cd: r.ceil(1.0) + ot: 1.0 + - cd: r.ceil(1.5) + ot: 2.0 + - cd: r.ceil(-0.5) + ot: 0.0 + - cd: r.ceil(-1.0) + ot: -1.0 + - cd: r.ceil(-1.5) + ot: -1.0 + + - cd: r.expr('X').ceil() + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + + - cd: r.round(1.0).type_of() + ot: "NUMBER" + - cd: r.round(1.0) + ot: 1.0 + - cd: r.expr(1.0).round() + ot: 1.0 + + - cd: r.round(0.5) + ot: 1.0 + - cd: r.round(-0.5) + ot: -1.0 + + - cd: r.round(0.0) + ot: 0.0 + - cd: r.round(1.0) + ot: 1.0 + - cd: r.round(10.0) + ot: 10.0 + - cd: r.round(1000000000.0) + ot: 1000000000.0 + - cd: r.round(1e20) + ot: 1e20 + + - cd: r.round(-1.0) + ot: -1.0 + - cd: r.round(-10.0) + ot: -10.0 + - cd: r.round(-1000000000.0) + ot: -1000000000.0 + - cd: r.round(-1e20) + ot: -1e20 + + - cd: r.round(0.1) + ot: 0.0 + - cd: r.round(1.1) + ot: 1.0 + - cd: r.round(10.1) + ot: 10.0 + - cd: r.round(1000000000.1) + ot: 1000000000.0 + + - cd: r.round(-1.1) + ot: -1.0 + - cd: r.round(-10.1) + ot: -10.0 + - cd: r.round(-1000000000.1) + ot: -1000000000.0 + + - cd: r.round(0.9) + ot: 1.0 + - cd: r.round(9.9) + ot: 10.0 + - cd: r.round(999999999.9) + ot: 1000000000.0 + + - cd: r.round(-0.9) + ot: -1.0 + - cd: r.round(-9.9) + ot: -10.0 + - cd: r.round(-999999999.9) + ot: -1000000000.0 + + - cd: r.expr('X').round() + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/logic.yaml b/ext/librethinkdbxx/test/upstream/math_logic/logic.yaml new file mode 100644 index 00000000..84dbc574 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/logic.yaml @@ -0,0 +1,169 @@ +desc: These tests are aimed at &&, ||, and ! +tests: + + ## basic operator usage + + # Python overloads '&' for 'and' + - py: + - r.expr(true) & true + - true & r.expr(true) + - r.and_(true,true) + - r.expr(true).and_(true) + rb: + - r(true) & true + - r(true) & r(true) + - r.and(true,true) + - r(true).and(true) + js: + - r.and(true,true) + - r(true).and(true) + ot: true + - py: + - r.expr(true) & false + - r.expr(false) & false + - true & r.expr(false) + - false & r.expr(false) + - r.and_(true,false) + - r.and_(false,false) + - r.expr(true).and_(false) + - r.expr(false).and_(false) + rb: + - r(true) & false + - r(false) & false + - r(true) & r(false) + - r(false) & r(false) + - r.and(true,false) + - r.and(false,false) + - r(true).and(false) + - r(false).and(false) + js: + - r.and(true,false) + - r.and(false,false) + - r(true).and(false) + - r(false).and(false) + ot: false + + # Python overloads '|' for 'or' + - py: + - r.expr(true) | true + - r.expr(true) | false + - true | r.expr(true) + - true | r.expr(false) + - r.or_(true,true) + - r.or_(true,false) + - r.expr(true).or_(true) + - r.expr(true).or_(false) + rb: + - r(true) | true + - r(true) | false + - r(true) | r(true) + - r(true) | r(false) + - r.or(true,true) + - r.or(true,false) + - r(true).or(true) + - r(true).or(false) + js: + - r.or(true,true) + - r.or(true,false) + - r(true).or(true) + - r(true).or(false) + ot: true + - py: + - r.expr(false) | false + - false | r.expr(false) + - r.and_(false,false) + - r.expr(false).and_(false) + rb: + - r(false) | false + - r(false) | r(false) + - r.and(false,false) + - r(false).and(false) + js: + - r.and(false,false) + - r(false).and(false) + ot: false + + # Python overloads '~' for 'not' + - py: + - ~r.expr(True) + - r.not_(True) + cd: r(true).not() + ot: false + - py: + - ~r.expr(False) + - r.not_(False) + cd: r(false).not() + ot: true + - py: r.expr(True).not_() + cd: r(true).not() + ot: false + - py: r.expr(False).not_() + cd: r(false).not() + ot: true + + ## DeMorgan's rules! + + - py: + - ~r.and_(True, True) == r.or_(~r.expr(True), ~r.expr(True)) + - ~r.and_(True, False) == r.or_(~r.expr(True), ~r.expr(False)) + - ~r.and_(False, False) == r.or_(~r.expr(False), ~r.expr(False)) + - ~r.and_(False, True) == r.or_(~r.expr(False), ~r.expr(True)) + cd: + - r(true).and(true).not().eq(r(true).not().or(r(true).not())) + - r(true).and(false).not().eq(r(true).not().or(r(false).not())) + - r(false).and(false).not().eq(r(false).not().or(r(false).not())) + - r(false).and(true).not().eq(r(false).not().or(r(true).not())) + ot: true + + # Test multiple arguments to 'and' and 'or' + - cd: r(true).and(true, true, true, true) + py: r.and_(True, True, True, True, True) + ot: true + - cd: r(true).and(true, true, false, true) + py: r.and_(True, True, True, False, True) + ot: false + - cd: r(true).and(false, true, false, true) + py: r.and_(True, False, True, False, True) + ot: false + - cd: r(false).or(false, false, false, false) + py: r.or_(False, False, False, False, False) + ot: false + - cd: r(false).or(false, false, true, false) + py: r.or_(False, False, False, True, False) + ot: true + - cd: r(false).or(true, false, true, false) + py: r.or_(False, True, False, True, False) + ot: true + + # Test that precedence errors are detected + - js: r.expr(r.expr('a')('b')).default(2) + py: r.expr(r.expr('a')['b']).default(2) + rb: r(r('a')['b']).default(2) + ot: err("ReqlQueryLogicError", "Cannot perform bracket on a non-object non-sequence `\"a\"`.", []) + - py: r.expr(r.expr(True) & r.expr(False) == r.expr(False) | r.expr(True)) + ot: err("ReqlDriverCompileError", "Calling '==' on result of infix bitwise operator:", []) + - py: r.expr(r.and_(True, False) == r.or_(False, True)) + ot: False + - rb: r.expr(r.expr(True) & r.expr(False) >= r.expr(False) | r.expr(True)) + py: r.expr(r.expr(True) & r.expr(False) >= r.expr(False) | r.expr(True)) + ot: err("ReqlDriverCompileError", "Calling '>=' on result of infix bitwise operator:", []) + - cd: r.expr(r.and(True, False) >= r.or(False, True)) + py: r.expr(r.and_(True, False) >= r.or_(False, True)) + ot: False + + # Type errors + - py: r.expr(1) & True + cd: r(1).and(true) + ot: true + + - py: r.expr(False) | 'str' + cd: r(false).or('str') + ot: ("str") + + - py: ~r.expr(1) + cd: r(1).not() + ot: false + + - py: ~r.expr(null) + cd: r(null).not() + ot: true diff --git a/ext/librethinkdbxx/test/upstream/math_logic/math.yaml b/ext/librethinkdbxx/test/upstream/math_logic/math.yaml new file mode 100644 index 00000000..5010ac51 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/math.yaml @@ -0,0 +1,11 @@ +desc: Tests of nested arithmetic expressions +tests: + + - py: (((4 + 2 * (r.expr(26) % 18)) / 5) - 3) + js: r(4).add(r(2).mul(r(26).mod(18))).div(5).sub(3) + rb: + - ((((r 4) + (r 2) * ((r 26) % 18)) / 5) -3) + - (((4 + 2 * ((r 26) % 18)) / 5) -3) + ot: 1 + + # Prescedence set by host langauge diff --git a/ext/librethinkdbxx/test/upstream/math_logic/mod.yaml b/ext/librethinkdbxx/test/upstream/math_logic/mod.yaml new file mode 100644 index 00000000..80a0a32c --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/mod.yaml @@ -0,0 +1,34 @@ +desc: Tests for the basic usage of the mod operation +tests: + + - cd: r.expr(10).mod(3) + py: + - r.expr(10) % 3 + - 10 % r.expr(3) + - r.expr(10).mod(3) + rb: + - (r 10) % 3 + - r(10).mod 3 + - 10 % (r 3) + ot: 1 + + - cd: r.expr(-10).mod(-3) + py: r.expr(-10) % -3 + rb: (r -10) % -3 + ot: -1 + + # Type errors + - cd: r.expr(4).mod('a') + py: r.expr(4) % 'a' + rb: r(4) % 'a' + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) + + - cd: r.expr('a').mod(1) + py: r.expr('a') % 1 + rb: r('a') % 1 + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - cd: r.expr('a').mod('b') + py: r.expr('a') % 'b' + rb: r('a') % 'b' + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/mul.yaml b/ext/librethinkdbxx/test/upstream/math_logic/mul.yaml new file mode 100644 index 00000000..ba9b3f9d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/mul.yaml @@ -0,0 +1,60 @@ +desc: Tests for the basic usage of the multiplication operation +tests: + + - cd: r.expr(1).mul(2) + py: + - r.expr(1) * 2 + - 1 * r.expr(2) + - r.expr(1).mul(2) + rb: + - (r 1) * 2 + - r(1).mul(2) + - 1 * (r 2) + ot: 2 + + - py: r.expr(-1) * -1 + js: r(-1).mul(-1) + rb: (r -1) * -1 + ot: 1 + + - cd: r.expr(1.5).mul(4.5) + py: r.expr(1.5) * 4.5 + rb: (r 1.5) * 4.5 + ot: 6.75 + + - py: r.expr([1,2,3]) * 3 + js: r([1,2,3]).mul(3) + rb: (r [1,2,3]) * 3 + ot: [1,2,3,1,2,3,1,2,3] + + - cd: r.expr(1).mul(2,3,4,5) + ot: 120 + + - cd: r(2).mul([1,2,3], 2) + py: # this form does not work in Python + ot: [1,2,3,1,2,3,1,2,3,1,2,3] + + - cd: r([1,2,3]).mul(2, 2) + py: # this form does not work in Python + ot: [1,2,3,1,2,3,1,2,3,1,2,3] + + - cd: r(2).mul(2, [1,2,3]) + py: # this form does not work in Python + ot: [1,2,3,1,2,3,1,2,3,1,2,3] + + # Type errors + - py: r.expr('a') * 0.8 + cd: r('a').mul(0.8) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - py: r.expr(1) * 'a' + cd: r(1).mul('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) + + - py: r.expr('b') * 'a' + cd: r('b').mul('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - py: r.expr([]) * 1.5 + cd: r([]).mul(1.5) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/sub.yaml b/ext/librethinkdbxx/test/upstream/math_logic/sub.yaml new file mode 100644 index 00000000..2c91bdaa --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/sub.yaml @@ -0,0 +1,37 @@ +desc: Tests for basic usage of the subtraction operation +tests: + + - cd: r.expr(1).sub(1) + py: + - r.expr(1) - 1 + - 1 - r.expr(1) + - r.expr(1).sub(1) + rb: + - (r 1) - 1 + - 1 - (r 1) + - r(1).sub(1) + - r.expr(1).sub(1) + ot: 0 + + - cd: r.expr(-1).sub(1) + py: r.expr(-1) - 1 + rb: (r -1) - 1 + ot: -2 + + - cd: r.expr(1.75).sub(8.5) + py: r.expr(1.75) - 8.5 + rb: (r 1.75) - 8.5 + ot: -6.75 + + - cd: r.expr(1).sub(2,3,4,5) + ot: -13 + + # Type errors + - cd: r.expr('a').sub(0.8) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - cd: r.expr(1).sub('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) + + - cd: r.expr('b').sub('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/meta/composite.py.yaml b/ext/librethinkdbxx/test/upstream/meta/composite.py.yaml new file mode 100644 index 00000000..59a8cf5b --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/meta/composite.py.yaml @@ -0,0 +1,14 @@ +desc: Tests meta operations in composite queries +tests: + + - py: r.expr([1,2,3]).for_each(r.db_create('db_' + r.row.coerce_to('string'))) + ot: ({'dbs_created':3,'config_changes':arrlen(3)}) + + - py: | + r.db_list().set_difference(["rethinkdb", "test"]).for_each(lambda db_name: + r.expr([1,2,3]).for_each(lambda i: + r.db(db_name).table_create('tbl_' + i.coerce_to('string')))) + ot: partial({'tables_created':9}) + + - py: r.db_list().set_difference(["rethinkdb", "test"]).for_each(r.db_drop(r.row)) + ot: partial({'dbs_dropped':3,'tables_dropped':9}) diff --git a/ext/librethinkdbxx/test/upstream/meta/dbs.yaml b/ext/librethinkdbxx/test/upstream/meta/dbs.yaml new file mode 100644 index 00000000..c49460e3 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/meta/dbs.yaml @@ -0,0 +1,51 @@ +desc: Tests meta queries for databases +tests: + + # We should always start out with a 'test' database and the special 'rethinkdb' + # database + - cd: r.db_list() + ot: bag(['rethinkdb', 'test']) + + ## DB create + + - cd: r.db_create('a') + ot: partial({'dbs_created':1}) + - cd: r.db_create('b') + ot: partial({'dbs_created':1}) + + ## DB list + + - cd: r.db_list() + ot: bag(['rethinkdb', 'a', 'b', 'test']) + + ## DB config + + - cd: r.db('a').config() + ot: {'name':'a','id':uuid()} + + ## DB drop + + - cd: r.db_drop('b') + ot: partial({'dbs_dropped':1}) + + - cd: r.db_list() + ot: bag(['rethinkdb', 'a', 'test']) + + - cd: r.db_drop('a') + ot: partial({'dbs_dropped':1}) + + - cd: r.db_list() + ot: bag(['rethinkdb', 'test']) + + ## DB errors + - cd: r.db_create('bar') + ot: partial({'dbs_created':1}) + + - cd: r.db_create('bar') + ot: err('ReqlOpFailedError', 'Database `bar` already exists.', [0]) + + - cd: r.db_drop('bar') + ot: partial({'dbs_dropped':1}) + + - cd: r.db_drop('bar') + ot: err('ReqlOpFailedError', 'Database `bar` does not exist.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/meta/table.yaml b/ext/librethinkdbxx/test/upstream/meta/table.yaml new file mode 100644 index 00000000..940a9d27 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/meta/table.yaml @@ -0,0 +1,365 @@ +desc: Tests meta queries for creating and deleting tables +tests: + + - def: db = r.db('test') + + - cd: db.table_list() + ot: [] + + - cd: r.db('rethinkdb').info() + ot: ({'type':'DB','name':'rethinkdb','id':null}) + + - cd: r.db('rethinkdb').table('stats').info() + ot: partial({'db':{'type':'DB','name':'rethinkdb','id':null}, + 'type':'TABLE','id':null,'name':'stats', + 'indexes':[],'primary_key':'id'}) + + # Table create + - cd: db.table_create('a') + ot: partial({'tables_created':1}) + + - cd: db.table_list() + ot: ['a'] + + - cd: db.table_create('b') + ot: partial({'tables_created':1}) + + - cd: db.table_list() + ot: bag(['a', 'b']) + + # Table drop + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + + - cd: db.table_list() + ot: ['b'] + + - cd: db.table_drop('b') + ot: partial({'tables_dropped':1}) + + - cd: db.table_list() + ot: [] + + # Table create options + - py: db.table_create('ab', durability='soft') + js: db.table_create('ab', {durability:'soft'}) + rb: db.table_create('ab', :durability => 'soft') + ot: partial({'tables_created':1,'config_changes':[partial({'new_val':partial({'durability':'soft'})})]}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', durability='hard') + js: db.table_create('ab', {durability:'hard'}) + rb: db.table_create('ab', :durability => 'hard') + ot: partial({'tables_created':1,'config_changes':[partial({'new_val':partial({'durability':'hard'})})]}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', durability='fake') + js: db.table_create('ab', {durability:'fake'}) + rb: db.table_create('ab', :durability => 'fake') + ot: err('ReqlQueryLogicError', 'Durability option `fake` unrecognized (options are "hard" and "soft").') + + - py: db.table_create('ab', primary_key='bar', shards=2, replicas=1) + js: db.tableCreate('ab', {primary_key:'bar', shards:2, replicas:1}) + rb: db.table_create('ab', {:primary_key => 'bar', :shards => 1, :replicas => 1}) + ot: partial({'tables_created':1}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', primary_key='bar', primary_replica_tag='default') + js: db.tableCreate('ab', {primary_key:'bar', primaryReplicaTag:'default'}) + rb: db.table_create('ab', {:primary_key => 'bar', :primary_replica_tag => 'default'}) + ot: partial({'tables_created':1}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', nonvoting_replica_tags=['default']) + js: db.tableCreate('ab', {nonvotingReplicaTags:['default']}) + rb: db.table_create('ab', {:nonvoting_replica_tags => ['default']}) + ot: partial({'tables_created':1}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + # Table reconfigure + - cd: db.table_create('a') + ot: partial({'tables_created':1}) + + - py: db.table('a').reconfigure(shards=1, replicas=1) + js: db.table('a').reconfigure({shards:1, replicas:1}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 1) + ot: partial({'reconfigured':1}) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":1}, nonvoting_replica_tags=['default'], primary_replica_tag='default') + js: db.table('a').reconfigure({shards:1, replicas:{default:1}, nonvoting_replica_tags:['default'], primary_replica_tag:'default'}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 1}, :nonvoting_replica_tags => ['default'], :primary_replica_tag => 'default') + ot: partial({'reconfigured':1}) + + - py: db.table('a').reconfigure(shards=1, replicas=1, dry_run=True) + js: db.table('a').reconfigure({shards:1, replicas:1, dry_run:true}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 1, :dry_run => true) + ot: partial({'reconfigured':0}) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback") + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback"}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback") + ot: err('ReqlOpFailedError', 'This table doesn\'t need to be repaired.', []) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback", dry_run=True) + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback", dry_run:true}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback", :dry_run => true) + ot: err('ReqlOpFailedError', 'This table doesn\'t need to be repaired.', []) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback_or_erase") + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback_or_erase"}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback_or_erase") + ot: err('ReqlOpFailedError', 'This table doesn\'t need to be repaired.', []) + + - py: db.table('a').reconfigure(emergency_repair=None, shards=1, replicas=1, dry_run=True) + js: db.table('a').reconfigure({emergency_repair:null, shards:1, replicas:1, dry_run:true}) + rb: db.table('a').reconfigure(:emergency_repair => null, :shards => 1, :replicas => 1, :dry_run => true) + ot: partial({'reconfigured':0}) + + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + + # Table errors + - cd: db.table_create('foo') + ot: partial({'tables_created':1}) + + - cd: db.table_create('foo') + ot: err('ReqlOpFailedError', 'Table `test.foo` already exists.', [0]) + + - cd: db.table_drop('foo') + ot: partial({'tables_dropped':1}) + + - cd: db.table_drop('foo') + ot: err('ReqlOpFailedError', 'Table `test.foo` does not exist.', [0]) + + - cd: db.table_create('nonsense', 'foo') + ot: + js: err('ReqlCompileError', 'Expected 1 argument (not including options) but found 2.', []) + rb: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + py: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + + - js: db.table_create('nonsense', {'foo':'bar'}) + py: db.table_create('nonsense', foo='bar') + rb: db.table_create('nonsense', :foo => 'bar') + ot: err('ReqlCompileError', "Unrecognized optional argument `foo`.", []) + + # RSI(reql_admin): Add tests for table_create() with configuration parameters + + # Table reconfigure errors + - cd: db.table_create('a') + ot: partial({'tables_created':1}) + + - py: db.table('a').reconfigure(shards=0, replicas=1) + js: db.table('a').reconfigure({shards:0, replicas:1}) + rb: db.table('a').reconfigure(:shards => 0, :replicas => 1) + ot: err('ReqlQueryLogicError', 'Every table must have at least one shard.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":1}, primary_replica_tag="foo") + js: db.table('a').reconfigure({shards:1, replicas:{default:1}, primary_replica_tag:"foo"}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 1}, :primary_replica_tag => "foo") + ot: err('ReqlOpFailedError', 'Can\'t use server tag `foo` for primary replicas because you specified no replicas in server tag `foo`.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":1}, primary_replica_tag="default", nonvoting_replica_tags=["foo"]) + js: db.table('a').reconfigure({shards:1, replicas:{"default":1}, primary_replica_tag:"default", nonvoting_replica_tags:["foo"]}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 1}, :primary_replica_tag => "default", :nonvoting_replica_tags => ["foo"]) + ot: err('ReqlOpFailedError', 'You specified that the replicas in server tag `foo` should be non-voting, but you didn\'t specify a number of replicas in server tag `foo`.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"foo":0}, primary_replica_tag="foo") + js: db.table('a').reconfigure({shards:1, replicas:{foo:0}, primary_replica_tag:"foo"}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:foo => 0}, :primary_replica_tag => "foo") + ot: err('ReqlOpFailedError', 'You must set `replicas` to at least one. `replicas` includes the primary replica; if there are zero replicas, there is nowhere to put the data.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":0}) + js: db.table('a').reconfigure({shards:1, replicas:{default:0}}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 0}) + ot: err('ReqlQueryLogicError', '`primary_replica_tag` must be specified when `replicas` is an OBJECT.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":-3}, primary_replica_tag='default') + js: db.table('a').reconfigure({shards:1, replicas:{default:-3}, primary_replica_tag:'default'}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => -3}, :primary_replica_tag => 'default') + ot: err('ReqlQueryLogicError', 'Can\'t have a negative number of replicas', []) + + - py: db.table('a').reconfigure(shards=1, replicas=3, primary_replica_tag='foo') + js: db.table('a').reconfigure({shards:1, replicas:3, primary_replica_tag:'foo'}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 3, :primary_replica_tag => 'foo') + ot: err('ReqlQueryLogicError', '`replicas` must be an OBJECT if `primary_replica_tag` is specified.', []) + + - py: db.table('a').reconfigure(shards=1, replicas=3, nonvoting_replica_tags=['foo']) + js: db.table('a').reconfigure({shards:1, replicas:3, nonvoting_replica_tags:['foo']}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 3, :nonvoting_replica_tags => ['foo']) + ot: err('ReqlQueryLogicError', '`replicas` must be an OBJECT if `nonvoting_replica_tags` is specified.', []) + + - py: db.reconfigure(emergency_repair="unsafe_rollback") + js: db.reconfigure({emergency_repair:"unsafe_rollback"}) + rb: db.reconfigure(:emergency_repair => "unsafe_rollback") + ot: err('ReqlQueryLogicError', 'Can\'t emergency repair an entire database at once; instead you should run `reconfigure()` on each table individually.') + + - py: db.table('a').reconfigure(emergency_repair="foo") + js: db.table('a').reconfigure({emergency_repair:"foo"}) + rb: db.table('a').reconfigure(:emergency_repair => "foo") + ot: err('ReqlQueryLogicError', '`emergency_repair` should be "unsafe_rollback" or "unsafe_rollback_or_erase"', []) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback", shards=1, replicas=1) + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback", shards:1, replicas:1}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback", :shards => 1, :replicas => 1) + ot: err('ReqlQueryLogicError', 'In emergency repair mode, you can\'t specify shards, replicas, etc.') + + # Test reconfigure auto-sharding without data + - py: db.table('a').reconfigure(shards=2, replicas=1) + js: db.table('a').reconfigure({shards:2, replicas:1}) + rb: db.table('a').reconfigure(:shards => 2, :replicas => 1) + ot: partial({'reconfigured':1}) + + - py: db.table('a').wait(wait_for="all_replicas_ready") + js: db.table('a').wait({"waitFor":"all_replicas_ready"}) + rb: db.table('a').wait(:wait_for=>"all_replicas_ready") + ot: {"ready":1} + + # Insert some data so that `reconfigure()` can pick shard points + - py: db.table('a').insert([{"id":1}, {"id":2}, {"id":3}, {"id":4}]) + js: db.table('a').insert([{id:1}, {id:2}, {id:3}, {id:4}]) + rb: db.table('a').insert([{"id" => 1}, {"id" => 2}, {"id" => 3}, {"id" => 4}]) + ot: partial({"inserted":4}) + + - py: db.table('a').reconfigure(shards=2, replicas=1) + js: db.table('a').reconfigure({shards:2, replicas:1}) + rb: db.table('a').reconfigure(:shards => 2, :replicas => 1) + ot: partial({'reconfigured':1}) + + - py: db.table('a').reconfigure(shards=1, replicas=2) + js: db.table('a').reconfigure({shards:1, replicas:2}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 2) + ot: err('ReqlOpFailedError', 'Can\'t put 2 replicas on servers with the tag `default` because there are only 1 servers with the tag `default`. It\'s impossible to have more replicas of the data than there are servers.', []) + + # Test wait and rebalance + - py: db.table('a').wait(wait_for="all_replicas_ready") + js: db.table('a').wait({"waitFor":"all_replicas_ready"}) + rb: db.table('a').wait(:wait_for=>"all_replicas_ready") + ot: {"ready":1} + - cd: db.table('a').rebalance() + ot: partial({'rebalanced':1}) + + - py: db.wait(wait_for="all_replicas_ready") + js: db.wait({"waitFor":"all_replicas_ready"}) + rb: db.wait(:wait_for=>"all_replicas_ready") + ot: {"ready":1} + - cd: db.rebalance() + ot: partial({'rebalanced':1}) + + - cd: r.wait() + ot: + py: err('AttributeError', "'module' object has no attribute 'wait'", []) + # different sub-versions of node have different messages #5617 + js: err('TypeError') + rb: err('ReqlQueryLogicError', '`wait` can only be called on a table or database.', []) + - cd: r.rebalance() + ot: + py: err('AttributeError', "'module' object has no attribute 'rebalance'", []) + # different sub-versions of node have different messages #5617 + js: err('TypeError') + rb: err('ReqlQueryLogicError', '`rebalance` can only be called on a table or database.', []) + + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + + # Reconfiguring all tables in a database + - cd: db.table_create('a') + - cd: db.table_create('b') + - cd: db.table_create('c') + + - py: db.reconfigure(shards=0, replicas=1) + js: db.reconfigure({shards:0, replicas:1}) + rb: db.reconfigure(:shards => 0, :replicas => 1) + ot: err('ReqlQueryLogicError', 'Every table must have at least one shard.', []) + + - py: db.reconfigure(shards=1, replicas={"default":0}) + js: db.reconfigure({shards:1, replicas:{default:0}}) + rb: db.reconfigure(:shards => 1, :replicas => {:default => 0}) + ot: err('ReqlQueryLogicError', '`primary_replica_tag` must be specified when `replicas` is an OBJECT.', []) + + - py: db.reconfigure(shards=1, replicas={"default":-3}, primary_replica_tag='default') + js: db.reconfigure({shards:1, replicas:{default:-3}, primary_replica_tag:'default'}) + rb: db.reconfigure(:shards => 1, :replicas => {:default => -3}, :primary_replica_tag => 'default') + ot: err('ReqlQueryLogicError', 'Can\'t have a negative number of replicas', []) + + - py: db.reconfigure(shards=1, replicas=3, primary_replica_tag='foo') + js: db.reconfigure({shards:1, replicas:3, primary_replica_tag:'foo'}) + rb: db.reconfigure(:shards => 1, :replicas => 3, :primary_replica_tag => 'foo') + ot: err('ReqlQueryLogicError', '`replicas` must be an OBJECT if `primary_replica_tag` is specified.', []) + + - py: db.reconfigure(shards=2, replicas=1) + js: db.reconfigure({shards:2, replicas:1}) + rb: db.reconfigure(:shards => 2, :replicas => 1) + ot: partial({'reconfigured':3}) + + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + - cd: db.table_drop('b') + ot: partial({'tables_dropped':1}) + - cd: db.table_drop('c') + ot: partial({'tables_dropped':1}) + + # table_config and table_status porcelains + - cd: r.db_create("test2") + ot: partial({'dbs_created':1}) + + - def: db2 = r.db("test2") + + - cd: db.table_create("testA") + ot: partial({'tables_created':1}) + - cd: db.table_create("testB") + ot: partial({'tables_created':1}) + - cd: db2.table_create("test2B") + ot: partial({'tables_created':1}) + + - cd: r.table('testA').config().pluck('db','name') + ot: {'db':'test','name':'testA'} + + - cd: r.table('doesntexist').config() + ot: err('ReqlOpFailedError', 'Table `test.doesntexist` does not exist.', []) + + - cd: r.table('test2B').config() + ot: err('ReqlOpFailedError', 'Table `test.test2B` does not exist.', []) + + - cd: r.db('rethinkdb').table('table_config').filter({'name':'testA'}).nth(0).eq(r.table('testA').config()) + ot: True + + - cd: r.db('rethinkdb').table('table_status').filter({'name':'testA'}).nth(0).eq(r.table('testA').status()) + ot: True + + - py: r.db('rethinkdb').table('table_config', identifier_format='uuid').nth(0)["db"] + js: r.db('rethinkdb').table('table_config', {identifierFormat:'uuid'}).nth(0)("db") + rb: r.db('rethinkdb').table('table_config', {:identifier_format=>'uuid'}).nth(0)["db"] + ot: uuid() + + - py: r.table('testA', identifier_format='uuid').count() + js: r.table('testA', {identifierFormat:'uuid'}).count() + rb: r.table('testA', {:identifier_format=>'uuid'}).count() + ot: 0 + + - py: r.wait(wait_for='all_replicas_ready', timeout=5) + js: r.wait({waitFor:'all_replicas_ready', timeout:5}) + rb: r.wait(:wait_for=>'all_replicas_ready', :timeout => 5) + ot: + py: err('AttributeError', "'module' object has no attribute 'wait'", []) + # different sub-versions of node have different messages #5617 + js: err('TypeError') + rb: err('ReqlQueryLogicError', '`wait` can only be called on a table or database.', []) + + - cd: db.table_drop('testA') + ot: partial({'tables_dropped':1}) + + - cd: db.table_drop('testB') + ot: partial({'tables_dropped':1}) + + - cd: r.db_drop('test2') + ot: partial({'dbs_dropped':1,'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml b/ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml new file mode 100644 index 00000000..ef1621a7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml @@ -0,0 +1,93 @@ +desc: Tests replacement of selections +table_variable_name: tbl +tests: + + # old version of argument + - cd: tbl.insert({'id':0}, :return_vals => true).pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_vals=True).pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return_vals':true}).pluck('changes', 'first__error') + ot: err("ReqlQueryLogicError", "Error:"+" encountered obsolete optarg `return_vals`. Use `return_changes` instead.", [0]) + + - cd: tbl.insert({'id':0}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_changes=True).pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':null,'new_val':{'id':0}}]}) + - cd: tbl.insert({'id':0}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_changes=True).pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[], 'first_error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}"}) + - cd: tbl.insert({'id':0}, return_changes:'always').pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_changes='always').pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return__changes':'always'}).pluck('changes', 'first__error') + ot: ({'first_error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}",'changes':[{'old_val':{'id':0},'new_val':{'id':0},'error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}"}]}) + - cd: tbl.insert([{'id':1}], :return_changes => true) + py: tbl.insert([{'id':1}], return_changes=True) + js: tbl.insert([{'id':1}], {'return__changes':true}) + ot: ({'changes':[{'new_val':{'id':1},'old_val':null}], 'errors':0, 'deleted':0, 'unchanged':0, 'skipped':0, 'replaced':0, 'inserted':1}) + - cd: tbl.insert([{'id':0}], :return_changes => true).pluck('changes', 'first_error') + py: tbl.insert([{'id':0}], return_changes=True).pluck('changes', 'first_error') + js: tbl.insert([{'id':0}], {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[],'first_error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}"}) + + - cd: tbl.get(0).update({'x':1}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).update({'x':1}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).update({'x':1}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':{'id':0},'new_val':{'id':0,'x':1}}]}) + - cd: tbl.get(0).update({'x':r.error("a")}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).update({'x':r.error("a")}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).update({'x':r.error("a")}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[],'first_error':'a'}) + - rb: tbl.update({'x':3}, :return_changes => true).pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + py: tbl.update({'x':3}, return_changes=True).pluck('changes', 'first_error').do(lambda d:d.merge({'changes':d['changes'].order_by(lambda a:a['old_val']['id'])})) + js: tbl.update({'x':3}, {'return__changes':true}).pluck('changes', 'first__error').do(function(p){return p.merge({'changes':p('changes').orderBy(function(a){return a('old__val')('id')})})}) + ot: ({'changes':[{'old_val':{'id':0, 'x':1},'new_val':{'id':0, 'x':3}}, {'old_val':{'id':1},'new_val':{'id':1, 'x':3}}]}) + + - cd: tbl.get(0).replace({'id':0,'x':2}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).replace({'id':0,'x':2}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).replace({'id':0,'x':2}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':{'id':0,'x':3},'new_val':{'id':0,'x':2}}]}) + - cd: tbl.get(0).replace(:return_changes => true){{'x':r.error('a')}}.pluck('changes', 'first_error') + py: tbl.get(0).replace(lambda y:{'x':r.error('a')}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).replace(function(y){return {'x':r.error('a')}}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[],'first_error':'a'}) + - cd: tbl.get(0).replace(:return_changes => 'always'){{'x':r.error('a')}}.pluck('changes', 'first_error') + py: tbl.get(0).replace(lambda y:{'x':r.error('a')}, return_changes='always').pluck('changes', 'first_error') + js: tbl.get(0).replace(function(y){return {'x':r.error('a')}}, {'return__changes':'always'}).pluck('changes', 'first__error') + ot: ({'first_error':'a','changes':[{'old_val':{'id':0,'x':2},'new_val':{'id':0,'x':2},'error':'a'}]}) + - rb: tbl.replace( :return_changes => true) { |d| d.without('x')}.pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + py: tbl.replace(lambda y:y.without('x'), return_changes=True).pluck('changes', 'first_error').do(lambda d:d.merge({'changes':d['changes'].order_by(lambda a:a['old_val']['id'])})) + js: tbl.replace(function(p){return p.without('x')}, {'return__changes':true}).pluck('changes', 'first__error').do(function(p){return p.merge({'changes':p('changes').orderBy(function(a){return a('old__val')('id')})})}) + ot: ({'changes':[{'new_val':{'id':0},'old_val':{'id':0, 'x':2}}, {'new_val':{'id':1},'old_val':{'id':1,'x':3}}]}) + - rb: tbl.replace({'x':1}, :return_changes => 'always').pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + py: tbl.replace({'x':1}, return_changes='always').pluck('changes', 'first_error').do(lambda d:d.merge({'changes':d['changes'].order_by(lambda a:a['old_val']['id'])})) + js: tbl.replace({'x':1}, {'return__changes':'always'}).pluck('changes', 'first__error').do(function(p){return p.merge({'changes':p('changes').orderBy(function(a){return a('old__val')('id')})})}) + ot: ({'first_error':"Inserted object must have primary key `id`:\n{\n\t\"x\":\t1\n}", 'changes':[{'new_val':{'id':0},'old_val':{'id':0}, 'error':"Inserted object must have primary key `id`:\n{\n\t\"x\":\t1\n}"}, {'new_val':{'id':1},'old_val':{'id':1},'error':"Inserted object must have primary key `id`:\n{\n\t\"x\":\t1\n}"}]}) + + - rb: tbl.foreach{|row| [tbl.get(0).update(null, :return_changes => true), tbl.get(0).update({a:1}, :return_changes => true)]}.pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + ot: ({'changes':[{"new_val"=>{"a"=>1, "id"=>0}, "old_val"=>{"id"=>0}}]}) + - rb: tbl.get(0).update({a:r.literal()})['replaced'] + ot: 1 + + - rb: tbl.get(0).update({}).pluck('changes') + ot: ({}) + - rb: tbl.get(0).update({}, return_changes:true).pluck('changes') + ot: ({'changes':[]}) + - rb: tbl.get(0).update({}, return_changes:'always').pluck('changes') + ot: ({'changes':[{'new_val':{'id':0}, 'old_val':{'id':0}}]}) + + - rb: tbl.get(-1).update({}).pluck('changes') + ot: ({}) + - rb: tbl.get(-1).update({}, return_changes:true).pluck('changes') + ot: ({'changes':[]}) + - rb: tbl.get(-1).update({}, return_changes:'always').pluck('changes') + ot: ({'changes':[{'new_val':null, 'old_val':null}]}) + + - cd: tbl.get(0).delete(:return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).delete(return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).delete({'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':{'id':0},'new_val':null}]}) + - cd: tbl.delete(:return_changes => true) + py: tbl.delete(return_changes=True) + js: tbl.delete({'return__changes':true}) + ot: ({'deleted':1,'errors':0,'inserted':0,'replaced':0,'skipped':0,'unchanged':0,'changes':[{'new_val':null, 'old_val':{'id':1}}]}) + diff --git a/ext/librethinkdbxx/test/upstream/mutation/delete.yaml b/ext/librethinkdbxx/test/upstream/mutation/delete.yaml new file mode 100644 index 00000000..b222dba9 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/delete.yaml @@ -0,0 +1,50 @@ +desc: Tests deletes of selections +table_variable_name: tbl +tests: + + # Set up some data + + - py: tbl.insert([{'id':i} for i in xrange(100)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id: i}); + } + return res; + }()) + rb: tbl.insert((1..100).map{ |i| {"id" => i} }) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':100}) + + - cd: tbl.count() + ot: 100 + + # Point delete + + - cd: tbl.get(12).delete() + ot: ({'deleted':1,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0}) + + # Attempt deletion with bad durability flag. + + - js: tbl.skip(50).delete({durability:'wrong'}) + rb: tbl.skip(50).delete({ :durability => 'wrong' }) + py: tbl.skip(50).delete(durability='wrong') + ot: err('ReqlQueryLogicError', 'Durability option `wrong` unrecognized (options are "hard" and "soft").', [0]) + + # Delete selection of table, soft durability flag. + + - js: tbl.skip(50).delete({durability:'soft'}) + rb: tbl.skip(50).delete({ :durability => 'soft' }) + py: tbl.skip(50).delete(durability='soft') + ot: ({'deleted':49,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0}) + + # Delete whole table, hard durability flag. + + - js: tbl.delete({durability:'hard'}) + rb: tbl.delete({ :durability => 'hard' }) + py: tbl.delete(durability='hard') + ot: ({'deleted':50,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0}) + + # test deletion on a non-deletable object + - cd: r.expr([1, 2]).delete() + ot: err('ReqlQueryLogicError', 'Expected type SELECTION but found DATUM:', [0]) diff --git a/ext/librethinkdbxx/test/upstream/mutation/insert.yaml b/ext/librethinkdbxx/test/upstream/mutation/insert.yaml new file mode 100644 index 00000000..623a6edc --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/insert.yaml @@ -0,0 +1,239 @@ +desc: Tests insertion into tables +table_variable_name: tbl +tests: + + # Set up our secondary test table + - cd: r.db('test').table_create('test2') + ot: partial({'tables_created':1}) + + - def: tbl2 = r.db('test').table('test2') + + # Single doc insert + - cd: tbl.insert({'id':0,'a':0}) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + - cd: tbl.count() + ot: 1 + + # Hard durability insert + - py: tbl.insert({'id':1, 'a':1}, durability='hard') + js: tbl.insert({id:1, a:1}, {durability:'hard'}) + rb: tbl.insert({ :id => 1, :a => 1 }, { :durability => 'hard' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + - cd: tbl.count() + ot: 2 + + # Soft durability insert + - py: tbl.insert({'id':2, 'a':2}, durability='soft') + js: tbl.insert({id:2, a:2}, {durability:'soft'}) + rb: tbl.insert({ :id => 2, :a => 2 }, { :durability => 'soft' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + - cd: tbl.count() + ot: 3 + + # Wrong durability insert + - py: tbl.insert({'id':3, 'a':3}, durability='wrong') + js: tbl.insert({id:3, a:3}, {durability:'wrong'}) + rb: tbl.insert({ :id => 3, :a => 3 }, { :durability => 'wrong' }) + ot: err('ReqlQueryLogicError', 'Durability option `wrong` unrecognized (options are "hard" and "soft").', [0]) + - cd: tbl.count() + ot: 3 + + # Cleanup. + - cd: tbl.get(2).delete() + ot: {'deleted':1,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0} + + # Multi doc insert + - cd: tbl.insert([{'id':2,'a':2}, {'id':3,'a':3}]) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':2} + + # Stream insert + - cd: tbl2.insert(tbl) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':4} + + # test pkey clash error + - cd: tbl.insert({'id':2,'b':20}) + ot: {'first_error':"Duplicate primary key `id`:\n{\n\t\"a\":\t2,\n\t\"id\":\t2\n}\n{\n\t\"b\":\t20,\n\t\"id\":\t2\n}",'deleted':0,'replaced':0,'unchanged':0,'errors':1,'skipped':0,'inserted':0} + + # test error conflict option (object exists) + - py: tbl.insert({'id':2,'b':20}, conflict='error') + js: tbl.insert({'id':2,'b':20}, {conflict:'error'}) + rb: tbl.insert({:id => 2, :b => 20}, { :conflict => 'error' }) + ot: {'first_error':"Duplicate primary key `id`:\n{\n\t\"a\":\t2,\n\t\"id\":\t2\n}\n{\n\t\"b\":\t20,\n\t\"id\":\t2\n}",'deleted':0,'replaced':0,'unchanged':0,'errors':1,'skipped':0,'inserted':0} + + # test error conflict option (object doesn't exist) + - py: tbl.insert({'id':15,'b':20}, conflict='error') + js: tbl.insert({'id':15,'b':20}, {conflict:'error'}) + rb: tbl.insert({:id => 15, :b => 20}, { :conflict => 'error' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tbl.get(15) + ot: {'id':15,'b':20} + + # test replace conflict option (object exists) + - py: tbl.insert({'id':2,'b':20}, conflict='replace') + js: tbl.insert({'id':2,'b':20}, {conflict:'replace'}) + rb: tbl.insert({:id => 2, :b => 20}, { :conflict => 'replace' }) + ot: {'deleted':0,'replaced':1,'unchanged':0,'errors':0,'skipped':0,'inserted':0} + + - cd: tbl.get(2) + ot: {'id':2,'b':20} + + # test replace conflict option (object doesn't exist) + - py: tbl.insert({'id':20,'b':20}, conflict='replace') + js: tbl.insert({'id':20,'b':20}, {conflict:'replace'}) + rb: tbl.insert({:id => 20, :b => 20}, { :conflict => 'replace' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tbl.get(20) + ot: {'id':20,'b':20} + + # test update conflict option (object exists) + - py: tbl.insert({'id':2,'c':30}, conflict='update') + js: tbl.insert({'id':2,'c':30}, {conflict:'update'}) + rb: tbl.insert({:id => 2, :c => 30}, { :conflict => 'update' }) + ot: {'deleted':0,'replaced':1,'unchanged':0,'errors':0,'skipped':0,'inserted':0} + + - cd: tbl.get(2) + ot: {'id':2, 'b':20, 'c':30} + + # test update conflict option (object doesn't exist) + - py: tbl.insert({'id':30,'b':20}, conflict='update') + js: tbl.insert({'id':30,'b':20}, {conflict:'update'}) + rb: tbl.insert({:id => 30, :b => 20}, { :conflict => 'update' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tbl.get(30) + ot: {'id':30,'b':20} + + # test incorrect conflict option + - py: tbl.insert({'id':3, 'a':3}, conflict='wrong') + js: tbl.insert({id:3, a:3}, {conflict:'wrong'}) + rb: tbl.insert({ :id => 3, :a => 3 }, { :conflict => 'wrong' }) + ot: err('ReqlQueryLogicError', 'Conflict option `wrong` unrecognized (options are "error", "replace" and "update").', [0]) + + # test auto pkey generation + - py: r.db('test').table_create('testpkey', primary_key='foo') + js: r.db('test').tableCreate('testpkey', {primaryKey:'foo'}) + rb: r.db('test').table_create('testpkey', { :primary_key => 'foo' }) + ot: partial({'tables_created':1}) + + def: tblpkey = r.db('test').table('testpkey') + + - cd: tblpkey.insert({}) + ot: {'deleted':0,'replaced':0,'generated_keys':arrlen(1,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tblpkey + ot: [{'foo':uuid()}] + + # test replace conflict pkey generation + - py: tblpkey.insert({'b':20}, conflict='replace') + js: tblpkey.insert({'b':20}, {conflict:'replace'}) + rb: tblpkey.insert({:b => 20}, { :conflict => 'replace' }) + ot: {'deleted':0,'replaced':0,'generated_keys':arrlen(1,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + # test update conflict pkey generation + - py: tblpkey.insert({'b':20}, conflict='update') + js: tblpkey.insert({'b':20}, {conflict:'update'}) + rb: tblpkey.insert({:b => 20}, { :conflict => 'update' }) + ot: {'deleted':0,'replaced':0,'generated_keys':arrlen(1,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: r.db('test').table_drop('testpkey') + ot: partial({'tables_dropped':1}) + + # Insert within for each + - py: tbl.for_each(lambda row: tbl2.insert(row.merge({'id':row['id'] + 100 })) ) + js: tbl.forEach(function(row) { return tbl2.insert(row.merge({'id':row('id').add(100)})); }) + rb: tbl.for_each(proc { |row| tbl2.insert(row.merge({'id'=>row['id'] + 100 })) }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':7} + + # Insert unwritable data + - cd: tbl.insert({'value':r.minval}) + rb: tbl.insert({:value => r.minval}) + ot: partial({'errors':1,'first_error':'`r.minval` and `r.maxval` cannot be written to disk.'}) + + - cd: tbl.insert({'value':r.maxval}) + rb: tbl.insert({:value => r.maxval}) + ot: partial({'errors':1,'first_error':'`r.minval` and `r.maxval` cannot be written to disk.'}) + + # Crash 5683 + - py: tbl.insert([{'id':666}, {'id':666}], return_changes="always") + ot: {'changes': [{'new_val': {'id': 666}, 'old_val': None},{'error': 'Duplicate primary key `id`:\n{\n\t"id":\t666\n}\n{\n\t"id":\t666\n}','new_val': {'id': 666},'old_val': {'id': 666}}],'deleted': 0,'errors': 1,'first_error': 'Duplicate primary key `id`:\n{\n\t"id":\t666\n}\n{\n\t"id":\t666\n}','inserted': 1,'replaced': 0,'skipped': 0,'unchanged': 0} + + # Confirm inserts are ordered in return_changes always + - py: tbl.insert([{'id':100+i, 'ordered-num':i} for i in range(1,100)], return_changes="always") + ot: partial({'changes':[{'old_val': None, 'new_val': {'id': 100+i, 'ordered-num': i}} for i in range(1,100)] }) + + # Confirm inserts are ordered in return_changes always with complicated key + - py: tbl.insert([{'id':[1, "blah", 200+i], 'ordered-num':i} for i in range(1,100)], return_changes="always") + ot: partial({'changes':[{'old_val': None, 'new_val': {'id': [1,"blah", 200+i], 'ordered-num': i}} for i in range(1,100)] }) + + # Confirm inserts are ordered in return_changes always with return_changes=true + - py: tbl.insert([{'id':[1, "blah", 300+i], 'ordered-num':i} for i in range(1,100)], return_changes=true) + ot: partial({'changes':[{'old_val': None, 'new_val': {'id': [1,"blah", 300+i], 'ordered-num': i}} for i in range(1,100)] }) + + # Confirm errors are property returned with return_changes="always" + - py: tbl.insert([{'id':100 + i, 'ordered-num':i} for i in range(1,100)], return_changes="always") + ot: partial({'changes':[{'old_val': {'id':100+i, 'ordered-num':i}, 'new_val': {'id':100+i, 'ordered-num':i}, 'error':'Duplicate primary key `id`:\n{\n\t"id":\t'+str(100+i)+',\n\t"ordered-num":\t'+str(i)+'\n}\n{\n\t"id":\t'+str(100+i)+',\n\t"ordered-num":\t'+str(i)+'\n}'} for i in range(1,100)]}) + + # Trivial errors with return_changes="always", this has to be long test, testing order and message for this type + - py: tbl.insert([{'id':123}, {'id':'a'*500}, {'id':321}], return_changes="always") + ot: {'changes': [{'error': 'Duplicate primary key `id`:\n{\n\t"id":\t123,\n\t"ordered-num":\t23\n}\n{\n\t"id":\t123\n}', 'new_val': {'id': 123, 'ordered-num': 23}, 'old_val': {'id': 123, 'ordered-num': 23}}, {'error': 'Primary key too long (max 127 characters): "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"', 'new_val': None, 'old_val': None}, {'new_val': {'id': 321}, 'old_val': None}], 'deleted': 0, 'errors': 2, 'first_error': 'Primary key too long (max 127 characters): "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"', 'inserted': 1, 'replaced': 0, 'skipped': 0, 'unchanged': 0} + + # No errors returned with return_changes=true + - py: tbl.insert([{'id':100 + i, 'ordered-num':i} for i in range(1,100)], return_changes=true) + ot: partial({'changes':[]}) + + - py: tbl.insert({'a':r.minval}, return_changes="always") + ot: partial({'changes': [{'old_val': None, 'new_val': None, 'error': '`r.minval` and `r.maxval` cannot be written to disk.'}]}) + + # Tests for insert conflict resolution function + + # Using a conflict function + - cd: tbl.insert({'id':42, 'foo':1, 'bar':1}) + ot: partial({'inserted':1}) + - py: tbl.insert({'id':42, 'foo':5, 'bar':5}, conflict=lambda id, old_row, new_row: old_row.merge(new_row.pluck("bar"))) + ot: partial({'replaced':1}) + - py: tbl.get(42) + ot: {'id':42, 'foo':1, 'bar':5} + - rb: tbl.insert({:id=>42, :foo=>6, :bar=>6}, conflict: lambda {|id, old_row, new_row| return old_row.merge(new_row.pluck("bar"))}) + ot: partial({'replaced':1}) + - rb: tbl.get(42) + ot: {'id':42, 'foo':1, 'bar':6} + - js: tbl.insert({'id':42, 'foo':7, 'bar':7}, {conflict: function(id, old_row, new_row) {return old_row.merge(new_row.pluck("bar"))}}) + ot: partial({'replaced':1}) + - js: tbl.get(42) + ot: {'id':42, 'foo':1, 'bar':7} + + # Inserting and deleting an item + - js: tbl.insert({id: "toggle"},{conflict: function(x,y,z) { return null},returnChanges: true}) + ot: partial({'inserted': 1}) + - js: tbl.insert({id: "toggle"},{conflict: function(x,y,z) { return null},returnChanges: true}) + ot: partial({'deleted': 1}) + + # Returning the wrong thing from the conflict function + - py: tbl.insert({'id':42, 'foo':1, 'bar':1}, conflict=lambda a,b,c: 2) + ot: partial({'first_error': 'Inserted value must be an OBJECT (got NUMBER):\n2'}) + + # Incorrect Arity + - py: tbl.insert({'id':42}, conflict=lambda a,b: a) + ot: err("ReqlQueryLogicError", "The conflict function passed to `insert` should expect 3 arguments.") + + # Non atomic operation + - py: tbl.insert({'id':42}, conflict=lambda a,b,c: tbl.get(42)) + ot: err("ReqlQueryLogicError", "The conflict function passed to `insert` must be deterministic.") + + - py: tbl.insert({'id':42}, conflict=lambda a,b,c: {'id':42, 'num':'424'}) + ot: partial({'replaced': 1}) + - py: tbl.get(42) + ot: {'id':42, 'num':'424'} + + # Get unreturnable data + - cd: r.minval + ot: err('ReqlQueryLogicError','Cannot convert `r.minval` to JSON.') + + - cd: r.maxval + ot: err('ReqlQueryLogicError','Cannot convert `r.maxval` to JSON.') + + # clean up + - cd: r.db('test').table_drop('test2') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/mutation/replace.yaml b/ext/librethinkdbxx/test/upstream/mutation/replace.yaml new file mode 100644 index 00000000..38cf4f85 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/replace.yaml @@ -0,0 +1,110 @@ +desc: Tests replacement of selections +table_variable_name: tbl +tests: + + # Set up some data + + - py: tbl.insert([{'id':i} for i in xrange(100)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id:i}); + } + return res; + }()) + rb: tbl.insert((1..100).map{ |i| {:id => i } }) + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100}) + + - cd: tbl.count() + ot: 100 + + # Identity + + - py: tbl.get(12).replace(lambda row:{'id':row['id']}) + js: tbl.get(12).replace(function(row) { return {'id':row('id')}; }) + rb: tbl.get(12).replace{ |row| { :id => row[:id] } } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # Replace single row + + - py: tbl.get(12).replace(lambda row:{'id':row['id'], 'a':row['id']}) + js: tbl.get(12).replace(function(row) { return {'id':row('id'), 'a':row('id')}; }) + rb: tbl.get(12).replace{ |row| { :id => row[:id], :a => row[:id] } } + ot: ({'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - py: tbl.get(13).replace(lambda row:null) + js: tbl.get(13).replace(function(row) { return null; }) + rb: tbl.get(13).replace{ |row| null } + ot: ({'deleted':1,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # Replace selection of table + + - py: tbl.between(10, 20, right_bound='closed').replace(lambda row:{'a':1}) + js: tbl.between(10, 20, {'right_bound':'closed'}).replace(function(row) { return {'a':1}; }) + ot: ({'first_error':'Inserted object must have primary key `id`:\n{\n\t\"a\":\t1\n}','deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':10,'skipped':0.0,'inserted':0.0}) + + - py: tbl.filter(lambda row:(row['id'] >= 10) & (row['id'] < 20)).replace(lambda row:{'id':row['id'], 'a':row['id']}) + js: tbl.filter(function(row) { return row('id').ge(10).and(row('id').lt(20))}).replace(function(row) { return {'id':row('id'), 'a':row('id')}; }) + rb: tbl.filter{ |row| + (row[:id] >= 10).and(row[:id] < 20) + }.replace{ |row| + { :id => row[:id], :a => row[:id] } } + ot: ({'deleted':0.0,'replaced':8,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # trying to change pkey of a document + - cd: tbl.get(1).replace({'id':2,'a':1}) + ot: ({'first_error':"Primary key `id` cannot be changed (`{\n\t\"id\":\t1\n}` -> `{\n\t\"a\":\t1,\n\t\"id\":\t2\n}`).",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':1,'skipped':0.0,'inserted':0.0}) + + + # not passing a pkey in the first place + - cd: tbl.get(1).replace({'a':1}) + ot: ({'first_error':"Inserted object must have primary key `id`:\n{\n\t\"a\":\t1\n}",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':1,'skipped':0.0,'inserted':0.0}) + + # check r.row, static value and otherwise + - py: tbl.get(1).replace({'id':r.row['id'],'a':'b'}) + js: tbl.get(1).replace({'id':r.row('id'),'a':'b'}) + rb: tbl.get(1).replace{ |row| { :id => row[:id], :a => 'b' } } + ot: ({'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - cd: tbl.get(1).replace(r.row.merge({'a':'b'})) + rb: tbl.get(1).replace{ |row| row.merge({'a':'b'}) } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # test atomicity constraints + - cd: tbl.get(1).replace(r.row.merge({'c':r.js('5')})) + rb: tbl.get(1).replace{ |row| row.merge({'c':r.js('5')}) } + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - cd: tbl.get(1).replace(r.row.merge({'c':tbl.nth(0)})) + rb: tbl.get(1).replace{ |row| row.merge({'c':tbl.nth(0)}) } + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - py: tbl.get(1).replace(r.row.merge({'c':r.js('5')}), non_atomic=True) + js: tbl.get(1).replace(r.row.merge({'c':r.js('5')}), {'nonAtomic':true}) + rb: tbl.get(1).replace({ :non_atomic => true }){ |row| row.merge({ :c => r.js('5') })} + ot: ({'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - cd: tbl.get(1).replace({}, 'foo') + ot: + cd: err('ReqlCompileError', 'Expected 2 arguments but found 3.') + js: err('ReqlCompileError', 'Expected 1 argument (not including options) but found 2.') + + - cd: tbl.get(1).replace({}, {'foo':'bar'}) + py: tbl.get(1).replace({}, foo='bar') + ot: err('ReqlCompileError', 'Unrecognized optional argument `foo`.') + + # Replace whole table + + - py: tbl.replace(lambda row:null) + js: tbl.replace(function(row) { return null; }) + rb: tbl.replace{ |row| null } + ot: ({'deleted':99,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - cd: tbl.get('sdfjk').replace({'id':'sdfjk'})['inserted'] + js: tbl.get('sdfjk').replace({'id':'sdfjk'})('inserted') + ot: 1 + - cd: tbl.get('sdfjki').replace({'id':'sdfjk'})['errors'] + js: tbl.get('sdfjki').replace({'id':'sdfjk'})('errors') + ot: 1 + diff --git a/ext/librethinkdbxx/test/upstream/mutation/sync.yaml b/ext/librethinkdbxx/test/upstream/mutation/sync.yaml new file mode 100644 index 00000000..789b3c47 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/sync.yaml @@ -0,0 +1,52 @@ +desc: Tests syncing tables +tests: + + # Set up our test tables + - cd: r.db('test').table_create('test1') + ot: partial({'tables_created':1}) + - cd: r.db('test').table_create('test1soft') + ot: partial({'tables_created':1}) + - cd: r.db('test').table('test1soft').config().update({'durability':'soft'}) + ot: {'skipped':0, 'deleted':0, 'unchanged':0, 'errors':0, 'replaced':1, 'inserted':0} + - def: tbl = r.db('test').table('test1') + - def: tbl_soft = r.db('test').table('test1soft') + - cd: tbl.index_create('x') + ot: partial({'created':1}) + - cd: tbl.index_wait('x').pluck('index', 'ready') + ot: [{'ready':True, 'index':'x'}] + + # This is the only way one can use sync legally at the moment + - cd: tbl.sync() + ot: {'synced':1} + - cd: tbl_soft.sync() + ot: {'synced':1} + - cd: tbl.sync() + ot: {'synced':1} + runopts: + durability: "soft" + - cd: tbl.sync() + ot: {'synced':1} + runopts: + durability: "hard" + + # This is of type table, but sync should still fail (because it makes little sense) + - cd: tbl.between(1, 2).sync() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [1]) + py: err('AttributeError', "'Between' object has no attribute 'sync'") + + # These are not even a table. Sync should fail with a different error message + - cd: r.expr(1).sync() + ot: + cd: err("ReqlQueryLogicError", 'Expected type TABLE but found DATUM:', [1]) + py: err('AttributeError', "'Datum' object has no attribute 'sync'") + - js: tbl.order_by({index:'x'}).sync() + rb: tbl.order_by({:index => 'soft'}).sync() + ot: err("ReqlQueryLogicError", 'Expected type TABLE but found TABLE_SLICE:', [1]) + + # clean up + - cd: r.db('test').table_drop('test1') + ot: partial({'tables_dropped':1}) + - cd: r.db('test').table_drop('test1soft') + ot: partial({'tables_dropped':1}) + diff --git a/ext/librethinkdbxx/test/upstream/mutation/update.yaml b/ext/librethinkdbxx/test/upstream/mutation/update.yaml new file mode 100644 index 00000000..3cde0f34 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/update.yaml @@ -0,0 +1,170 @@ +desc: Tests updates of selections +table_variable_name: tbl, tbl2 +tests: + + # Set up some data + - py: tbl.insert([{'id':i} for i in xrange(100)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id:i}); + } + return res; + }()) + rb: tbl.insert((0...100).map{ |i| { :id => i } }) + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100}) + + - cd: tbl.count() + ot: 100 + + - py: tbl2.insert([{'id':i, 'foo':{'bar':i}} for i in xrange(100)]) + js: | + tbl2.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id:i,foo:{bar:i}}); + } + return res; + }()) + rb: tbl2.insert((0...100).map{ |i| { :id => i, :foo => { :bar => i } } }) + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100}) + + - cd: tbl2.count() + ot: 100 + + # Identity + - py: tbl.get(12).update(lambda row:row) + js: tbl.get(12).update(function(row) { return row; }) + rb: tbl.get(12).update{ |row| row} + ot: {'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # Soft durability point update + - py: tbl.get(12).update(lambda row:{'a':row['id'] + 1}, durability='soft') + js: tbl.get(12).update(function(row) { return {'a':row('id').add(1)}; }, {durability:'soft'}) + rb: tbl.get(12).update({ :durability => 'soft' }) { |row| { :a => row[:id] + 1 } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl.get(12) + ot: {'id':12, 'a':13} + + # Hard durability point update + - py: tbl.get(12).update(lambda row:{'a':row['id'] + 2}, durability='hard') + js: tbl.get(12).update(function(row) { return {'a':row('id').add(2)}; }, {durability:'hard'}) + rb: tbl.get(12).update({ :durability => 'hard' }) { |row| { :a => row[:id] + 2 } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl.get(12) + ot: {'id':12, 'a':14} + + # Wrong durability point update + - py: tbl.get(12).update(lambda row:{'a':row['id'] + 3}, durability='wrong') + js: tbl.get(12).update(function(row) { return {'a':row('id').add(3)}; }, {durability:'wrong'}) + rb: tbl.get(12).update({ :durability => 'wrong' }) { |row| { :a => row[:id] + 3 } } + ot: err('ReqlQueryLogicError', 'Durability option `wrong` unrecognized (options are "hard" and "soft").', [0]) + + - cd: tbl.get(12) + ot: {'id':12, 'a':14} + + # Point update + - py: tbl.get(12).update(lambda row:{'a':row['id']}) + js: tbl.get(12).update(function(row) { return {'a':row('id')}; }) + rb: tbl.get(12).update{ |row| { :a => row[:id] } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl.get(12) + ot: {'id':12, 'a':12} + + # undo the point update + - cd: tbl.get(12).update({'a':r.literal()}) + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # Update selection of table + + - py: tbl.between(10, 20).update(lambda row:{'a':row['id']}) + js: tbl.between(10, 20).update(function(row) { return {'a':row('id')}; }) + rb: tbl.between(10, 20).update{ |row| { :a => row[:id] } } + ot: {'deleted':0.0,'replaced':10,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - py: tbl.filter(lambda row:(row['id'] >= 10) & (row['id'] < 20)).update(lambda row:{'a':row['id']}) + js: tbl.filter(function(row) { return row('id').ge(10).and(row('id').lt(20))}).update(function(row) { return {'a':row('id')}; }) + rb: tbl.filter{ |row| (row[:id] >= 10).and(row[:id] < 20) }.update{ |row| { :a => row[:id] } } + ot: {'deleted':0.0,'replaced':0.0,'unchanged':10,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - py: tbl.filter(lambda row:(row['id'] >= 10) & (row['id'] < 20)).update(lambda row:{'b':row['id']}) + js: tbl.filter(function(row) { return row('id').ge(10).and(row('id').lt(20))}).update(function(row) { return {'b':row('id')}; }) + rb: tbl.filter{ |row| (row[:id] >= 10).and(row[:id] < 20) }.update{ |row| { :b => row[:id] } } + ot: {'deleted':0.0,'replaced':10,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # now undo that update + - cd: tbl.between(10, 20).update({'a':r.literal()}) + ot: {'deleted':0.0,'replaced':10,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # trying to change pkey of a document + - cd: tbl.get(1).update({'id':2,'d':1}) + ot: {'first_error':"Primary key `id` cannot be changed (`{\n\t\"id\":\t1\n}` -> `{\n\t\"d\":\t1,\n\t\"id\":\t2\n}`).",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':1,'skipped':0.0,'inserted':0.0} + + # check r.row, static value and otherwise + - py: tbl.get(1).update({'id':r.row['id'],'d':'b'}) + js: tbl.get(1).update({'id':r.row('id'),'d':'b'}) + rb: tbl.get(1).update{ |row| { :id => row[:id], :d => 'b' } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # I don't we don't need the merge, just testing r.row just in case + - cd: tbl.get(1).update(r.row.merge({'d':'b'})) + rb: tbl.get(1).update{ |row| row.merge({'d':'b'}) } + ot: {'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # test atomicity constraints (positive and negative test) + - cd: tbl.get(1).update({'d':r.js('5')}) + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - cd: tbl.get(1).update({'d':tbl.nth(0)}) + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - py: tbl.get(1).update({'d':r.js('5')}, non_atomic=True) + js: tbl.get(1).update({'d':r.js('5')}, {'nonAtomic':true}) + rb: tbl.get(1).update({ :d => r.js('5') }, { :non_atomic => true }) + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - js: tbl.get(1).update({}, 'foo') + ot: err('ReqlCompileError', 'Expected 1 argument (not including options) but found 2.') + + - js: tbl.get(1).update({}, {'foo':'bar'}) + ot: err('ReqlCompileError', 'Unrecognized optional argument `foo`.') + + # Update whole table + - py: tbl.update(lambda row:{'a':row['id']}) + js: tbl.update(function(row) { return {'a':row('id')}; }) + rb: tbl.update{ |row| { :a => row['id'] } } + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # undo the update on the whole table + - cd: tbl.update({'a':r.literal()}) + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # recursive merge + - cd: tbl2.update({'foo':{'bar':2}}) + ot: {'deleted':0.0,'replaced':99,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.update({'foo':r.literal({'bar':2})}) + ot: {'deleted':0.0,'replaced':0,'unchanged':100,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - rb: tbl2.update{|row| {'foo':r.literal({'bar':2})}} + ot: {'deleted':0.0,'replaced':0,'unchanged':100,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.order_by('id').nth(0) + ot: {'id':0,'foo':{'bar':2}} + + - cd: tbl2.update({'foo':{'buzz':2}}) + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.order_by('id').nth(0) + ot: {'id':0,'foo':{'bar':2,'buzz':2}} + + - cd: tbl2.update({'foo':r.literal(1)}) + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.order_by('id').nth(0) + ot: {'id':0,'foo':1} + diff --git a/ext/librethinkdbxx/test/upstream/parsePolyglot.py b/ext/librethinkdbxx/test/upstream/parsePolyglot.py new file mode 100755 index 00000000..ce048d1b --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/parsePolyglot.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os, re, sys + +# == globals + +printDebug = False + +try: + unicode +except NameError: + unicode = str + +# == + +class yamlValue(unicode): + linenumber = None + def __new__(cls, value, linenumber=None): + if isinstance(value, unicode): + real = unicode.__new__(cls, value) + else: + real = unicode.__new__(cls, value, "utf-8") + if linenumber is not None: + real.linenumber = int(linenumber) + return real + + def __repr__(self): + real = super(yamlValue, self).__repr__() + return real.lstrip('u') + +def parseYAML(source): + + def debug(message): + if printDebug and message: + message = str(message).rstrip() + if message: + print(message) + sys.stdout.flush() + + commentLineRegex = re.compile('^\s*#') + yamlLineRegex = re.compile('^(?P *)((?P- +)(?P.*)|((?P[\w\.]+)(?P: *))?(?P.*))\s*$') + + def parseYAML_inner(source, indent): + returnItem = None + + for linenumber, line in source: + if line == '': # no newline, so EOF + break + + debug('line %d (%d):%s' % (linenumber, indent, line)) + + if line.strip() == '' or commentLineRegex.match(line): # empty or comment line, ignore + debug('\tempty/comment line') + continue + + # - parse line + + parsedLine = yamlLineRegex.match(line) + if not parsedLine: + raise Exception('Unparseable YAML line %d: %s' % (linenumber, line.rstrip())) + + lineIndent = len(parsedLine.group('indent')) + lineItemMarker = parsedLine.group('itemMarker') + lineKey = parsedLine.group('key') or '' + lineKeyExtra = parsedLine.group('keyExtra') or '' + lineContent = (parsedLine.group('content') or parsedLine.group('itemContent') or '').strip() + + # - handle end-of-sections + if lineIndent < indent: + # we have dropped out of this item, push back the line and return what we have + source.send((linenumber, line)) + debug('\tout one level') + return returnItem + + # - array item + if lineItemMarker: + debug('\tarray item') + # item in an array + if returnItem is None: + debug('\tnew array, indent is %d' % lineIndent) + returnItem = [] + indent = lineIndent + elif not isinstance(returnItem, list): + raise Exception('Bad YAML, got a list item while working on a %s on line %d: %s' % (returnItem.__class__.__name__, linenumber, line.rstrip())) + indentLevel = lineIndent + len(lineItemMarker) + source.send((linenumber, (' ' * (indentLevel) )+ lineContent)) + returnItem += [parseYAML_inner(source=source, indent=indent + 1)] + + # - dict item + elif lineKey: + debug('\tdict item') + if returnItem is None: + debug('\tnew dict, indent is %d' % lineIndent) + # new dict + returnItem = {} + indent = lineIndent + elif not isinstance(returnItem, dict): + raise Exception('Bad YAML, got a dict value while working on a %s on line %d: %s' % (returnItem.__class__.__name__, linenumber, line.rstrip())) + indentLevel = lineIndent + len(lineKey) + len(lineKeyExtra) + source.send((linenumber, (' ' * indentLevel) + lineContent)) + returnItem[lineKey] = parseYAML_inner(source=source, indent=indent + 1) + + # - data - one or more lines of text + else: + debug('\tvalue') + if returnItem is None: + returnItem = yamlValue('', linenumber) + if lineContent.strip() in ('|', '|-', '>'): + continue # yaml multiline marker + elif not isinstance(returnItem, yamlValue): + raise Exception('Bad YAML, got a value while working on a %s on line %d: %s' % (returnItem.__class__.__name__, linenumber, line.rstrip())) + if returnItem: + returnItem = yamlValue(returnItem + "\n" + lineContent, returnItem.linenumber) # str subclasses are not fun + else: + returnItem = yamlValue(lineContent, linenumber) + return returnItem + + def parseYAML_generator(source): + if hasattr(source, 'capitalize'): + if os.path.isfile(source): + source = open(source, 'r') + else: + source = source.splitlines(True) + elif hasattr(source, 'readlines'): + pass # the for loop will already work + + backlines = [] + for linenumber, line in enumerate(source): + backline = None + usedLine = False + while usedLine is False or backlines: + if backlines: + backline = yield backlines.pop() + else: + usedLine = True + backline = yield (linenumber + 1, line) + while backline: # loops returning None for every send() + assert isinstance(backline, tuple) + assert isinstance(backline[0], int) + backlines.append(backline) + backline = yield None + + return parseYAML_inner(parseYAML_generator(source), indent=0) + +if __name__ == '__main__': + import optparse, pprint + + parser = optparse.OptionParser() + parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, help="print debug information") + (options, args) = parser.parse_args() + printDebug = options.debug + + if len(args) < 1: + parser.error('%s needs files to process' % os.path.basename(__file__)) + + for filePath in args: + if not os.path.isfile(filePath): + sys.exit('target is not an existing file: %s' % os.path.basename(__file__)) + + for filePath in args: + print('=== %s' % filePath) + pprint.pprint(parseYAML(filePath)) diff --git a/ext/librethinkdbxx/test/upstream/polymorphism.yaml b/ext/librethinkdbxx/test/upstream/polymorphism.yaml new file mode 100644 index 00000000..14817289 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/polymorphism.yaml @@ -0,0 +1,33 @@ +desc: Tests that manipulation data in tables +table_variable_name: tbl +tests: + + - def: obj = r.expr({'id':0,'a':0}) + + - py: tbl.insert([{'id':i, 'a':i} for i in xrange(3)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 3; i++) { + res.push({id:i, 'a':i}); + } + return res; + }()) + rb: tbl.insert((0..2).map{ |i| { :id => i, :a => i } }) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':3}) + + # Polymorphism + - cd: + - tbl.merge({'c':1}).nth(0) + - obj.merge({'c':1}) + ot: ({'id':0,'c':1,'a':0}) + + - cd: + - tbl.without('a').nth(0) + - obj.without('a') + ot: ({'id':0}) + + - cd: + - tbl.pluck('a').nth(0) + - obj.pluck('a') + ot: ({'a':0}) diff --git a/ext/librethinkdbxx/test/upstream/random.yaml b/ext/librethinkdbxx/test/upstream/random.yaml new file mode 100644 index 00000000..59e2cde6 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/random.yaml @@ -0,0 +1,180 @@ +desc: Tests randomization functions +tests: + +# Test sample + - cd: r.expr([1,2,3]).sample(3).distinct().count() + ot: 3 + - cd: r.expr([1,2,3]).sample(3).count() + ot: 3 + - cd: r.expr([1,2,3,4,5,6]).sample(3).distinct().count() + ot: 3 + - cd: r.expr([1,2,3]).sample(4).distinct().count() + ot: 3 + - rb: r.expr([[1,2,3], 2]).do{|x| x[0].sample(x[1])}.distinct().count() + ot: 2 + - cd: r.expr([1,2,3]).sample(-1) + ot: err('ReqlQueryLogicError', 'Number of items to sample must be non-negative, got `-1`.', [0]) + - cd: r.expr(1).sample(1) + ot: err('ReqlQueryLogicError', 'Cannot convert NUMBER to SEQUENCE', [0]) + - cd: r.expr({}).sample(1) + ot: err('ReqlQueryLogicError', 'Cannot convert OBJECT to SEQUENCE', [0]) + +# Test r.random with floating-point values + # These expressions should be equivalent + - py: + - r.random().do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(1, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(0, 1, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(1, 0, float=True).do(lambda x:r.and_(x.le(1), x.gt(0))) + - r.random(r.expr(0), 1, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(1, r.expr(0), float=True).do(lambda x:r.and_(x.le(1), x.gt(0))) + - r.random(r.expr(1), r.expr(0), float=True).do(lambda x:r.and_(x.le(1), x.gt(0))) + ot: True + + # Single-argument + - py: + - r.random(0.495, float=True).do(lambda x:r.and_(x.ge(0), x.lt(0.495))) + - r.random(-0.495, float=True).do(lambda x:r.and_(x.le(0), x.gt(-0.495))) + - r.random(1823756.24, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1823756.24))) + - r.random(-1823756.24, float=True).do(lambda x:r.and_(x.le(0), x.gt(-1823756.24))) + ot: True + + # Non-zero-based random numbers + - py: + - r.random(10.5, 20.153, float=True).do(lambda x:r.and_(x.ge(10.5), x.lt(20.153))) + - r.random(20.153, 10.5, float=True).do(lambda x:r.and_(x.le(20.153), x.gt(10.5))) + - r.random(31415926.1, 31415926, float=True).do(lambda x:r.and_(x.le(31415926.1), x.gt(31415926))) + ot: True + + # Negative random numbers + - py: + - r.random(-10.5, 20.153, float=True).do(lambda x:r.and_(x.ge(-10.5), x.lt(20.153))) + - r.random(-20.153, -10.5, float=True).do(lambda x:r.and_(x.ge(-20.153), x.lt(-10.5))) + - r.random(-31415926, -31415926.1, float=True).do(lambda x:r.and_(x.le(-31415926), x.gt(-31415926.1))) + ot: True + + # There is a very small chance of collision here + - py: + - r.expr([r.random(), r.random()]).distinct().count() + - r.expr([r.random(1, float=True), r.random(1, float=True)]).distinct().count() + - r.expr([r.random(0, 1, float=True), r.random(0, 1, float=True)]).distinct().count() + ot: 2 + + # Zero range random + - py: + - r.random(0, float=True).eq(0) + - r.random(5, 5, float=True).eq(5) + - r.random(-499384756758, -499384756758, float=True).eq(-499384756758) + - r.random(-93.94757, -93.94757, float=True).eq(-93.94757) + - r.random(294.69148, 294.69148, float=True).eq(294.69148) + ot: True + + # Test limits of doubles + - def: + py: float_max = sys.float_info.max + js: float_max = Number.MAX_VALUE + rb: float_max = Float::MAX + - def: + py: float_min = sys.float_info.min + js: float_min = Number.MIN_VALUE + rb: float_min = Float::MIN + - py: + - r.random(-float_max, float_max, float=True).do(lambda x:r.and_(x.ge(-float_max), x.lt(float_max))) + - r.random(float_max, -float_max, float=True).do(lambda x:r.and_(x.le(float_max), x.gt(-float_max))) + - r.random(float_min, float_max, float=True).do(lambda x:r.and_(x.ge(float_min), x.lt(float_max))) + - r.random(float_min, -float_max, float=True).do(lambda x:r.and_(x.le(float_min), x.gt(-float_max))) + - r.random(-float_min, float_max, float=True).do(lambda x:r.and_(x.ge(-float_min), x.lt(float_max))) + - r.random(-float_min, -float_max, float=True).do(lambda x:r.and_(x.le(-float_min), x.gt(-float_max))) + ot: True + +# Test r.random with integer values + - def: + py: upper_limit = 2**53 - 1 + js: upper_limit = Math.pow(2,53) - 1 + rb: upper_limit = 2**53 - 1 + - def: + py: lower_limit = 1 - (2**53) + js: lower_limit = 1 - Math.pow(2,53) + rb: lower_limit = 1 - (2**53) + # These expressions should be equivalent + - py: + - r.random(256).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(0, 256).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(r.expr(256)).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(r.expr(0), 256).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(0, r.expr(256)).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(r.expr(0), r.expr(256)).do(lambda x:r.and_(x.ge(0), x.lt(256))) + ot: True + + # Non-zero-based random numbers + - py: + - r.random(10, 20).do(lambda x:r.and_(x.ge(10), x.lt(20))) + - r.random(9347849, 120937493).do(lambda x:r.and_(x.ge(9347849), x.lt(120937493))) + js: + - r.random(10, 20).do(function(x){return r.and(x.ge(10), x.lt(20))}) + - r.random(9347849, 120937493).do(function(x){return r.and(x.ge(9347849), x.lt(120937493))}) + rb: + - r.random(10, 20).do{|x| r.and(x.ge(10), x.lt(20))} + - r.random(9347849, 120937493).do{|x| r.and(x.ge(9347849), x.lt(120937493))} + ot: True + + # Negative random numbers + - py: + - r.random(-10, 20).do(lambda x:r.and_(x.ge(-10), x.lt(20))) + - r.random(-20, -10).do(lambda x:r.and_(x.ge(-20), x.lt(-10))) + - r.random(-120937493, -9347849).do(lambda x:r.and_(x.ge(-120937493), x.lt(-9347849))) + js: + - r.random(-10, 20).do(function(x){return r.and(x.ge(-10), x.lt(20))}) + - r.random(-20, -10).do(function(x){return r.and(x.ge(-20), x.lt(-10))}) + - r.random(-120937493, -9347849).do(function(x){return r.and(x.ge(-120937493), x.lt(-9347849))}) + rb: + - r.random(-10, 20).do{|x| r.and(x.ge(-10), x.lt(20))} + - r.random(-20, -10).do{|x| r.and(x.ge(-20), x.lt(-10))} + - r.random(-120937493, -9347849).do{|x| r.and(x.ge(-120937493), x.lt(-9347849))} + ot: True + + # There is a very small chance of collision here + - cd: r.expr([r.random(upper_limit), r.random(upper_limit)]).distinct().count() + ot: 2 + - py: r.expr([upper_limit,upper_limit]).map(lambda x:r.random(x)).distinct().count() + js: r.expr([upper_limit,upper_limit]).map(function(x){return r.random(x)}).distinct().count() + rb: r.expr([upper_limit,upper_limit]).map{|x| r.random(x)}.distinct().count() + ot: 2 + + # Error cases + + # Non-integer limits + - cd: r.random(-0.5) + ot: err("ReqlQueryLogicError", "Upper bound (-0.5) could not be safely converted to an integer.", []) + - cd: r.random(0.25) + ot: err("ReqlQueryLogicError", "Upper bound (0.25) could not be safely converted to an integer.", []) + - cd: r.random(-10, 0.75) + ot: err("ReqlQueryLogicError", "Upper bound (0.75) could not be safely converted to an integer.", []) + - cd: r.random(-120549.25, 39458) + ot: err("ReqlQueryLogicError", "Lower bound (-120549.25) could not be safely converted to an integer.", []) + - cd: r.random(-6.5, 8.125) + ot: err("ReqlQueryLogicError", "Lower bound (-6.5) could not be safely converted to an integer.", []) + + # Forced integer random with no bounds + - py: r.random(float=False) + js: r.random({float:false}) + rb: r.random({:float => false}) + ot: err("ReqlQueryLogicError", "Generating a random integer requires one or two bounds.", []) + + # Lower bound not less than upper bound + - cd: r.random(0) + ot: err("ReqlQueryLogicError", "Lower bound (0) is not less than upper bound (0).", []) + - cd: r.random(0, 0) + ot: err("ReqlQueryLogicError", "Lower bound (0) is not less than upper bound (0).", []) + - cd: r.random(515, 515) + ot: err("ReqlQueryLogicError", "Lower bound (515) is not less than upper bound (515).", []) + - cd: r.random(-956, -956) + ot: err("ReqlQueryLogicError", "Lower bound (-956) is not less than upper bound (-956).", []) + - cd: r.random(-10) + ot: err("ReqlQueryLogicError", "Lower bound (0) is not less than upper bound (-10).", []) + - cd: r.random(20, 2) + ot: err("ReqlQueryLogicError", "Lower bound (20) is not less than upper bound (2).", []) + - cd: r.random(2, -20) + ot: err("ReqlQueryLogicError", "Lower bound (2) is not less than upper bound (-20).", []) + - cd: r.random(1456, 0) + ot: err("ReqlQueryLogicError", "Lower bound (1456) is not less than upper bound (0).", []) diff --git a/ext/librethinkdbxx/test/upstream/range.yaml b/ext/librethinkdbxx/test/upstream/range.yaml new file mode 100644 index 00000000..da9a575f --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/range.yaml @@ -0,0 +1,53 @@ +desc: Tests RQL range generation +tests: + - cd: r.range().type_of() + ot: 'STREAM' + + - cd: r.range().limit(4) + ot: [0, 1, 2, 3] + + - cd: r.range(4) + ot: [0, 1, 2, 3] + + - cd: r.range(2, 5) + ot: [2, 3, 4] + + - cd: r.range(0) + ot: [] + + - cd: r.range(5, 2) + ot: [] + + - cd: r.range(-5, -2) + ot: [-5, -4, -3] + + - cd: r.range(-5, 2) + ot: [-5, -4, -3, -2, -1, 0, 1] + + - cd: r.range(2, 5, 8) + ot: err("ReqlCompileError", "Expected between 0 and 2 arguments but found 3.", []) + + - cd: r.range("foo") + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + # Using 9007199254740994 instead of 9007199254740993 due to #2157 + - cd: r.range(9007199254740994) + ot: err_regex("ReqlQueryLogicError", "Number not an integer \\(>2\\^53\\). 9007199254740994", []) + + - cd: r.range(-9007199254740994) + ot: err_regex("ReqlQueryLogicError", "Number not an integer \\(<-2\\^53\\). -9007199254740994", []) + + - cd: r.range(0.5) + ot: err_regex("ReqlQueryLogicError", "Number not an integer. 0\\.5", []) + + - cd: r.range().count() + ot: err("ReqlQueryLogicError", "Cannot use an infinite stream with an aggregation function (`reduce`, `count`, etc.) or coerce it to an array.", []) + + - cd: r.range().coerce_to("ARRAY") + ot: err("ReqlQueryLogicError", "Cannot use an infinite stream with an aggregation function (`reduce`, `count`, etc.) or coerce it to an array.", []) + + - cd: r.range().coerce_to("OBJECT") + ot: err("ReqlQueryLogicError", "Cannot use an infinite stream with an aggregation function (`reduce`, `count`, etc.) or coerce it to an array.", []) + + - cd: r.range(4).count() + ot: 4 diff --git a/ext/librethinkdbxx/test/upstream/regression/1001.yaml b/ext/librethinkdbxx/test/upstream/regression/1001.yaml new file mode 100644 index 00000000..15327350 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1001.yaml @@ -0,0 +1,20 @@ +desc: 1001 (null + between + sindexes) +table_variable_name: tbl +tests: + - cd: tbl.insert({'a':null}) + rb: tbl.insert({:a => null}) + - cd: tbl.index_create('a') + - cd: tbl.index_create('b') + - cd: tbl.index_wait().pluck('index', 'ready') + + - cd: tbl.between(r.minval, r.maxval).count() + ot: 1 + - py: tbl.between(r.minval, r.maxval, index='a').count() + js: tbl.between(r.minval, r.maxval, {index:'a'}).count() + rb: tbl.between(r.minval, r.maxval, :index => 'a').count() + ot: 0 + - py: tbl.between(r.minval, r.maxval, index='b').count() + js: tbl.between(r.minval, r.maxval, {index:'b'}).count() + rb: tbl.between(r.minval, r.maxval, :index => 'b').count() + ot: 0 + diff --git a/ext/librethinkdbxx/test/upstream/regression/1005.yaml b/ext/librethinkdbxx/test/upstream/regression/1005.yaml new file mode 100644 index 00000000..09ab329c --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1005.yaml @@ -0,0 +1,19 @@ +desc: Regression test for issue #1005. +tests: + - py: r.expr(str(r.table_list())) + ot: "r.table_list()" + + - py: r.expr(str(r.table_create('a'))) + ot: "r.table_create('a')" + + - py: r.expr(str(r.table_drop('a'))) + ot: "r.table_drop('a')" + + - py: r.expr(str(r.db('a').table_list())) + ot: "r.db('a').table_list()" + + - py: r.expr(str(r.db('a').table_create('a'))) + ot: "r.db('a').table_create('a')" + + - py: r.expr(str(r.db('a').table_drop('a'))) + ot: "r.db('a').table_drop('a')" diff --git a/ext/librethinkdbxx/test/upstream/regression/1023.yaml b/ext/librethinkdbxx/test/upstream/regression/1023.yaml new file mode 100644 index 00000000..9c017e04 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1023.yaml @@ -0,0 +1,65 @@ +desc: Tests key sorting of all usable types in primary indexes +table_variable_name: tbl +tests: + + + # Test key sorting + - def: + py: binary_a = r.binary(b'') + rb: binary_a = r.binary('') + js: binary_a = Buffer('') + + - def: + py: binary_b = r.binary(b'5aurhbviunr') + rb: binary_b = r.binary('5aurhbviunr') + js: binary_b = Buffer('5aurhbviunr') + + # Define a set of rows in order of increasing sindex keys + - def: + cd: trows = [{'num':0,'id':[0]}, + {'num':1,'id':[1, 2, 3, 4, 0]}, + {'num':2,'id':[1, 2, 3, 4, 4]}, + {'num':3,'id':[1, 2, 3, 4, 4, 5]}, + {'num':4,'id':[1, 2, 3, 4, 8, 1]}, + {'num':5,'id':[1, 3, r.epoch_time(0)]}, + {'num':6,'id':[1, 3, r.epoch_time(0), r.epoch_time(0)]}, + {'num':7,'id':[1, 3, r.epoch_time(0), r.epoch_time(1)]}, + {'num':8,'id':[1, 4, 3, 4, 8, 2]}, + {'num':9,'id':False}, + {'num':10,'id':True}, + {'num':11,'id':-500}, + {'num':12,'id':500}, + {'num':13,'id':binary_a}, + {'num':14,'id':binary_b}, + {'num':15,'id':r.epoch_time(0)}, + {'num':16,'id':''}, + {'num':17,'id':' str'}] + + - def: + cd: expected = r.range(tbl.count()).coerce_to('array') + + - cd: tbl.insert(trows)['inserted'] + js: tbl.insert(trows)('inserted') + ot: 18 + + - rb: tbl.order_by({:index => 'id'}).map{|row| row['num']}.coerce_to('array').eq(expected) + js: tbl.order_by({index:'id'}).map(r.row('num')).coerce_to('array').eq(expected) + py: tbl.order_by(index='id').map(r.row['num']).coerce_to('array').eq(expected) + ot: true + + # Test minval and maxval + - rb: tbl.order_by(:index => 'id').between(r.minval, r.maxval).map{|x| x['num']}.coerce_to('array').eq(expected) + js: tbl.order_by({index:'id'}).between(r.minval, r.maxval).map(r.row('num')).coerce_to('array').eq(expected) + py: tbl.order_by(index='id').between(r.minval, r.maxval).map(r.row['num']).coerce_to('array').eq(expected) + ot: true + + - py: tbl.order_by(index='id').between([1,2,3,4,4],[1,2,3,5]).map(r.row['num']).coerce_to('array') + js: tbl.order_by({index:'id'}).between([1,2,3,4,4],[1,2,3,5]).map(r.row('num')).coerce_to('array') + rb: tbl.order_by(:index => 'id').between([1,2,3,4,4],[1,2,3,5]).map{|x| x['num']}.coerce_to('array') + ot: [2,3,4] + + - py: tbl.order_by(index='id').between([1,2,3,4,4,r.minval],[1,2,3,4,4,r.maxval]).map(r.row['num']).coerce_to('array') + js: tbl.order_by({index:'id'}).between([1,2,3,4,4,r.minval],[1,2,3,4,4,r.maxval]).map(r.row('num')).coerce_to('array') + rb: tbl.order_by(:index => 'id').between([1,2,3,4,4,r.minval],[1,2,3,4,4,r.maxval]).map{|x| x['num']}.coerce_to('array') + ot: [3] + diff --git a/ext/librethinkdbxx/test/upstream/regression/1081.yaml b/ext/librethinkdbxx/test/upstream/regression/1081.yaml new file mode 100644 index 00000000..ef7d2fc4 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1081.yaml @@ -0,0 +1,39 @@ +desc: 1081 union two streams +tests: + + - rb: r.db('test').table_create('t1081') + def: t = r.db('test').table('t1081') + + - rb: t.insert([{'id':0}, {'id':1}]) + + - rb: r([]).union([]).typeof + ot: ("ARRAY") + - rb: t.union(t).typeof + ot: ("STREAM") + - rb: t.union([]).typeof + ot: ("STREAM") + + - rb: r.db('test').table_drop('t1081') + + - rb: r.table_create('1081') + ot: partial({'tables_created':1}) + + - rb: r.table('1081').insert({:password => 0})[:inserted] + ot: 1 + + - rb: r.table('1081').index_create('password') + ot: ({'created':1}) + - rb: r.table('1081').index_wait('password').pluck('index', 'ready') + ot: ([{'ready':True, 'index':'password'}]) + + - rb: r.table('1081').get_all(0, :index => 'password').typeof + ot: ("SELECTION") + - rb: r.table('1081').get_all(0, :index => 'password').without('id').typeof + ot: ("STREAM") + - rb: r.table('1081').get_all(0, 0, :index => 'password').typeof + ot: ("SELECTION") + - rb: r.table('1081').get_all(0, 0, :index => 'password').without('id').typeof + ot: ("STREAM") + + - rb: r.table_drop('1081') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/regression/1132.yaml b/ext/librethinkdbxx/test/upstream/regression/1132.yaml new file mode 100644 index 00000000..6208b5d6 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1132.yaml @@ -0,0 +1,4 @@ +desc: 1132 JSON duplicate key +tests: + - cd: r.json('{"a":1,"a":2}') + ot: err("ReqlQueryLogicError", "Duplicate key \"a\" in JSON.", []) diff --git a/ext/librethinkdbxx/test/upstream/regression/1133.yaml b/ext/librethinkdbxx/test/upstream/regression/1133.yaml new file mode 100644 index 00000000..853a3566 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1133.yaml @@ -0,0 +1,19 @@ +desc: Regression tests for issue #1133, which concerns circular references in the drivers. + +tests: + - def: a = {} + - def: b = {'a':a} + - def: a['b'] = b + + - cd: r.expr(a) + ot: + cd: err('ReqlDriverCompileError', 'Nesting depth limit exceeded.', []) + rb: err('ReqlDriverCompileError', 'Maximum expression depth exceeded (you can override this with `r.expr(X, MAX_DEPTH)`).', []) + + - cd: r.expr({'a':{'a':{'a':{'a':{'a':{'a':{'a':{}}}}}}}}, 7) + ot: + cd: err('ReqlDriverCompileError', 'Nesting depth limit exceeded.', []) + rb: err('ReqlDriverCompileError', 'Maximum expression depth exceeded (you can override this with `r.expr(X, MAX_DEPTH)`).', []) + + - cd: r.expr({'a':{'a':{'a':{'a':{'a':{'a':{'a':{}}}}}}}}, 10) + ot: ({'a':{'a':{'a':{'a':{'a':{'a':{'a':{}}}}}}}}) diff --git a/ext/librethinkdbxx/test/upstream/regression/1155.yaml b/ext/librethinkdbxx/test/upstream/regression/1155.yaml new file mode 100644 index 00000000..08dbb6c7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1155.yaml @@ -0,0 +1,5 @@ +desc: 1155 -- Empty batched_replaces_t constructed +table_variable_name: tbl +tests: + - rb: tbl.insert([{:id => '2'}, {:id => '4'}])['inserted'] + ot: 2 diff --git a/ext/librethinkdbxx/test/upstream/regression/1179.yaml b/ext/librethinkdbxx/test/upstream/regression/1179.yaml new file mode 100644 index 00000000..1c04c051 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1179.yaml @@ -0,0 +1,26 @@ +desc: 1179 -- BRACKET term +table_variable_name: tbl +tests: + - js: r.expr([1])(r.expr(0)) + py: r.expr([1])[r.expr(0)] + rb: r.expr([1])[r.expr(0)] + ot: 1 + - js: r.expr({"foo":1})('foo') + ot: 1 + - js: r.expr([1])(0) + ot: 1 + - js: tbl.insert([{'id':42},{'id':4},{'id':89},{'id':6},{'id':43}]).pluck('inserted','first_error') + ot: ({'inserted':5}) + + # test [] grouped data semantics + - js: tbl.group('id')(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + - js: tbl.coerce_to('array').group('id')(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + + # test nth grouped data semantics + - js: tbl.group('id').nth(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + - js: tbl.coerce_to('array').group('id').nth(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + diff --git a/ext/librethinkdbxx/test/upstream/regression/1468.yaml b/ext/librethinkdbxx/test/upstream/regression/1468.yaml new file mode 100644 index 00000000..0b8cfc15 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1468.yaml @@ -0,0 +1,7 @@ +desc: 1468 -- Empty batched_replaces_t constructed +table_variable_name: tbl +tests: + - rb: tbl.insert([{}, {}, {}])['inserted'] + ot: (3) + - rb: tbl.replace(non_atomic:'true'){|row| r.js("{}")} + ot: ({"unchanged"=>0,"skipped"=>0,"replaced"=>0,"inserted"=>0,"first_error"=>"Cannot convert javascript `undefined` to ql::datum_t.","errors"=>3,"deleted"=>0}) diff --git a/ext/librethinkdbxx/test/upstream/regression/1789.yaml b/ext/librethinkdbxx/test/upstream/regression/1789.yaml new file mode 100644 index 00000000..5ad614b7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1789.yaml @@ -0,0 +1,22 @@ +desc: 1789 -- deleting a secondary index on a table that contains non-inline stored documents corrupts db +table_variable_name: tbl +tests: + - rb: tbl.insert({:foo => 'a', :data => "AAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}).pluck('inserted') + ot: ({'inserted':1}) + + - rb: tbl.index_create('foo') + ot: ({'created':1}) + + - rb: tbl.index_wait('foo').pluck('index', 'ready') + ot: ([{'index':'foo', 'ready':true}]) + + - rb: tbl.index_drop('foo') + ot: ({'dropped':1}) + + - rb: tbl.coerce_to('ARRAY').count() + ot: (1) + diff --git a/ext/librethinkdbxx/test/upstream/regression/2052.yaml b/ext/librethinkdbxx/test/upstream/regression/2052.yaml new file mode 100644 index 00000000..c7c62bc2 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2052.yaml @@ -0,0 +1,10 @@ +desc: 2052 -- Verify that the server rejects bogus global options. +tests: + - cd: r.expr(1) + runopts: + array_limit: 16 + ot: 1 + - cd: r.expr(1) + runopts: + obviously_bogus: 16 + ot: err("ReqlCompileError", "Unrecognized global optional argument `obviously_bogus`.", []) diff --git a/ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml b/ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml new file mode 100644 index 00000000..6f646bac --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml @@ -0,0 +1,45 @@ +desc: 2399 literal terms not removed under certain circumstances +table_variable_name: t +tests: + - rb: t.insert({}) + - rb: t.update({:a => {:b => r.literal({})}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{}}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => r.literal()}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a': {}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => {:c => {:d => r.literal({})}}}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{'c':{'d':{}}}}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => [[[{:c => r.literal({})}]]]}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':[[[{'c':{}}]]]}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => [r.literal()]}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':[]}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => {:a => 'A', :b => 'B', :c => 'C', :cc => r.literal(), :d => 'D'}}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{'a':'A', 'b':'B', 'c':'C', 'd':'D'}}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => {:a => 'A', :b => 'B', :c => 'C', :cc => r.literal('CC'), :d => 'D'}}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{'a':'A', 'b':'B', 'c':'C', 'cc':'CC', 'd':'D'}}}] + - rb: t.delete() + diff --git a/ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml b/ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml new file mode 100644 index 00000000..70f91761 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml @@ -0,0 +1,8 @@ +desc: 2639 -- Coroutine stacks should not overflow during the query compilation phase. +tests: + - rb: r.expr({id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}, 1000) + ot: partial({}) + + - rb: r.expr([[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],1000).and(nil) + ot: nil + diff --git a/ext/librethinkdbxx/test/upstream/regression/2696.yaml b/ext/librethinkdbxx/test/upstream/regression/2696.yaml new file mode 100644 index 00000000..3ca88b55 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2696.yaml @@ -0,0 +1,6 @@ +desc: Regression test for issue 2696, delete_at with end bounds. +tests: + - cd: r.expr([1,2,3,4]).delete_at(4,4) + ot: [1,2,3,4] + - cd: r.expr([]).delete_at(0,0) + ot: [] diff --git a/ext/librethinkdbxx/test/upstream/regression/2697.yaml b/ext/librethinkdbxx/test/upstream/regression/2697.yaml new file mode 100644 index 00000000..ba488172 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2697.yaml @@ -0,0 +1,31 @@ +desc: 2697 -- Array insert and splice operations don't check array size limit. +table_variable_name: tbl +tests: + # make enormous > 100,000 element array + - def: ten_l = r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - js: tbl.insert({'id':1, 'a':r.expr(ten_l).concatMap(function(l) { return ten_l }).concatMap(function(l) { return ten_l }).concatMap(function(l) { return ten_l }).concatMap(function(l) { return ten_l })}).pluck('first_error', 'inserted') + py: tbl.insert({'id':1, 'a':r.expr(ten_l).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11)))}).pluck('first_error', 'inserted') + rb: tbl.insert({'id':1, 'a':r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}}).pluck('first_error', 'inserted') + ot: ({'inserted':1}) + - cd: tbl.get(1).replace({'id':1, 'a':r.row['a'].splice_at(0, [2])}).pluck('first_error') + js: tbl.get(1).replace({'id':1, 'a':r.row('a').spliceAt(0, [2])}).pluck('first__error') + rb: tbl.get(1).replace{|old| {:id => 1, :a => old['a'].splice_at(0, [2])}}.pluck('first_error') + ot: ({'first_error':'Array over size limit `100000`.'}) + - cd: tbl.get(1)['a'].count() + js: tbl.get(1)('a').count() + ot: 100000 + - cd: tbl.get(1).replace({'id':1, 'a':r.row['a'].insert_at(0, [2])}).pluck('first_error') + js: tbl.get(1).replace({'id':1, 'a':r.row('a').insertAt(0, [2])}).pluck('first__error') + rb: tbl.get(1).replace{|old| {:id => 1, :a => old['a'].insert_at(0, [2])}}.pluck('first_error') + ot: ({'first_error':'Array over size limit `100000`.'}) + - cd: tbl.get(1)['a'].count() + js: tbl.get(1)('a').count() + ot: 100000 + - js: r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).spliceAt(0, [1]).count() + py: r.expr(ten_l).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).splice_at(0, [1]).count() + rb: r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.splice_at(0, [1]).count() + ot: err("ReqlResourceLimitError", "Array over size limit `100000`.", []) + - js: r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).insertAt(0, [1]).count() + py: r.expr(ten_l).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).insert_at(0, [1]).count() + rb: r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.insert_at(0, [1]).count() + ot: err("ReqlResourceLimitError", "Array over size limit `100000`.", []) diff --git a/ext/librethinkdbxx/test/upstream/regression/2709.yaml b/ext/librethinkdbxx/test/upstream/regression/2709.yaml new file mode 100644 index 00000000..a5ffe12d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2709.yaml @@ -0,0 +1,21 @@ +desc: 2709 -- Guarantee failed with [max_els >= min_els] +table_variable_name: tbl +tests: + - py: tbl.insert([{'result':i} for i in range(1,1000)]).pluck('first_error', 'inserted') + runopts: + min_batch_rows: 10 + max_batch_rows: 13 + ot: ({'inserted':999}) + + - py: tbl.map(lambda thing:'key').count() + runopts: + min_batch_rows: 10 + max_batch_rows: 13 + ot: (999) + + - py: tbl.map(lambda thing:'key').count() + runopts: + min_batch_rows: 10 + max_batch_rows: 13 + ot: (999) + diff --git a/ext/librethinkdbxx/test/upstream/regression/2710.yaml b/ext/librethinkdbxx/test/upstream/regression/2710.yaml new file mode 100644 index 00000000..ac5fa738 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2710.yaml @@ -0,0 +1,6 @@ +desc: Test pseudo literal strings in JSON. +tests: + - js: r.expr({"a":{"b":1, "c":2}}).merge(r.json('{"a":{"$reql_'+'type$":"LITERAL", "value":{"b":2}}}')) + py: r.expr({"a":{"b":1, "c":2}}).merge(r.json('{"a":{"$reql_type$":"LITERAL", "value":{"b":2}}}')) + rb: r.expr({:a => {:b => 1, :c => 2}}).merge(r.json('{"a":{"$reql_type$":"LITERAL", "value":{"b":2}}}')) + ot: ({'a':{'b':2}}) diff --git a/ext/librethinkdbxx/test/upstream/regression/2766.yaml b/ext/librethinkdbxx/test/upstream/regression/2766.yaml new file mode 100644 index 00000000..ffb562fb --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2766.yaml @@ -0,0 +1,25 @@ +desc: Stop people treating ptypes as objects +tests: + - cd: r.now()['epoch_time'] + js: r.now()('epoch_time') + ot: err("ReqlQueryLogicError", "Cannot call `bracket` on objects of type `PTYPE